Media caches cleanups (#14087)

src/OrchardCore.Cms.Web/appsettings.json

@@ -41,6 +41,8 @@
     //  "SupportedSizes": [ 16, 32, 50, 100, 160, 240, 480, 600, 1024, 2048 ],
     //  "MaxBrowserCacheDays": 30,
     //  "MaxCacheDays": 365,
+    //  "ResizedCacheMaxStale": "01:00:00", // The time before a staled item is removed from the resized media cache, if not provided there is no cleanup.
+    //  "RemoteCacheMaxStale": "01:00:00", // The time before a staled item is removed from the remote media cache, if not provided there is no cleanup.
     //  "MaxFileSize": 30000000,
     //  "CdnBaseUrl": "https://your-cdn.com",
     //  "AssetsRequestPath": "/media",

src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs

@@ -87,6 +87,8 @@ public void Configure(MediaOptions options)
 
             options.MaxBrowserCacheDays = section.GetValue("MaxBrowserCacheDays", DefaultMaxBrowserCacheDays);
             options.MaxCacheDays = section.GetValue("MaxCacheDays", DefaultMaxCacheDays);
+            options.ResizedCacheMaxStale = section.GetValue<TimeSpan?>(nameof(options.ResizedCacheMaxStale));
+            options.RemoteCacheMaxStale = section.GetValue<TimeSpan?>(nameof(options.RemoteCacheMaxStale));
             options.MaxFileSize = section.GetValue("MaxFileSize", DefaultMaxFileSize);
             options.CdnBaseUrl = section.GetValue("CdnBaseUrl", string.Empty).TrimEnd('/').ToLower();
             options.AssetsRequestPath = section.GetValue("AssetsRequestPath", DefaultAssetsRequestPath);

src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs

@@ -0,0 +1,130 @@
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using OrchardCore.BackgroundTasks;
+using OrchardCore.Environment.Shell;
+using OrchardCore.Media.Core;
+using OrchardCore.Modules;
+
+namespace OrchardCore.Media.Services;
+
+[BackgroundTask(Schedule = "30 0 * * *", Description = "'Remote media cache cleanup.")]
+public class RemoteMediaCacheBackgroundTask : IBackgroundTask
+{
+    private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true };
+
+    private readonly IMediaFileStore _mediaFileStore;
+    private readonly ILogger _logger;
+
+    private readonly string _cachePath;
+    private readonly TimeSpan? _cacheMaxStale;
+
+    public RemoteMediaCacheBackgroundTask(
+        ShellSettings shellSettings,
+        IMediaFileStore mediaFileStore,
+        IWebHostEnvironment webHostEnvironment,
+        IOptions<MediaOptions> mediaOptions,
+        ILogger<RemoteMediaCacheBackgroundTask> logger)
+    {
+        _mediaFileStore = mediaFileStore;
+
+        _cachePath = Path.Combine(
+            webHostEnvironment.WebRootPath,
+            shellSettings.Name,
+            DefaultMediaFileStoreCacheFileProvider.AssetsCachePath);
+
+        _cacheMaxStale = mediaOptions.Value.RemoteCacheMaxStale;
+        _logger = logger;
+    }
+
+    public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken)
+    {
+        // Ensure that the cache folder exists and should be cleaned.
+        if (!_cacheMaxStale.HasValue || !Directory.Exists(_cachePath))
+        {
+            return;
+        }
+
+        // Ensure that a remote media cache has been registered.
+        if (serviceProvider.GetService<IMediaFileStoreCache>() is null)
+        {
+            return;
+        }
+
+        // The min write time for an item to be retained in the cache,
+        // without having to get the item info from the remote store.
+        var minWriteTimeUtc = DateTimeOffset.UtcNow - _cacheMaxStale.Value;
+        try
+        {
+            // Lookup for all cache directories.
+            var directories = Directory.GetDirectories(_cachePath, "*", _enumerationOptions);
+            foreach (var directory in directories)
+            {
+                // Check if the directory is retained.
+                var directoryInfo = new DirectoryInfo(directory);
+                if (!directoryInfo.Exists || directoryInfo.LastWriteTimeUtc > minWriteTimeUtc)
+                {
+                    continue;
+                }
+
+                var path = Path.GetRelativePath(_cachePath, directoryInfo.FullName);
+
+                // Check if the remote directory doesn't exist.
+                var entry = await _mediaFileStore.GetDirectoryInfoAsync(path);
+                if (entry is null)
+                {
+                    Directory.Delete(directoryInfo.FullName, true);
+                }
+            }
+
+            // Lookup for all cache files.
+            var files = Directory.GetFiles(_cachePath, "*", _enumerationOptions);
+            foreach (var file in files)
+            {
+                // Check if the file is retained.
+                var fileInfo = new FileInfo(file);
+                if (!fileInfo.Exists || fileInfo.LastWriteTimeUtc > minWriteTimeUtc)
+                {
+                    continue;
+                }
+
+                var path = Path.GetRelativePath(_cachePath, fileInfo.FullName);
+
+                // Check if the remote media doesn't exist or was updated.
+                var entry = await _mediaFileStore.GetFileInfoAsync(path);
+                if (entry is null ||
+                    (entry.LastModifiedUtc > fileInfo.LastWriteTimeUtc &&
+                    entry.LastModifiedUtc < minWriteTimeUtc))
+                {
+                    File.Delete(fileInfo.FullName);
+                }
+            }
+        }
+        catch (Exception ex) when (ex is DirectoryNotFoundException)
+        {
+        }
+        catch (Exception ex) when (ex.IsFileSharingViolation())
+        {
+            if (_logger.IsEnabled(LogLevel.Warning))
+            {
+                _logger.LogWarning(
+                    ex,
+                    "Sharing violation while cleaning the remote media cache at '{CachePath}'.",
+                    _cachePath);
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(
+                ex,
+                "Failed to clean the remote media cache at '{CachePath}'.",
+                _cachePath);
+        }
+
+    }
+}

src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs

@@ -0,0 +1,105 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using OrchardCore.BackgroundTasks;
+using OrchardCore.Modules;
+using SixLabors.ImageSharp.Web.Caching;
+using SixLabors.ImageSharp.Web.Middleware;
+
+namespace OrchardCore.Media.Services;
+
+[BackgroundTask(Schedule = "0 0 * * *", Description = "'Resized media cache cleanup.")]
+public class ResizedMediaCacheBackgroundTask : IBackgroundTask
+{
+    private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true };
+
+    private readonly ILogger _logger;
+
+    private readonly string _cachePath;
+    private readonly string _cacheFolder;
+    private readonly TimeSpan _cacheMaxAge;
+    private readonly TimeSpan? _cacheMaxStale;
+
+    public ResizedMediaCacheBackgroundTask(
+        IWebHostEnvironment webHostEnvironment,
+        IOptions<MediaOptions> mediaOptions,
+        IOptions<ImageSharpMiddlewareOptions> middlewareOptions,
+        IOptions<PhysicalFileSystemCacheOptions> cacheOptions,
+        ILogger<ResizedMediaCacheBackgroundTask> logger)
+    {
+        _cachePath = Path.Combine(webHostEnvironment.WebRootPath, cacheOptions.Value.CacheFolder);
+        _cacheFolder = Path.GetFileName(cacheOptions.Value.CacheFolder);
+        _cacheMaxAge = middlewareOptions.Value.CacheMaxAge;
+        _cacheMaxStale = mediaOptions.Value.ResizedCacheMaxStale;
+        _logger = logger;
+    }
+
+    public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken)
+    {
+        // Ensure that the cache folder exists and should be cleaned.
+        if (!_cacheMaxStale.HasValue || !Directory.Exists(_cachePath))
+        {
+            return Task.CompletedTask;
+        }
+
+        // The min write time for an item to be retained in the cache.
+        var minWriteTimeUtc = DateTime.UtcNow.Subtract(_cacheMaxAge + _cacheMaxStale.Value);
+        try
+        {
+            // Lookup for all meta files.
+            var files = Directory.GetFiles(_cachePath, "*.meta", _enumerationOptions);
+            foreach (var file in files)
+            {
+                // Check if the file is retained.
+                var fileInfo = new FileInfo(file);
+                if (!fileInfo.Exists || fileInfo.LastWriteTimeUtc > minWriteTimeUtc)
+                {
+                    continue;
+                }
+
+                // Delete the folder including the media item.
+                Directory.Delete(fileInfo.DirectoryName, true);
+
+                // Delete new empty parent directories.
+                var parent = fileInfo.Directory.Parent;
+                while (parent is not null && parent.Name != _cacheFolder)
+                {
+                    Directory.Delete(parent.FullName);
+
+                    parent = parent.Parent;
+                    if (!parent.Exists || parent.EnumerateFileSystemInfos().Any())
+                    {
+                        break;
+                    }
+                }
+            }
+        }
+        catch (Exception ex) when (ex is DirectoryNotFoundException)
+        {
+        }
+        catch (Exception ex) when (ex.IsFileSharingViolation())
+        {
+            if (_logger.IsEnabled(LogLevel.Warning))
+            {
+                _logger.LogWarning(
+                    ex,
+                    "Sharing violation while cleaning the resized media cache at '{CachePath}'.",
+                    _cachePath);
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(
+                ex,
+                "Failed to clean the resized media cache at '{CachePath}'.",
+                _cachePath);
+        }
+
+        return Task.CompletedTask;
+    }
+}

src/OrchardCore.Modules/OrchardCore.Media/Startup.cs

@@ -74,6 +74,10 @@ public override void ConfigureServices(IServiceCollection services)
         {
             services.AddSingleton<IAnchorTag, MediaAnchorTag>();
 
+            // Resized media and remote media caches cleanups.
+            services.AddSingleton<IBackgroundTask, ResizedMediaCacheBackgroundTask>();
+            services.AddSingleton<IBackgroundTask, RemoteMediaCacheBackgroundTask>();
+
             services.Configure<TemplateOptions>(o =>
             {
                 o.MemberAccessStrategy.Register<DisplayMediaFieldViewModel>();
@@ -100,6 +104,7 @@ public override void ConfigureServices(IServiceCollection services)
                 {
                     Directory.CreateDirectory(mediaPath);
                 }
+
                 return new MediaFileProvider(options.AssetsRequestPath, mediaPath);
             });
 

src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs

@@ -31,6 +31,16 @@ public class MediaOptions
         /// </summary>
         public int MaxCacheDays { get; set; }
 
+        /// <summary>
+        /// The time before a staled item is removed from the resized media cache, if not provided there is no cleanup.
+        /// </summary>
+        public TimeSpan? ResizedCacheMaxStale { get; set; }
+
+        /// <summary>
+        /// The time before a staled remote media item is removed from the cache, if not provided there is no cleanup.
+        /// </summary>
+        public TimeSpan? RemoteCacheMaxStale { get; set; }
+
         /// <summary>
         /// The maximum size of an uploaded file in bytes.
         /// NB: You might still need to configure the limit in IIS (https://docs.microsoft.com/en-us/iis/configuration/system.webserver/security/requestfiltering/requestlimits/)