Media caches cleanups

src/OrchardCore.Cms.Web/Properties/serviceDependencies.json

@@ -0,0 +1,8 @@
+{
+  "dependencies": {
+    "storage1": {
+      "type": "storage",
+      "connectionId": "StorageConnectionString"
+    }
+  }
+}

src/OrchardCore.Cms.Web/Properties/serviceDependencies.local.json

@@ -0,0 +1,9 @@
+{
+  "dependencies": {
+    "storage1": {
+      "secretStore": null,
+      "type": "storage.emulator",
+      "connectionId": "StorageConnectionString"
+    }
+  }
+}

src/OrchardCore.Cms.Web/appsettings.json

@@ -37,8 +37,8 @@
     //  "Extensions": "nohtml+advanced"
     //},
     // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media/#configuration to configure media.
-    //"OrchardCore_Media": {
-    //  "SupportedSizes": [ 16, 32, 50, 100, 160, 240, 480, 600, 1024, 2048 ],
+    "OrchardCore_Media": {
+    // "SupportedSizes": [ 16, 32, 50, 100, 160, 240, 480, 600, 1024, 2048 ],
     //  "MaxBrowserCacheDays": 30,
     //  "MaxCacheDays": 365,
     //  "MaxFileSize": 30000000,
@@ -48,7 +48,9 @@
     //  "UseTokenizedQueryString": true,
     //  "AllowedFileExtensions": [".jpg",".jpeg",".png",".gif",".ico",".svg",".webp",".pdf",".doc",".docx",".ppt",".pptx",".pps",".ppsx",".odt",".xls",".xlsx",".psd",".mp3",".m4a",".ogg",".wav",".mp4",".m4v",".mov",".wmv",".avi",".mpg",".ogv",".3gp"],
     //  "ContentSecurityPolicy": "default-src 'self'; style-src 'unsafe-inline'"
-    //}
+      "ResizedCacheMaxStale": "00:00:02:00", // The time before a staled item is removed from the resized media cache, if not provided there is no cleanup.
+      "RemoteCacheMaxStale": "00:00:02:00" // The time before a staled item is removed from the remote media cache, if not provided there is no cleanup.
+    },
     // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media.AmazonS3/#configuration to configure media storage in Amazon S3 Storage.
     //"OrchardCore_Media_AmazonS3": {
     //  "Region": "eu-central-1",
@@ -64,15 +66,14 @@
     //  "BucketName": ""
     //},
     // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media.Azure/#configuration to configure media storage in Azure Blob Storage.
-    //"OrchardCore_Media_Azure":
-    //{
-    //  "ConnectionString": "", // Set to your Azure Storage account connection string.
-    //  "ContainerName": "somecontainer", // Set to the Azure Blob container name. Templatable, refer docs.
-    //  "BasePath": "some/base/path", // Optionally, set to a path to store media in a subdirectory inside your container. Templatable, refer docs.
-    //  "CreateContainer": true // Activates an event to create the container if it does not already exist.
-    //  "RemoveContainer": true // Whether the 'Container' is deleted if the tenant is removed, false by default.
-    //},
-    // See https://stackexchange.github.io/StackExchange.Redis/Configuration.html
+    "OrchardCore_Media_Azure": {
+      "ConnectionString": "UseDevelopmentStorage=true", // Set to your Azure Storage account connection string.
+      "ContainerName": "StorageContainer", // Set to the Azure Blob container name. Templatable, refer docs.
+      //"BasePath": "some/base/path", // Optionally, set to a path to store media in a subdirectory inside your container. Templatable, refer docs.
+      "CreateContainer": true // Activates an event to create the container if it does not already exist.
+      //"RemoveContainer": true // Whether the 'Container' is deleted if the tenant is removed, false by default.
+    }
+     //See https://stackexchange.github.io/StackExchange.Redis/Configuration.html
     //"OrchardCore_Redis": {
     //  "Configuration": "192.168.99.100:6379,allowAdmin=true", // Redis Configuration string.
     //  "InstancePrefix": "" // Optional prefix allowing a Redis instance to be shared by different applications.

src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs

@@ -23,8 +23,10 @@ public MediaImageSharpConfiguration(IOptions<MediaOptions> mediaOptions)
         public void Configure(ImageSharpMiddlewareOptions options)
         {
             options.Configuration = Configuration.Default;
-            options.BrowserMaxAge = TimeSpan.FromDays(_mediaOptions.MaxBrowserCacheDays);
-            options.CacheMaxAge = TimeSpan.FromDays(_mediaOptions.MaxCacheDays);
+            //options.BrowserMaxAge = TimeSpan.FromDays(_mediaOptions.MaxBrowserCacheDays);
+            //options.CacheMaxAge = TimeSpan.FromDays(_mediaOptions.MaxCacheDays);
+            options.BrowserMaxAge = TimeSpan.FromSeconds(10);
+            options.CacheMaxAge = TimeSpan.FromSeconds(30);
             options.CacheHashLength = 12;
             options.OnParseCommandsAsync = context =>
             {

src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs

@@ -83,6 +83,8 @@ public void Configure(MediaOptions options)
 
             options.MaxBrowserCacheDays = section.GetValue("MaxBrowserCacheDays", DefaultMaxBrowserCacheDays);
             options.MaxCacheDays = section.GetValue("MaxCacheDays", DefaultMaxCacheDays);
+            options.ResizedCacheMaxStale = section.GetValue<TimeSpan?>(nameof(options.ResizedCacheMaxStale));
+            options.RemoteCacheMaxStale = section.GetValue<TimeSpan?>(nameof(options.RemoteCacheMaxStale));
             options.MaxFileSize = section.GetValue("MaxFileSize", DefaultMaxFileSize);
             options.CdnBaseUrl = section.GetValue("CdnBaseUrl", String.Empty).TrimEnd('/').ToLower();
             options.AssetsRequestPath = section.GetValue("AssetsRequestPath", DefaultAssetsRequestPath);

src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs

@@ -0,0 +1,130 @@
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using OrchardCore.BackgroundTasks;
+using OrchardCore.Environment.Shell;
+using OrchardCore.Media.Core;
+using OrchardCore.Modules;
+
+namespace OrchardCore.Media.Services;
+
+[BackgroundTask(Schedule = "* * * * *", Description = "'Remote media cache cleanup.")]
+public class RemoteMediaCacheBackgroundTask : IBackgroundTask
+{
+    private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true };
+
+    private readonly IMediaFileStore _mediaFileStore;
+    private readonly ILogger _logger;
+
+    private readonly string _cachePath;
+    private readonly TimeSpan? _cacheMaxStale;
+
+    public RemoteMediaCacheBackgroundTask(
+        ShellSettings shellSettings,
+        IMediaFileStore mediaFileStore,
+        IWebHostEnvironment webHostEnvironment,
+        IOptions<MediaOptions> mediaOptions,
+        ILogger<RemoteMediaCacheBackgroundTask> logger)
+    {
+        _mediaFileStore = mediaFileStore;
+
+        _cachePath = Path.Combine(
+            webHostEnvironment.WebRootPath,
+            shellSettings.Name,
+            DefaultMediaFileStoreCacheFileProvider.AssetsCachePath);
+
+        _cacheMaxStale = mediaOptions.Value.RemoteCacheMaxStale;
+        _logger = logger;
+    }
+
+    public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken)
+    {
+        // Ensure that the cache folder exists and should be cleaned.
+        if (!_cacheMaxStale.HasValue || !Directory.Exists(_cachePath))
+        {
+            return;
+        }
+
+        // Ensure that a remote media cache has been registered.
+        if (serviceProvider.GetService<IMediaFileStoreCache>() is null)
+        {
+            return;
+        }
+
+        // The min write time for an item to be retained in the cache,
+        // without having to get the item info from the remote store.
+        var minWriteTimeUtc = DateTimeOffset.UtcNow - _cacheMaxStale.Value;
+        try
+        {
+            // Lookup for all cache directories.
+            var directories = Directory.GetDirectories(_cachePath, "*", _enumerationOptions);
+            foreach (var directory in directories)
+            {
+                // Check if the directory is retained.
+                var directoryInfo = new DirectoryInfo(directory);
+                if (!directoryInfo.Exists || directoryInfo.LastWriteTimeUtc > minWriteTimeUtc)
+                {
+                    continue;
+                }
+
+                var path = Path.GetRelativePath(_cachePath, directoryInfo.FullName);
+
+                // Check if the remote directory doesn't exist.
+                var entry = await _mediaFileStore.GetDirectoryInfoAsync(path);
+                if (entry is null)
+                {
+                    Directory.Delete(directoryInfo.FullName, true);
+                }
+            }
+
+            // Lookup for all cache files.
+            var files = Directory.GetFiles(_cachePath, "*", _enumerationOptions);
+            foreach (var file in files)
+            {
+                // Check if the file is retained.
+                var fileInfo = new FileInfo(file);
+                if (!fileInfo.Exists || fileInfo.LastWriteTimeUtc > minWriteTimeUtc)
+                {
+                    continue;
+                }
+
+                var path = Path.GetRelativePath(_cachePath, fileInfo.FullName);
+
+                // Check if the remote media doesn't exist or was updated.
+                var entry = await _mediaFileStore.GetFileInfoAsync(path);
+                if (entry is null ||
+                    (entry.LastModifiedUtc > fileInfo.LastWriteTimeUtc &&
+                    entry.LastModifiedUtc < minWriteTimeUtc))
+                {
+                    File.Delete(fileInfo.FullName);
+                }
+            }
+        }
+        catch (Exception ex) when (ex is DirectoryNotFoundException)
+        {
+        }
+        catch (Exception ex) when (ex.IsFileSharingViolation())
+        {
+            if (_logger.IsEnabled(LogLevel.Warning))
+            {
+                _logger.LogWarning(
+                    ex,
+                    "Sharing violation while cleaning the remote media cache at '{CachePath}'.",
+                    _cachePath);
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(
+                ex,
+                "Failed to clean the remote media cache at '{CachePath}'.",
+                _cachePath);
+        }
+
+    }
+}

src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs

@@ -0,0 +1,105 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using OrchardCore.BackgroundTasks;
+using OrchardCore.Modules;
+using SixLabors.ImageSharp.Web.Caching;
+using SixLabors.ImageSharp.Web.Middleware;
+
+namespace OrchardCore.Media.Services;
+
+[BackgroundTask(Schedule = "* * * * *", Description = "'Resized media cache cleanup.")]
+public class ResizedMediaCacheBackgroundTask : IBackgroundTask
+{
+    private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true };
+
+    private readonly ILogger _logger;
+
+    private readonly string _cachePath;
+    private readonly string _cacheFolder;
+    private readonly TimeSpan _cacheMaxAge;
+    private readonly TimeSpan? _cacheMaxStale;
+
+    public ResizedMediaCacheBackgroundTask(
+        IWebHostEnvironment webHostEnvironment,
+        IOptions<MediaOptions> mediaOptions,
+        IOptions<ImageSharpMiddlewareOptions> middlewareOptions,
+        IOptions<PhysicalFileSystemCacheOptions> cacheOptions,
+        ILogger<ResizedMediaCacheBackgroundTask> logger)
+    {
+        _cachePath = Path.Combine(webHostEnvironment.WebRootPath, cacheOptions.Value.CacheFolder);
+        _cacheFolder = Path.GetFileName(cacheOptions.Value.CacheFolder);
+        _cacheMaxAge = middlewareOptions.Value.CacheMaxAge;
+        _cacheMaxStale = mediaOptions.Value.ResizedCacheMaxStale;
+        _logger = logger;
+    }
+
+    public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken)
+    {
+        // Ensure that the cache folder exists and should be cleaned.
+        if (!_cacheMaxStale.HasValue || !Directory.Exists(_cachePath))
+        {
+            return Task.CompletedTask;
+        }
+
+        // The min write time for an item to be retained in the cache.
+        var minWriteTimeUtc = DateTime.UtcNow.Subtract(_cacheMaxAge + _cacheMaxStale.Value);
+        try
+        {
+            // Lookup for all meta files.
+            var files = Directory.GetFiles(_cachePath, "*.meta", _enumerationOptions);
+            foreach (var file in files)
+            {
+                // Check if the file is retained.
+                var fileInfo = new FileInfo(file);
+                if (!fileInfo.Exists || fileInfo.LastWriteTimeUtc > minWriteTimeUtc)
+                {
+                    continue;
+                }
+
+                // Delete the folder including the media item.
+                Directory.Delete(fileInfo.DirectoryName, true);
+
+                // Delete new empty parent directories.
+                var parent = fileInfo.Directory.Parent;
+                while (parent is not null && parent.Name != _cacheFolder)
+                {
+                    Directory.Delete(parent.FullName);
+
+                    parent = parent.Parent;
+                    if (!parent.Exists || parent.EnumerateFileSystemInfos().Any())
+                    {
+                        break;
+                    }
+                }
+            }
+        }
+        catch (Exception ex) when (ex is DirectoryNotFoundException)
+        {
+        }
+        catch (Exception ex) when (ex.IsFileSharingViolation())
+        {
+            if (_logger.IsEnabled(LogLevel.Warning))
+            {
+                _logger.LogWarning(
+                    ex,
+                    "Sharing violation while cleaning the resized media cache at '{CachePath}'.",
+                    _cachePath);
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(
+                ex,
+                "Failed to clean the resized media cache at '{CachePath}'.",
+                _cachePath);
+        }
+
+        return Task.CompletedTask;
+    }
+}

src/OrchardCore.Modules/OrchardCore.Media/Startup.cs

@@ -11,6 +11,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using OrchardCore.Admin;
+using OrchardCore.BackgroundTasks;
 using OrchardCore.ContentManagement;
 using OrchardCore.ContentManagement.Display.ContentDisplay;
 using OrchardCore.ContentManagement.Handlers;
@@ -72,6 +73,10 @@ public override void ConfigureServices(IServiceCollection services)
         {
             services.AddSingleton<IAnchorTag, MediaAnchorTag>();
 
+            // Resized media and remote media caches cleanups.
+            services.AddSingleton<IBackgroundTask, ResizedMediaCacheBackgroundTask>();
+            services.AddSingleton<IBackgroundTask, RemoteMediaCacheBackgroundTask>();
+
             services.Configure<TemplateOptions>(o =>
             {
                 o.MemberAccessStrategy.Register<DisplayMediaFieldViewModel>();
@@ -96,6 +101,7 @@ public override void ConfigureServices(IServiceCollection services)
                 {
                     Directory.CreateDirectory(mediaPath);
                 }
+
                 return new MediaFileProvider(options.AssetsRequestPath, mediaPath);
             });
 

src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Http;
@@ -30,6 +31,16 @@ public class MediaOptions
         /// </summary>
         public int MaxCacheDays { get; set; }
 
+        /// <summary>
+        /// The time before a staled item is removed from the resized media cache, if not provided there is no cleanup.
+        /// </summary>
+        public TimeSpan? ResizedCacheMaxStale { get; set; }
+
+        /// <summary>
+        /// The time before a staled remote media item is removed from the cache, if not provided there is no cleanup.
+        /// </summary>
+        public TimeSpan? RemoteCacheMaxStale { get; set; }
+
         /// <summary>
         /// The maximum size of an uploaded file in bytes.
         /// NB: You might still need to configure the limit in IIS (https://docs.microsoft.com/en-us/iis/configuration/system.webserver/security/requestfiltering/requestlimits/)