From 248e790c1890af0c1edfb5a54861a858ad1036a7 Mon Sep 17 00:00:00 2001 From: jtkech Date: Tue, 1 Aug 2023 02:27:50 +0200 Subject: [PATCH 01/13] Testing Media caching --- .../Properties/serviceDependencies.json | 8 + .../Properties/serviceDependencies.local.json | 9 + src/OrchardCore.Cms.Web/appsettings.json | 17 +- .../MediaImageSharpConfiguration.cs | 2 + .../Services/ImageSharpCacheBackgroundTask.cs | 157 ++++++++++++++++++ .../OrchardCore.Media/Startup.cs | 3 + 6 files changed, 187 insertions(+), 9 deletions(-) create mode 100644 src/OrchardCore.Cms.Web/Properties/serviceDependencies.json create mode 100644 src/OrchardCore.Cms.Web/Properties/serviceDependencies.local.json create mode 100644 src/OrchardCore.Modules/OrchardCore.Media/Services/ImageSharpCacheBackgroundTask.cs diff --git a/src/OrchardCore.Cms.Web/Properties/serviceDependencies.json b/src/OrchardCore.Cms.Web/Properties/serviceDependencies.json new file mode 100644 index 00000000000..3f9031befba --- /dev/null +++ b/src/OrchardCore.Cms.Web/Properties/serviceDependencies.json @@ -0,0 +1,8 @@ +{ + "dependencies": { + "storage1": { + "type": "storage", + "connectionId": "StorageConnectionString" + } + } +} \ No newline at end of file diff --git a/src/OrchardCore.Cms.Web/Properties/serviceDependencies.local.json b/src/OrchardCore.Cms.Web/Properties/serviceDependencies.local.json new file mode 100644 index 00000000000..752df81bbd8 --- /dev/null +++ b/src/OrchardCore.Cms.Web/Properties/serviceDependencies.local.json @@ -0,0 +1,9 @@ +{ + "dependencies": { + "storage1": { + "secretStore": null, + "type": "storage.emulator", + "connectionId": "StorageConnectionString" + } + } +} \ No newline at end of file diff --git a/src/OrchardCore.Cms.Web/appsettings.json b/src/OrchardCore.Cms.Web/appsettings.json index 3caf75ab56d..7dcb042b871 100644 --- a/src/OrchardCore.Cms.Web/appsettings.json +++ b/src/OrchardCore.Cms.Web/appsettings.json @@ -64,15 +64,14 @@ // "BucketName": "" //}, // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media.Azure/#configuration to configure media storage in Azure Blob Storage. - //"OrchardCore_Media_Azure": - //{ - // "ConnectionString": "", // Set to your Azure Storage account connection string. - // "ContainerName": "somecontainer", // Set to the Azure Blob container name. Templatable, refer docs. - // "BasePath": "some/base/path", // Optionally, set to a path to store media in a subdirectory inside your container. Templatable, refer docs. - // "CreateContainer": true // Activates an event to create the container if it does not already exist. - // "RemoveContainer": true // Whether the 'Container' is deleted if the tenant is removed, false by default. - //}, - // See https://stackexchange.github.io/StackExchange.Redis/Configuration.html + "OrchardCore_Media_Azure": { + "ConnectionString": "UseDevelopmentStorage=true", // Set to your Azure Storage account connection string. + //"ContainerName": "StorageContainer", // Set to the Azure Blob container name. Templatable, refer docs. + "BasePath": "OrchardCore_Media_Azure", // Optionally, set to a path to store media in a subdirectory inside your container. Templatable, refer docs. + "CreateContainer": true // Activates an event to create the container if it does not already exist. + //"RemoveContainer": true // Whether the 'Container' is deleted if the tenant is removed, false by default. + } + //See https://stackexchange.github.io/StackExchange.Redis/Configuration.html //"OrchardCore_Redis": { // "Configuration": "192.168.99.100:6379,allowAdmin=true", // Redis Configuration string. // "InstancePrefix": "" // Optional prefix allowing a Redis instance to be shared by different applications. diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs b/src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs index e9341fcf037..28a1777275e 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs @@ -25,6 +25,8 @@ public void Configure(ImageSharpMiddlewareOptions options) options.Configuration = Configuration.Default; options.BrowserMaxAge = TimeSpan.FromDays(_mediaOptions.MaxBrowserCacheDays); options.CacheMaxAge = TimeSpan.FromDays(_mediaOptions.MaxCacheDays); + //options.BrowserMaxAge = TimeSpan.FromSeconds(10); + //options.CacheMaxAge = TimeSpan.FromSeconds(30); options.CacheHashLength = 12; options.OnParseCommandsAsync = context => { diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/ImageSharpCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/ImageSharpCacheBackgroundTask.cs new file mode 100644 index 00000000000..0f6ad00c872 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/ImageSharpCacheBackgroundTask.cs @@ -0,0 +1,157 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using OrchardCore.BackgroundTasks; +using OrchardCore.Environment.Shell; +using OrchardCore.Media.Core; +using OrchardCore.Modules; +using SixLabors.ImageSharp.Web.Caching; +using SixLabors.ImageSharp.Web.Middleware; + +namespace OrchardCore.Media.Services; + +[BackgroundTask(Schedule = "* * * * *", Description = "'ImageSharp' cache cleanup.")] +public class ImageSharpCacheBackgroundTask : IBackgroundTask +{ + private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true }; + + private readonly IMediaFileStore _mediaFileStore; + private readonly ImageSharpMiddlewareOptions _middlewareOptions; + private readonly ILogger _logger; + + private readonly string _imageSharpCachePath; + private readonly string _imageSharpCacheFolder; + private readonly string _remoteImageCachePath; + + public ImageSharpCacheBackgroundTask( + ShellSettings shellSettings, + IMediaFileStore mediaFileStore, + IWebHostEnvironment webHostEnvironment, + IOptions middlewareOptions, + IOptions cacheOptions, + ILogger logger) + { + _mediaFileStore = mediaFileStore; + _middlewareOptions = middlewareOptions.Value; + + _imageSharpCachePath = Path.Combine( + webHostEnvironment.WebRootPath, + cacheOptions.Value.CacheFolder); + + _imageSharpCacheFolder = Path.GetFileName(cacheOptions.Value.CacheFolder); + + _remoteImageCachePath = Path.Combine( + webHostEnvironment.WebRootPath, + shellSettings.Name, + DefaultMediaFileStoreCacheFileProvider.AssetsCachePath); + + _logger = logger; + } + + public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) + { + try + { + var directories = Directory.GetDirectories(_remoteImageCachePath, "*", _enumerationOptions); + foreach (var directory in directories) + { + var directoryInfo = new DirectoryInfo(directory); + var path = Path.GetRelativePath(_remoteImageCachePath, directoryInfo.FullName); + + var entry = await _mediaFileStore.GetDirectoryInfoAsync(path); + if (entry is null) + { + Directory.Delete(directoryInfo.FullName, true); + } + } + + var files = Directory.GetFiles(_remoteImageCachePath, "*", _enumerationOptions); + foreach (var file in files) + { + var fileInfo = new FileInfo(file); + var path = Path.GetRelativePath(_remoteImageCachePath, fileInfo.FullName); + + var entry = await _mediaFileStore.GetFileInfoAsync(path); + if (entry is null || + entry.LastModifiedUtc > fileInfo.LastWriteTimeUtc) + { + File.Delete(fileInfo.FullName); + } + } + } + catch (Exception ex) when (ex is DirectoryNotFoundException) + { + } + catch (Exception ex) when (ex.IsFileSharingViolation()) + { + if (_logger.IsEnabled(LogLevel.Warning)) + { + _logger.LogWarning( + ex, + "Sharing violation while cleaning the image cache folder at '{CachePath}'.", + _remoteImageCachePath); + } + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Failed to clean the remote image cache folder at '{CachePath}'.", + _remoteImageCachePath); + } + + + try + { + var files = Directory.GetFiles(_imageSharpCachePath, "*.meta", _enumerationOptions); + foreach ( var file in files) + { + var fileInfo = new FileInfo(file); + + if (fileInfo.LastWriteTimeUtc > (DateTimeOffset.UtcNow - _middlewareOptions.CacheMaxAge)) + { + continue; + } + + Directory.Delete(fileInfo.DirectoryName, true); + + var parent = fileInfo.Directory.Parent; + while (parent is not null && parent.Name != _imageSharpCacheFolder) + { + Directory.Delete(parent.FullName); + + parent = parent.Parent; + if (parent.EnumerateFileSystemInfos().Any()) + { + break; + } + } + } + } + catch (Exception ex) when (ex is DirectoryNotFoundException) + { + } + catch (Exception ex) when (ex.IsFileSharingViolation()) + { + if (_logger.IsEnabled(LogLevel.Warning)) + { + _logger.LogWarning( + ex, + "Sharing violation while cleaning the image cache folder at '{CachePath}'.", + _imageSharpCachePath); + } + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Failed to clean the image cache folder at '{CachePath}'.", + _imageSharpCachePath); + } + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs b/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs index b34bd9888f3..866f5322fed 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using OrchardCore.Admin; +using OrchardCore.BackgroundTasks; using OrchardCore.ContentManagement; using OrchardCore.ContentManagement.Display.ContentDisplay; using OrchardCore.ContentManagement.Handlers; @@ -72,6 +73,8 @@ public override void ConfigureServices(IServiceCollection services) { services.AddSingleton(); + services.AddSingleton(); + services.Configure(o => { o.MemberAccessStrategy.Register(); From 0176831480cf30fbda77e672770eb18517bb72b3 Mon Sep 17 00:00:00 2001 From: jtkech Date: Tue, 1 Aug 2023 03:36:26 +0200 Subject: [PATCH 02/13] Revert testing change --- src/OrchardCore.Cms.Web/appsettings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/OrchardCore.Cms.Web/appsettings.json b/src/OrchardCore.Cms.Web/appsettings.json index 7dcb042b871..173ea070331 100644 --- a/src/OrchardCore.Cms.Web/appsettings.json +++ b/src/OrchardCore.Cms.Web/appsettings.json @@ -66,8 +66,8 @@ // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media.Azure/#configuration to configure media storage in Azure Blob Storage. "OrchardCore_Media_Azure": { "ConnectionString": "UseDevelopmentStorage=true", // Set to your Azure Storage account connection string. - //"ContainerName": "StorageContainer", // Set to the Azure Blob container name. Templatable, refer docs. - "BasePath": "OrchardCore_Media_Azure", // Optionally, set to a path to store media in a subdirectory inside your container. Templatable, refer docs. + "ContainerName": "StorageContainer", // Set to the Azure Blob container name. Templatable, refer docs. + //"BasePath": "some/base/path", // Optionally, set to a path to store media in a subdirectory inside your container. Templatable, refer docs. "CreateContainer": true // Activates an event to create the container if it does not already exist. //"RemoveContainer": true // Whether the 'Container' is deleted if the tenant is removed, false by default. } From d7c4847cdddafe5a70cf4d1f4869450512fcb1a2 Mon Sep 17 00:00:00 2001 From: jtkech Date: Sat, 5 Aug 2023 05:54:09 +0200 Subject: [PATCH 03/13] Wip --- .../Services/ImageSharpCacheBackgroundTask.cs | 157 ------------------ .../Services/MediaOptionsConfiguration.cs | 1 + .../RemoteImageCacheBackgroundTask.cs | 101 +++++++++++ .../ResizedImageCacheBackgroundTask.cs | 105 ++++++++++++ .../OrchardCore.Media/Startup.cs | 4 +- .../MediaOptions.cs | 6 + 6 files changed, 216 insertions(+), 158 deletions(-) delete mode 100644 src/OrchardCore.Modules/OrchardCore.Media/Services/ImageSharpCacheBackgroundTask.cs create mode 100644 src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteImageCacheBackgroundTask.cs create mode 100644 src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedImageCacheBackgroundTask.cs diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/ImageSharpCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/ImageSharpCacheBackgroundTask.cs deleted file mode 100644 index 0f6ad00c872..00000000000 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/ImageSharpCacheBackgroundTask.cs +++ /dev/null @@ -1,157 +0,0 @@ -using System; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using OrchardCore.BackgroundTasks; -using OrchardCore.Environment.Shell; -using OrchardCore.Media.Core; -using OrchardCore.Modules; -using SixLabors.ImageSharp.Web.Caching; -using SixLabors.ImageSharp.Web.Middleware; - -namespace OrchardCore.Media.Services; - -[BackgroundTask(Schedule = "* * * * *", Description = "'ImageSharp' cache cleanup.")] -public class ImageSharpCacheBackgroundTask : IBackgroundTask -{ - private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true }; - - private readonly IMediaFileStore _mediaFileStore; - private readonly ImageSharpMiddlewareOptions _middlewareOptions; - private readonly ILogger _logger; - - private readonly string _imageSharpCachePath; - private readonly string _imageSharpCacheFolder; - private readonly string _remoteImageCachePath; - - public ImageSharpCacheBackgroundTask( - ShellSettings shellSettings, - IMediaFileStore mediaFileStore, - IWebHostEnvironment webHostEnvironment, - IOptions middlewareOptions, - IOptions cacheOptions, - ILogger logger) - { - _mediaFileStore = mediaFileStore; - _middlewareOptions = middlewareOptions.Value; - - _imageSharpCachePath = Path.Combine( - webHostEnvironment.WebRootPath, - cacheOptions.Value.CacheFolder); - - _imageSharpCacheFolder = Path.GetFileName(cacheOptions.Value.CacheFolder); - - _remoteImageCachePath = Path.Combine( - webHostEnvironment.WebRootPath, - shellSettings.Name, - DefaultMediaFileStoreCacheFileProvider.AssetsCachePath); - - _logger = logger; - } - - public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) - { - try - { - var directories = Directory.GetDirectories(_remoteImageCachePath, "*", _enumerationOptions); - foreach (var directory in directories) - { - var directoryInfo = new DirectoryInfo(directory); - var path = Path.GetRelativePath(_remoteImageCachePath, directoryInfo.FullName); - - var entry = await _mediaFileStore.GetDirectoryInfoAsync(path); - if (entry is null) - { - Directory.Delete(directoryInfo.FullName, true); - } - } - - var files = Directory.GetFiles(_remoteImageCachePath, "*", _enumerationOptions); - foreach (var file in files) - { - var fileInfo = new FileInfo(file); - var path = Path.GetRelativePath(_remoteImageCachePath, fileInfo.FullName); - - var entry = await _mediaFileStore.GetFileInfoAsync(path); - if (entry is null || - entry.LastModifiedUtc > fileInfo.LastWriteTimeUtc) - { - File.Delete(fileInfo.FullName); - } - } - } - catch (Exception ex) when (ex is DirectoryNotFoundException) - { - } - catch (Exception ex) when (ex.IsFileSharingViolation()) - { - if (_logger.IsEnabled(LogLevel.Warning)) - { - _logger.LogWarning( - ex, - "Sharing violation while cleaning the image cache folder at '{CachePath}'.", - _remoteImageCachePath); - } - } - catch (Exception ex) - { - _logger.LogError( - ex, - "Failed to clean the remote image cache folder at '{CachePath}'.", - _remoteImageCachePath); - } - - - try - { - var files = Directory.GetFiles(_imageSharpCachePath, "*.meta", _enumerationOptions); - foreach ( var file in files) - { - var fileInfo = new FileInfo(file); - - if (fileInfo.LastWriteTimeUtc > (DateTimeOffset.UtcNow - _middlewareOptions.CacheMaxAge)) - { - continue; - } - - Directory.Delete(fileInfo.DirectoryName, true); - - var parent = fileInfo.Directory.Parent; - while (parent is not null && parent.Name != _imageSharpCacheFolder) - { - Directory.Delete(parent.FullName); - - parent = parent.Parent; - if (parent.EnumerateFileSystemInfos().Any()) - { - break; - } - } - } - } - catch (Exception ex) when (ex is DirectoryNotFoundException) - { - } - catch (Exception ex) when (ex.IsFileSharingViolation()) - { - if (_logger.IsEnabled(LogLevel.Warning)) - { - _logger.LogWarning( - ex, - "Sharing violation while cleaning the image cache folder at '{CachePath}'.", - _imageSharpCachePath); - } - } - catch (Exception ex) - { - _logger.LogError( - ex, - "Failed to clean the image cache folder at '{CachePath}'.", - _imageSharpCachePath); - } - } -} diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs index fdac10a53d8..04dbfe62e9d 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs @@ -83,6 +83,7 @@ public void Configure(MediaOptions options) options.MaxBrowserCacheDays = section.GetValue("MaxBrowserCacheDays", DefaultMaxBrowserCacheDays); options.MaxCacheDays = section.GetValue("MaxCacheDays", DefaultMaxCacheDays); + options.InvalidCacheLifetime = section.GetValue(nameof(options.InvalidCacheLifetime)); options.MaxFileSize = section.GetValue("MaxFileSize", DefaultMaxFileSize); options.CdnBaseUrl = section.GetValue("CdnBaseUrl", String.Empty).TrimEnd('/').ToLower(); options.AssetsRequestPath = section.GetValue("AssetsRequestPath", DefaultAssetsRequestPath); diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteImageCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteImageCacheBackgroundTask.cs new file mode 100644 index 00000000000..82459aa1c4c --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteImageCacheBackgroundTask.cs @@ -0,0 +1,101 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Logging; +using OrchardCore.BackgroundTasks; +using OrchardCore.Environment.Shell; +using OrchardCore.Media.Core; +using OrchardCore.Modules; + +namespace OrchardCore.Media.Services; + +[BackgroundTask(Schedule = "* * * * *", Description = "'Remote image cache cleanup.")] +public class RemoteImageCacheBackgroundTask : IBackgroundTask +{ + private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true }; + + private readonly IMediaFileStore _mediaFileStore; + private readonly ILogger _logger; + + private readonly string _cachePath; + + public RemoteImageCacheBackgroundTask( + ShellSettings shellSettings, + IMediaFileStore mediaFileStore, + IWebHostEnvironment webHostEnvironment, + ILogger logger) + { + _mediaFileStore = mediaFileStore; + + _cachePath = Path.Combine( + webHostEnvironment.WebRootPath, + shellSettings.Name, + DefaultMediaFileStoreCacheFileProvider.AssetsCachePath); + + _logger = logger; + } + + public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) + { + try + { + // Check if the cache exists. + if (!Directory.Exists(_cachePath)) + { + return; + } + + var directories = Directory.GetDirectories(_cachePath, "*", _enumerationOptions); + foreach (var directory in directories) + { + var directoryInfo = new DirectoryInfo(directory); + var path = Path.GetRelativePath(_cachePath, directoryInfo.FullName); + + // Check if the remote directory still exists. + var entry = await _mediaFileStore.GetDirectoryInfoAsync(path); + if (entry is null) + { + Directory.Delete(directoryInfo.FullName, true); + } + } + + var files = Directory.GetFiles(_cachePath, "*", _enumerationOptions); + foreach (var file in files) + { + var fileInfo = new FileInfo(file); + var path = Path.GetRelativePath(_cachePath, fileInfo.FullName); + + // Check if the remote image still exists or was updated. + var entry = await _mediaFileStore.GetFileInfoAsync(path); + if (entry is null || + entry.LastModifiedUtc > fileInfo.LastWriteTimeUtc) + { + File.Delete(fileInfo.FullName); + } + } + } + catch (Exception ex) when (ex is DirectoryNotFoundException) + { + } + catch (Exception ex) when (ex.IsFileSharingViolation()) + { + if (_logger.IsEnabled(LogLevel.Warning)) + { + _logger.LogWarning( + ex, + "Sharing violation while cleaning the remote image cache at '{CachePath}'.", + _cachePath); + } + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Failed to clean the remote image cache at '{CachePath}'.", + _cachePath); + } + + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedImageCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedImageCacheBackgroundTask.cs new file mode 100644 index 00000000000..06f327e19a7 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedImageCacheBackgroundTask.cs @@ -0,0 +1,105 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using OrchardCore.BackgroundTasks; +using OrchardCore.Modules; +using SixLabors.ImageSharp.Web.Caching; +using SixLabors.ImageSharp.Web.Middleware; + +namespace OrchardCore.Media.Services; + +[BackgroundTask(Schedule = "* * * * *", Description = "'Resized image cache cleanup.")] +public class ResizedImageCacheBackgroundTask : IBackgroundTask +{ + private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true }; + + private readonly ILogger _logger; + + private readonly string _cachePath; + private readonly string _cacheFolder; + private readonly TimeSpan? _cacheMaxAge; + + public ResizedImageCacheBackgroundTask( + IWebHostEnvironment webHostEnvironment, + IOptions mediaOptions, + IOptions middlewareOptions, + IOptions cacheOptions, + ILogger logger) + { + _cachePath = Path.Combine(webHostEnvironment.WebRootPath, cacheOptions.Value.CacheFolder); + _cacheFolder = Path.GetFileName(cacheOptions.Value.CacheFolder); + + if (mediaOptions.Value.InvalidCacheLifetime.HasValue) + { + _cacheMaxAge = middlewareOptions.Value.CacheMaxAge + mediaOptions.Value.InvalidCacheLifetime.Value; + } + + _logger = logger; + } + + public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) + { + try + { + // Check if the cache exists and should be cleaned. + if (!_cacheMaxAge.HasValue || !Directory.Exists(_cachePath)) + { + return Task.CompletedTask; + } + + var files = Directory.GetFiles(_cachePath, "*.meta", _enumerationOptions); + foreach (var file in files) + { + // Check from the meta file if the cache is still valid. + var fileInfo = new FileInfo(file); + if (fileInfo.LastWriteTimeUtc > (DateTimeOffset.UtcNow - _cacheMaxAge)) + { + continue; + } + + // Delete the folder including the resized image. + Directory.Delete(fileInfo.DirectoryName, true); + + // Delete all new empty parent directories. + var parent = fileInfo.Directory.Parent; + while (parent is not null && parent.Name != _cacheFolder) + { + Directory.Delete(parent.FullName); + + parent = parent.Parent; + if (parent.EnumerateFileSystemInfos().Any()) + { + break; + } + } + } + } + catch (Exception ex) when (ex is DirectoryNotFoundException) + { + } + catch (Exception ex) when (ex.IsFileSharingViolation()) + { + if (_logger.IsEnabled(LogLevel.Warning)) + { + _logger.LogWarning( + ex, + "Sharing violation while cleaning the resized image cache at '{CachePath}'.", + _cachePath); + } + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Failed to clean the resized image cache at '{CachePath}'.", + _cachePath); + } + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs b/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs index 866f5322fed..77ed1c2e569 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs @@ -73,7 +73,9 @@ public override void ConfigureServices(IServiceCollection services) { services.AddSingleton(); - services.AddSingleton(); + // Resized and remote media caches cleanups. + services.AddSingleton(); + services.AddSingleton(); services.Configure(o => { diff --git a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs index 10642a63069..5c95baed05f 100644 --- a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs +++ b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; @@ -30,6 +31,11 @@ public class MediaOptions /// public int MaxCacheDays { get; set; } + /// + /// The lifetime of a resized media whose cache has expired, if no value is provided there is no cache cleanup. + /// + public TimeSpan? InvalidCacheLifetime { get; set; } + /// /// The maximum size of an uploaded file in bytes. /// NB: You might still need to configure the limit in IIS (https://docs.microsoft.com/en-us/iis/configuration/system.webserver/security/requestfiltering/requestlimits/) From 073f8dcd80b5a231b1149bcaa251afb2bae9a608 Mon Sep 17 00:00:00 2001 From: jtkech Date: Sat, 5 Aug 2023 06:43:24 +0200 Subject: [PATCH 04/13] wip --- .../Services/MediaOptionsConfiguration.cs | 1 + .../Services/RemoteImageCacheBackgroundTask.cs | 8 ++++++-- .../OrchardCore.Media.Abstractions/MediaOptions.cs | 7 ++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs index 04dbfe62e9d..60aa6d303b3 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs @@ -84,6 +84,7 @@ public void Configure(MediaOptions options) options.MaxBrowserCacheDays = section.GetValue("MaxBrowserCacheDays", DefaultMaxBrowserCacheDays); options.MaxCacheDays = section.GetValue("MaxCacheDays", DefaultMaxCacheDays); options.InvalidCacheLifetime = section.GetValue(nameof(options.InvalidCacheLifetime)); + options.RemoteCacheCleanup = section.GetValue(nameof(options.RemoteCacheCleanup)); options.MaxFileSize = section.GetValue("MaxFileSize", DefaultMaxFileSize); options.CdnBaseUrl = section.GetValue("CdnBaseUrl", String.Empty).TrimEnd('/').ToLower(); options.AssetsRequestPath = section.GetValue("AssetsRequestPath", DefaultAssetsRequestPath); diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteImageCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteImageCacheBackgroundTask.cs index 82459aa1c4c..047a096242d 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteImageCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteImageCacheBackgroundTask.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using OrchardCore.BackgroundTasks; using OrchardCore.Environment.Shell; using OrchardCore.Media.Core; @@ -20,11 +21,13 @@ public class RemoteImageCacheBackgroundTask : IBackgroundTask private readonly ILogger _logger; private readonly string _cachePath; + private readonly bool _cacheCleanup; public RemoteImageCacheBackgroundTask( ShellSettings shellSettings, IMediaFileStore mediaFileStore, IWebHostEnvironment webHostEnvironment, + IOptions mediaOptions, ILogger logger) { _mediaFileStore = mediaFileStore; @@ -34,6 +37,7 @@ public RemoteImageCacheBackgroundTask( shellSettings.Name, DefaultMediaFileStoreCacheFileProvider.AssetsCachePath); + _cacheCleanup = mediaOptions.Value.RemoteCacheCleanup; _logger = logger; } @@ -41,8 +45,8 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke { try { - // Check if the cache exists. - if (!Directory.Exists(_cachePath)) + // Check if the cache exists and should be cleaned. + if (!_cacheCleanup || !Directory.Exists(_cachePath)) { return; } diff --git a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs index 5c95baed05f..dd648eb977c 100644 --- a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs +++ b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs @@ -32,10 +32,15 @@ public class MediaOptions public int MaxCacheDays { get; set; } /// - /// The lifetime of a resized media whose cache has expired, if no value is provided there is no cache cleanup. + /// The lifetime of a cached resized media that has expired, if no value is provided there is no cache cleanup. /// public TimeSpan? InvalidCacheLifetime { get; set; } + /// + /// Wether or not the remote media cache is cleanup if a related remote media no longer exists or was updated. + /// + public bool RemoteCacheCleanup { get; set; } + /// /// The maximum size of an uploaded file in bytes. /// NB: You might still need to configure the limit in IIS (https://docs.microsoft.com/en-us/iis/configuration/system.webserver/security/requestfiltering/requestlimits/) From 63f51dc88a61cdf73d61ea7442c829f128085005 Mon Sep 17 00:00:00 2001 From: jtkech Date: Sun, 6 Aug 2023 02:18:01 +0200 Subject: [PATCH 05/13] wip --- .../Services/MediaOptionsConfiguration.cs | 3 +- ...k.cs => RemoteMediaCacheBackgroundTask.cs} | 25 +++++++++------- ....cs => ResizedMediaCacheBackgroundTask.cs} | 30 +++++++++++-------- .../OrchardCore.Media/Startup.cs | 5 ++-- .../MediaOptions.cs | 7 ++++- 5 files changed, 42 insertions(+), 28 deletions(-) rename src/OrchardCore.Modules/OrchardCore.Media/Services/{RemoteImageCacheBackgroundTask.cs => RemoteMediaCacheBackgroundTask.cs} (83%) rename src/OrchardCore.Modules/OrchardCore.Media/Services/{ResizedImageCacheBackgroundTask.cs => ResizedMediaCacheBackgroundTask.cs} (79%) diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs index 60aa6d303b3..bea28588627 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs @@ -83,7 +83,8 @@ public void Configure(MediaOptions options) options.MaxBrowserCacheDays = section.GetValue("MaxBrowserCacheDays", DefaultMaxBrowserCacheDays); options.MaxCacheDays = section.GetValue("MaxCacheDays", DefaultMaxCacheDays); - options.InvalidCacheLifetime = section.GetValue(nameof(options.InvalidCacheLifetime)); + options.CacheMaxStale = section.GetValue(nameof(options.CacheMaxStale)); + options.CacheCleanup = section.GetValue(nameof(options.CacheCleanup)); options.RemoteCacheCleanup = section.GetValue(nameof(options.RemoteCacheCleanup)); options.MaxFileSize = section.GetValue("MaxFileSize", DefaultMaxFileSize); options.CdnBaseUrl = section.GetValue("CdnBaseUrl", String.Empty).TrimEnd('/').ToLower(); diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteImageCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs similarity index 83% rename from src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteImageCacheBackgroundTask.cs rename to src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs index 047a096242d..e62711ab4c8 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteImageCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs @@ -3,6 +3,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using OrchardCore.BackgroundTasks; @@ -13,7 +14,7 @@ namespace OrchardCore.Media.Services; [BackgroundTask(Schedule = "* * * * *", Description = "'Remote image cache cleanup.")] -public class RemoteImageCacheBackgroundTask : IBackgroundTask +public class RemoteMediaCacheBackgroundTask : IBackgroundTask { private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true }; @@ -23,12 +24,12 @@ public class RemoteImageCacheBackgroundTask : IBackgroundTask private readonly string _cachePath; private readonly bool _cacheCleanup; - public RemoteImageCacheBackgroundTask( + public RemoteMediaCacheBackgroundTask( ShellSettings shellSettings, IMediaFileStore mediaFileStore, IWebHostEnvironment webHostEnvironment, IOptions mediaOptions, - ILogger logger) + ILogger logger) { _mediaFileStore = mediaFileStore; @@ -43,21 +44,23 @@ public RemoteImageCacheBackgroundTask( public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) { - try + // Ensure that the cache folder exists and should be cleaned. + if (!_cacheCleanup || + serviceProvider.GetService() is null || + !Directory.Exists(_cachePath)) { - // Check if the cache exists and should be cleaned. - if (!_cacheCleanup || !Directory.Exists(_cachePath)) - { - return; - } + return; + } + try + { var directories = Directory.GetDirectories(_cachePath, "*", _enumerationOptions); foreach (var directory in directories) { var directoryInfo = new DirectoryInfo(directory); var path = Path.GetRelativePath(_cachePath, directoryInfo.FullName); - // Check if the remote directory still exists. + // Check if the remote directory no longer exists. var entry = await _mediaFileStore.GetDirectoryInfoAsync(path); if (entry is null) { @@ -71,7 +74,7 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke var fileInfo = new FileInfo(file); var path = Path.GetRelativePath(_cachePath, fileInfo.FullName); - // Check if the remote image still exists or was updated. + // Check if the remote media no longer exists or was updated. var entry = await _mediaFileStore.GetFileInfoAsync(path); if (entry is null || entry.LastModifiedUtc > fileInfo.LastWriteTimeUtc) diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedImageCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs similarity index 79% rename from src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedImageCacheBackgroundTask.cs rename to src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs index 06f327e19a7..746ad889726 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedImageCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs @@ -14,7 +14,7 @@ namespace OrchardCore.Media.Services; [BackgroundTask(Schedule = "* * * * *", Description = "'Resized image cache cleanup.")] -public class ResizedImageCacheBackgroundTask : IBackgroundTask +public class ResizedMediaCacheBackgroundTask : IBackgroundTask { private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true }; @@ -23,41 +23,45 @@ public class ResizedImageCacheBackgroundTask : IBackgroundTask private readonly string _cachePath; private readonly string _cacheFolder; private readonly TimeSpan? _cacheMaxAge; + private readonly bool _cacheCleanup; - public ResizedImageCacheBackgroundTask( + public ResizedMediaCacheBackgroundTask( IWebHostEnvironment webHostEnvironment, IOptions mediaOptions, IOptions middlewareOptions, IOptions cacheOptions, - ILogger logger) + ILogger logger) { _cachePath = Path.Combine(webHostEnvironment.WebRootPath, cacheOptions.Value.CacheFolder); _cacheFolder = Path.GetFileName(cacheOptions.Value.CacheFolder); - if (mediaOptions.Value.InvalidCacheLifetime.HasValue) + if (mediaOptions.Value.CacheMaxStale.HasValue) { - _cacheMaxAge = middlewareOptions.Value.CacheMaxAge + mediaOptions.Value.InvalidCacheLifetime.Value; + _cacheMaxAge = middlewareOptions.Value.CacheMaxAge + mediaOptions.Value.CacheMaxStale.Value; } + _cacheCleanup = mediaOptions.Value.CacheCleanup; _logger = logger; } public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) { - try + // Ensure that the cache folder exists and should be cleaned. + if (!_cacheCleanup || !_cacheMaxAge.HasValue || !Directory.Exists(_cachePath)) { - // Check if the cache exists and should be cleaned. - if (!_cacheMaxAge.HasValue || !Directory.Exists(_cachePath)) - { - return Task.CompletedTask; - } + return Task.CompletedTask; + } + var maxAge = DateTimeOffset.UtcNow - _cacheMaxAge; + try + { + // Lookup for all '*.meta' files. var files = Directory.GetFiles(_cachePath, "*.meta", _enumerationOptions); foreach (var file in files) { - // Check from the meta file if the cache is still valid. + // Check if the cache is not stale. var fileInfo = new FileInfo(file); - if (fileInfo.LastWriteTimeUtc > (DateTimeOffset.UtcNow - _cacheMaxAge)) + if (fileInfo.LastWriteTimeUtc > maxAge) { continue; } diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs b/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs index 77ed1c2e569..1b5240294ea 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs @@ -74,8 +74,8 @@ public override void ConfigureServices(IServiceCollection services) services.AddSingleton(); // Resized and remote media caches cleanups. - services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.Configure(o => { @@ -101,6 +101,7 @@ public override void ConfigureServices(IServiceCollection services) { Directory.CreateDirectory(mediaPath); } + return new MediaFileProvider(options.AssetsRequestPath, mediaPath); }); diff --git a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs index dd648eb977c..b2a82f51dea 100644 --- a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs +++ b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs @@ -34,7 +34,12 @@ public class MediaOptions /// /// The lifetime of a cached resized media that has expired, if no value is provided there is no cache cleanup. /// - public TimeSpan? InvalidCacheLifetime { get; set; } + public TimeSpan? CacheMaxStale { get; set; } + + /// + /// Wether or not the resized media cache is cleanup if the max age of an invalid resized media has expired. + /// + public bool CacheCleanup { get; set; } /// /// Wether or not the remote media cache is cleanup if a related remote media no longer exists or was updated. From 372a9ac340a6bd6616c387cd88662f89c147ff83 Mon Sep 17 00:00:00 2001 From: jtkech Date: Sun, 6 Aug 2023 04:37:33 +0200 Subject: [PATCH 06/13] wip --- .../Services/MediaOptionsConfiguration.cs | 3 +- .../RemoteMediaCacheBackgroundTask.cs | 31 +++++++++++++++---- .../ResizedMediaCacheBackgroundTask.cs | 15 +++++---- .../OrchardCore.Media/Startup.cs | 2 +- .../MediaOptions.cs | 11 ++----- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs index bea28588627..6241dd2ed19 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs @@ -83,9 +83,8 @@ public void Configure(MediaOptions options) options.MaxBrowserCacheDays = section.GetValue("MaxBrowserCacheDays", DefaultMaxBrowserCacheDays); options.MaxCacheDays = section.GetValue("MaxCacheDays", DefaultMaxCacheDays); - options.CacheMaxStale = section.GetValue(nameof(options.CacheMaxStale)); options.CacheCleanup = section.GetValue(nameof(options.CacheCleanup)); - options.RemoteCacheCleanup = section.GetValue(nameof(options.RemoteCacheCleanup)); + options.CacheMaxStale = section.GetValue(nameof(options.CacheMaxStale)); options.MaxFileSize = section.GetValue("MaxFileSize", DefaultMaxFileSize); options.CdnBaseUrl = section.GetValue("CdnBaseUrl", String.Empty).TrimEnd('/').ToLower(); options.AssetsRequestPath = section.GetValue("AssetsRequestPath", DefaultAssetsRequestPath); diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs index e62711ab4c8..e8b9d2ca4a4 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs @@ -22,6 +22,7 @@ public class RemoteMediaCacheBackgroundTask : IBackgroundTask private readonly ILogger _logger; private readonly string _cachePath; + private readonly TimeSpan? _cacheMaxStale; private readonly bool _cacheCleanup; public RemoteMediaCacheBackgroundTask( @@ -38,26 +39,39 @@ public RemoteMediaCacheBackgroundTask( shellSettings.Name, DefaultMediaFileStoreCacheFileProvider.AssetsCachePath); - _cacheCleanup = mediaOptions.Value.RemoteCacheCleanup; + _cacheMaxStale = mediaOptions.Value.CacheMaxStale.Value; + _cacheCleanup = mediaOptions.Value.CacheCleanup; _logger = logger; } public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) { // Ensure that the cache folder exists and should be cleaned. - if (!_cacheCleanup || - serviceProvider.GetService() is null || - !Directory.Exists(_cachePath)) + if (!_cacheCleanup || !_cacheMaxStale.HasValue || !Directory.Exists(_cachePath)) { return; } + // Ensure that a remote media cache has been registered. + if (serviceProvider.GetService() is null) + { + return; + } + + var maxStale = _cacheMaxStale.Value; + var minAge = DateTimeOffset.UtcNow - maxStale; try { var directories = Directory.GetDirectories(_cachePath, "*", _enumerationOptions); foreach (var directory in directories) { + // Check if the directory is too recent. var directoryInfo = new DirectoryInfo(directory); + if (directoryInfo.LastWriteTimeUtc > minAge) + { + continue; + } + var path = Path.GetRelativePath(_cachePath, directoryInfo.FullName); // Check if the remote directory no longer exists. @@ -71,13 +85,18 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke var files = Directory.GetFiles(_cachePath, "*", _enumerationOptions); foreach (var file in files) { + // Check if the file is too recent. var fileInfo = new FileInfo(file); + if (fileInfo.LastWriteTimeUtc > minAge) + { + continue; + } + var path = Path.GetRelativePath(_cachePath, fileInfo.FullName); // Check if the remote media no longer exists or was updated. var entry = await _mediaFileStore.GetFileInfoAsync(path); - if (entry is null || - entry.LastModifiedUtc > fileInfo.LastWriteTimeUtc) + if (entry is null || entry.LastModifiedUtc > (fileInfo.LastWriteTimeUtc + maxStale)) { File.Delete(fileInfo.FullName); } diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs index 746ad889726..038ee426164 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs @@ -22,7 +22,8 @@ public class ResizedMediaCacheBackgroundTask : IBackgroundTask private readonly string _cachePath; private readonly string _cacheFolder; - private readonly TimeSpan? _cacheMaxAge; + private readonly TimeSpan _cacheMaxAge; + private readonly TimeSpan? _cacheMaxStale; private readonly bool _cacheCleanup; public ResizedMediaCacheBackgroundTask( @@ -37,9 +38,11 @@ public ResizedMediaCacheBackgroundTask( if (mediaOptions.Value.CacheMaxStale.HasValue) { - _cacheMaxAge = middlewareOptions.Value.CacheMaxAge + mediaOptions.Value.CacheMaxStale.Value; + _cacheMaxStale = middlewareOptions.Value.CacheMaxAge + mediaOptions.Value.CacheMaxStale.Value; } + _cacheMaxAge = middlewareOptions.Value.CacheMaxAge; + _cacheMaxStale = mediaOptions.Value.CacheMaxStale; _cacheCleanup = mediaOptions.Value.CacheCleanup; _logger = logger; } @@ -47,21 +50,21 @@ public ResizedMediaCacheBackgroundTask( public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) { // Ensure that the cache folder exists and should be cleaned. - if (!_cacheCleanup || !_cacheMaxAge.HasValue || !Directory.Exists(_cachePath)) + if (!_cacheCleanup || !_cacheMaxStale.HasValue || !Directory.Exists(_cachePath)) { return Task.CompletedTask; } - var maxAge = DateTimeOffset.UtcNow - _cacheMaxAge; + var minAge = DateTimeOffset.UtcNow - _cacheMaxAge - _cacheMaxStale.Value; try { // Lookup for all '*.meta' files. var files = Directory.GetFiles(_cachePath, "*.meta", _enumerationOptions); foreach (var file in files) { - // Check if the cache is not stale. + // Check if the file is too recent. var fileInfo = new FileInfo(file); - if (fileInfo.LastWriteTimeUtc > maxAge) + if (fileInfo.LastWriteTimeUtc > minAge) { continue; } diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs b/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs index 1b5240294ea..e289fba750b 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Startup.cs @@ -73,7 +73,7 @@ public override void ConfigureServices(IServiceCollection services) { services.AddSingleton(); - // Resized and remote media caches cleanups. + // Resized media and remote media caches cleanups. services.AddSingleton(); services.AddSingleton(); diff --git a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs index b2a82f51dea..b2abe1b87ae 100644 --- a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs +++ b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs @@ -32,19 +32,14 @@ public class MediaOptions public int MaxCacheDays { get; set; } /// - /// The lifetime of a cached resized media that has expired, if no value is provided there is no cache cleanup. - /// - public TimeSpan? CacheMaxStale { get; set; } - - /// - /// Wether or not the resized media cache is cleanup if the max age of an invalid resized media has expired. + /// Wether or not the resized media and remote media caches cleanups are done periodically. /// public bool CacheCleanup { get; set; } /// - /// Wether or not the remote media cache is cleanup if a related remote media no longer exists or was updated. + /// The time before a staled media item is removed from the cache, if null no cleanup. /// - public bool RemoteCacheCleanup { get; set; } + public TimeSpan? CacheMaxStale { get; set; } /// /// The maximum size of an uploaded file in bytes. From 7aaa0d0efb9ee3bb4bb9adaac0044655f0d0d091 Mon Sep 17 00:00:00 2001 From: jtkech Date: Sun, 6 Aug 2023 08:38:48 +0200 Subject: [PATCH 07/13] wip --- .../Services/MediaOptionsConfiguration.cs | 1 - .../Services/RemoteMediaCacheBackgroundTask.cs | 18 ++++++++++-------- .../ResizedMediaCacheBackgroundTask.cs | 15 ++++----------- .../MediaOptions.cs | 7 +------ 4 files changed, 15 insertions(+), 26 deletions(-) diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs index 6241dd2ed19..984acc27692 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs @@ -83,7 +83,6 @@ public void Configure(MediaOptions options) options.MaxBrowserCacheDays = section.GetValue("MaxBrowserCacheDays", DefaultMaxBrowserCacheDays); options.MaxCacheDays = section.GetValue("MaxCacheDays", DefaultMaxCacheDays); - options.CacheCleanup = section.GetValue(nameof(options.CacheCleanup)); options.CacheMaxStale = section.GetValue(nameof(options.CacheMaxStale)); options.MaxFileSize = section.GetValue("MaxFileSize", DefaultMaxFileSize); options.CdnBaseUrl = section.GetValue("CdnBaseUrl", String.Empty).TrimEnd('/').ToLower(); diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs index e8b9d2ca4a4..8573e9d0262 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs @@ -23,7 +23,6 @@ public class RemoteMediaCacheBackgroundTask : IBackgroundTask private readonly string _cachePath; private readonly TimeSpan? _cacheMaxStale; - private readonly bool _cacheCleanup; public RemoteMediaCacheBackgroundTask( ShellSettings shellSettings, @@ -39,35 +38,37 @@ public RemoteMediaCacheBackgroundTask( shellSettings.Name, DefaultMediaFileStoreCacheFileProvider.AssetsCachePath); - _cacheMaxStale = mediaOptions.Value.CacheMaxStale.Value; - _cacheCleanup = mediaOptions.Value.CacheCleanup; + _cacheMaxStale = mediaOptions.Value.CacheMaxStale; _logger = logger; } public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) { // Ensure that the cache folder exists and should be cleaned. - if (!_cacheCleanup || !_cacheMaxStale.HasValue || !Directory.Exists(_cachePath)) + if (!_cacheMaxStale.HasValue || !Directory.Exists(_cachePath)) { return; } + var maxStale = _cacheMaxStale.Value; + // Ensure that a remote media cache has been registered. if (serviceProvider.GetService() is null) { return; } - var maxStale = _cacheMaxStale.Value; - var minAge = DateTimeOffset.UtcNow - maxStale; + // the time from which a cache item is too recent to be removed. + var recentTimeUtc = DateTimeOffset.UtcNow - maxStale; try { + // Lookup for all cache directories. var directories = Directory.GetDirectories(_cachePath, "*", _enumerationOptions); foreach (var directory in directories) { // Check if the directory is too recent. var directoryInfo = new DirectoryInfo(directory); - if (directoryInfo.LastWriteTimeUtc > minAge) + if (directoryInfo.LastWriteTimeUtc > recentTimeUtc) { continue; } @@ -82,12 +83,13 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke } } + // Lookup for all cache files. var files = Directory.GetFiles(_cachePath, "*", _enumerationOptions); foreach (var file in files) { // Check if the file is too recent. var fileInfo = new FileInfo(file); - if (fileInfo.LastWriteTimeUtc > minAge) + if (fileInfo.LastWriteTimeUtc > recentTimeUtc) { continue; } diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs index 038ee426164..183379b7b22 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs @@ -24,7 +24,6 @@ public class ResizedMediaCacheBackgroundTask : IBackgroundTask private readonly string _cacheFolder; private readonly TimeSpan _cacheMaxAge; private readonly TimeSpan? _cacheMaxStale; - private readonly bool _cacheCleanup; public ResizedMediaCacheBackgroundTask( IWebHostEnvironment webHostEnvironment, @@ -35,27 +34,21 @@ public ResizedMediaCacheBackgroundTask( { _cachePath = Path.Combine(webHostEnvironment.WebRootPath, cacheOptions.Value.CacheFolder); _cacheFolder = Path.GetFileName(cacheOptions.Value.CacheFolder); - - if (mediaOptions.Value.CacheMaxStale.HasValue) - { - _cacheMaxStale = middlewareOptions.Value.CacheMaxAge + mediaOptions.Value.CacheMaxStale.Value; - } - _cacheMaxAge = middlewareOptions.Value.CacheMaxAge; _cacheMaxStale = mediaOptions.Value.CacheMaxStale; - _cacheCleanup = mediaOptions.Value.CacheCleanup; _logger = logger; } public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) { // Ensure that the cache folder exists and should be cleaned. - if (!_cacheCleanup || !_cacheMaxStale.HasValue || !Directory.Exists(_cachePath)) + if (!_cacheMaxStale.HasValue || !Directory.Exists(_cachePath)) { return Task.CompletedTask; } - var minAge = DateTimeOffset.UtcNow - _cacheMaxAge - _cacheMaxStale.Value; + // the time from which a cache item is too recent to be removed. + var recentTimeUtc = DateTimeOffset.UtcNow - _cacheMaxAge - _cacheMaxStale.Value; try { // Lookup for all '*.meta' files. @@ -64,7 +57,7 @@ public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken canc { // Check if the file is too recent. var fileInfo = new FileInfo(file); - if (fileInfo.LastWriteTimeUtc > minAge) + if (fileInfo.LastWriteTimeUtc > recentTimeUtc) { continue; } diff --git a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs index b2abe1b87ae..8beea6bc92c 100644 --- a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs +++ b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs @@ -32,12 +32,7 @@ public class MediaOptions public int MaxCacheDays { get; set; } /// - /// Wether or not the resized media and remote media caches cleanups are done periodically. - /// - public bool CacheCleanup { get; set; } - - /// - /// The time before a staled media item is removed from the cache, if null no cleanup. + /// The time before a staled media item is removed from its cache, if no value is provided there is no cleanup. /// public TimeSpan? CacheMaxStale { get; set; } From c475925fd652452a304b430c70d39a187f94756c Mon Sep 17 00:00:00 2001 From: jtkech Date: Mon, 7 Aug 2023 04:38:41 +0200 Subject: [PATCH 08/13] wip --- src/OrchardCore.Cms.Web/appsettings.json | 8 +++--- .../MediaImageSharpConfiguration.cs | 8 +++--- .../Services/MediaOptionsConfiguration.cs | 3 ++- .../RemoteMediaCacheBackgroundTask.cs | 25 ++++++++++--------- .../ResizedMediaCacheBackgroundTask.cs | 22 ++++++++-------- .../MediaOptions.cs | 9 +++++-- 6 files changed, 42 insertions(+), 33 deletions(-) diff --git a/src/OrchardCore.Cms.Web/appsettings.json b/src/OrchardCore.Cms.Web/appsettings.json index 173ea070331..986613e6e57 100644 --- a/src/OrchardCore.Cms.Web/appsettings.json +++ b/src/OrchardCore.Cms.Web/appsettings.json @@ -37,8 +37,8 @@ // "Extensions": "nohtml+advanced" //}, // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media/#configuration to configure media. - //"OrchardCore_Media": { - // "SupportedSizes": [ 16, 32, 50, 100, 160, 240, 480, 600, 1024, 2048 ], + "OrchardCore_Media": { + // "SupportedSizes": [ 16, 32, 50, 100, 160, 240, 480, 600, 1024, 2048 ], // "MaxBrowserCacheDays": 30, // "MaxCacheDays": 365, // "MaxFileSize": 30000000, @@ -48,7 +48,9 @@ // "UseTokenizedQueryString": true, // "AllowedFileExtensions": [".jpg",".jpeg",".png",".gif",".ico",".svg",".webp",".pdf",".doc",".docx",".ppt",".pptx",".pps",".ppsx",".odt",".xls",".xlsx",".psd",".mp3",".m4a",".ogg",".wav",".mp4",".m4v",".mov",".wmv",".avi",".mpg",".ogv",".3gp"], // "ContentSecurityPolicy": "default-src 'self'; style-src 'unsafe-inline'" - //} + "ResizedCacheMaxStale": "00:00:02:00", // The time before a staled item is removed from the resized media cache, if not provided there is no cleanup. + "RemoteCacheMaxStale": "00:00:03:00" // The time before a staled item is removed from the remote media cache, if not provided there is no cleanup. + }, // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media.AmazonS3/#configuration to configure media storage in Amazon S3 Storage. //"OrchardCore_Media_AmazonS3": { // "Region": "eu-central-1", diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs b/src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs index 28a1777275e..b142bd30b59 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs @@ -23,10 +23,10 @@ public MediaImageSharpConfiguration(IOptions mediaOptions) public void Configure(ImageSharpMiddlewareOptions options) { options.Configuration = Configuration.Default; - options.BrowserMaxAge = TimeSpan.FromDays(_mediaOptions.MaxBrowserCacheDays); - options.CacheMaxAge = TimeSpan.FromDays(_mediaOptions.MaxCacheDays); - //options.BrowserMaxAge = TimeSpan.FromSeconds(10); - //options.CacheMaxAge = TimeSpan.FromSeconds(30); + //options.BrowserMaxAge = TimeSpan.FromDays(_mediaOptions.MaxBrowserCacheDays); + //options.CacheMaxAge = TimeSpan.FromDays(_mediaOptions.MaxCacheDays); + options.BrowserMaxAge = TimeSpan.FromSeconds(10); + options.CacheMaxAge = TimeSpan.FromSeconds(30); options.CacheHashLength = 12; options.OnParseCommandsAsync = context => { diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs index 984acc27692..9cf5fb1e1c1 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/MediaOptionsConfiguration.cs @@ -83,7 +83,8 @@ public void Configure(MediaOptions options) options.MaxBrowserCacheDays = section.GetValue("MaxBrowserCacheDays", DefaultMaxBrowserCacheDays); options.MaxCacheDays = section.GetValue("MaxCacheDays", DefaultMaxCacheDays); - options.CacheMaxStale = section.GetValue(nameof(options.CacheMaxStale)); + options.ResizedCacheMaxStale = section.GetValue(nameof(options.ResizedCacheMaxStale)); + options.RemoteCacheMaxStale = section.GetValue(nameof(options.RemoteCacheMaxStale)); options.MaxFileSize = section.GetValue("MaxFileSize", DefaultMaxFileSize); options.CdnBaseUrl = section.GetValue("CdnBaseUrl", String.Empty).TrimEnd('/').ToLower(); options.AssetsRequestPath = section.GetValue("AssetsRequestPath", DefaultAssetsRequestPath); diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs index 8573e9d0262..8a42b578004 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs @@ -13,7 +13,7 @@ namespace OrchardCore.Media.Services; -[BackgroundTask(Schedule = "* * * * *", Description = "'Remote image cache cleanup.")] +[BackgroundTask(Schedule = "* * * * *", Description = "'Remote media cache cleanup.")] public class RemoteMediaCacheBackgroundTask : IBackgroundTask { private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true }; @@ -38,7 +38,7 @@ public RemoteMediaCacheBackgroundTask( shellSettings.Name, DefaultMediaFileStoreCacheFileProvider.AssetsCachePath); - _cacheMaxStale = mediaOptions.Value.CacheMaxStale; + _cacheMaxStale = mediaOptions.Value.RemoteCacheMaxStale; _logger = logger; } @@ -58,24 +58,25 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke return; } - // the time from which a cache item is too recent to be removed. - var recentTimeUtc = DateTimeOffset.UtcNow - maxStale; + // The min write time for an item to be retained in the cache, + // without having to get the item info from the remote store. + var minWriteTimeUtc = DateTimeOffset.UtcNow - maxStale; try { // Lookup for all cache directories. var directories = Directory.GetDirectories(_cachePath, "*", _enumerationOptions); foreach (var directory in directories) { - // Check if the directory is too recent. + // Check if the directory is retained. var directoryInfo = new DirectoryInfo(directory); - if (directoryInfo.LastWriteTimeUtc > recentTimeUtc) + if (directoryInfo.LastWriteTimeUtc > minWriteTimeUtc) { continue; } var path = Path.GetRelativePath(_cachePath, directoryInfo.FullName); - // Check if the remote directory no longer exists. + // Check if the remote directory doesn't exist. var entry = await _mediaFileStore.GetDirectoryInfoAsync(path); if (entry is null) { @@ -87,16 +88,16 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke var files = Directory.GetFiles(_cachePath, "*", _enumerationOptions); foreach (var file in files) { - // Check if the file is too recent. + // Check if the file is retained. var fileInfo = new FileInfo(file); - if (fileInfo.LastWriteTimeUtc > recentTimeUtc) + if (fileInfo.LastWriteTimeUtc > minWriteTimeUtc) { continue; } var path = Path.GetRelativePath(_cachePath, fileInfo.FullName); - // Check if the remote media no longer exists or was updated. + // Check if the remote media doesn't exist or was updated. var entry = await _mediaFileStore.GetFileInfoAsync(path); if (entry is null || entry.LastModifiedUtc > (fileInfo.LastWriteTimeUtc + maxStale)) { @@ -113,7 +114,7 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke { _logger.LogWarning( ex, - "Sharing violation while cleaning the remote image cache at '{CachePath}'.", + "Sharing violation while cleaning the remote media cache at '{CachePath}'.", _cachePath); } } @@ -121,7 +122,7 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke { _logger.LogError( ex, - "Failed to clean the remote image cache at '{CachePath}'.", + "Failed to clean the remote media cache at '{CachePath}'.", _cachePath); } diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs index 183379b7b22..52ccc4377da 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs @@ -13,7 +13,7 @@ namespace OrchardCore.Media.Services; -[BackgroundTask(Schedule = "* * * * *", Description = "'Resized image cache cleanup.")] +[BackgroundTask(Schedule = "* * * * *", Description = "'Resized media cache cleanup.")] public class ResizedMediaCacheBackgroundTask : IBackgroundTask { private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true }; @@ -35,7 +35,7 @@ public ResizedMediaCacheBackgroundTask( _cachePath = Path.Combine(webHostEnvironment.WebRootPath, cacheOptions.Value.CacheFolder); _cacheFolder = Path.GetFileName(cacheOptions.Value.CacheFolder); _cacheMaxAge = middlewareOptions.Value.CacheMaxAge; - _cacheMaxStale = mediaOptions.Value.CacheMaxStale; + _cacheMaxStale = mediaOptions.Value.ResizedCacheMaxStale; _logger = logger; } @@ -47,25 +47,25 @@ public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken canc return Task.CompletedTask; } - // the time from which a cache item is too recent to be removed. - var recentTimeUtc = DateTimeOffset.UtcNow - _cacheMaxAge - _cacheMaxStale.Value; + // The min write time for an item to be retained in the cache. + var minWriteTimeUtc = DateTime.UtcNow.Subtract(_cacheMaxAge + _cacheMaxStale.Value); try { - // Lookup for all '*.meta' files. + // Lookup for all meta files. var files = Directory.GetFiles(_cachePath, "*.meta", _enumerationOptions); foreach (var file in files) { - // Check if the file is too recent. + // Check if the file is retained. var fileInfo = new FileInfo(file); - if (fileInfo.LastWriteTimeUtc > recentTimeUtc) + if (fileInfo.LastWriteTimeUtc > minWriteTimeUtc) { continue; } - // Delete the folder including the resized image. + // Delete the folder including the media item. Directory.Delete(fileInfo.DirectoryName, true); - // Delete all new empty parent directories. + // Delete new empty parent directories. var parent = fileInfo.Directory.Parent; while (parent is not null && parent.Name != _cacheFolder) { @@ -88,7 +88,7 @@ public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken canc { _logger.LogWarning( ex, - "Sharing violation while cleaning the resized image cache at '{CachePath}'.", + "Sharing violation while cleaning the resized media cache at '{CachePath}'.", _cachePath); } } @@ -96,7 +96,7 @@ public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken canc { _logger.LogError( ex, - "Failed to clean the resized image cache at '{CachePath}'.", + "Failed to clean the resized media cache at '{CachePath}'.", _cachePath); } diff --git a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs index 8beea6bc92c..7896fb69294 100644 --- a/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs +++ b/src/OrchardCore/OrchardCore.Media.Abstractions/MediaOptions.cs @@ -32,9 +32,14 @@ public class MediaOptions public int MaxCacheDays { get; set; } /// - /// The time before a staled media item is removed from its cache, if no value is provided there is no cleanup. + /// The time before a staled item is removed from the resized media cache, if not provided there is no cleanup. /// - public TimeSpan? CacheMaxStale { get; set; } + public TimeSpan? ResizedCacheMaxStale { get; set; } + + /// + /// The time before a staled remote media item is removed from the cache, if not provided there is no cleanup. + /// + public TimeSpan? RemoteCacheMaxStale { get; set; } /// /// The maximum size of an uploaded file in bytes. From d539096a3bc67c4959bbcd3e280eb03fe5f1ab52 Mon Sep 17 00:00:00 2001 From: jtkech Date: Tue, 8 Aug 2023 03:39:49 +0200 Subject: [PATCH 09/13] wip --- src/OrchardCore.Cms.Web/appsettings.json | 2 +- .../Services/RemoteMediaCacheBackgroundTask.cs | 12 ++++++------ .../Services/ResizedMediaCacheBackgroundTask.cs | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/OrchardCore.Cms.Web/appsettings.json b/src/OrchardCore.Cms.Web/appsettings.json index 986613e6e57..e257e371887 100644 --- a/src/OrchardCore.Cms.Web/appsettings.json +++ b/src/OrchardCore.Cms.Web/appsettings.json @@ -49,7 +49,7 @@ // "AllowedFileExtensions": [".jpg",".jpeg",".png",".gif",".ico",".svg",".webp",".pdf",".doc",".docx",".ppt",".pptx",".pps",".ppsx",".odt",".xls",".xlsx",".psd",".mp3",".m4a",".ogg",".wav",".mp4",".m4v",".mov",".wmv",".avi",".mpg",".ogv",".3gp"], // "ContentSecurityPolicy": "default-src 'self'; style-src 'unsafe-inline'" "ResizedCacheMaxStale": "00:00:02:00", // The time before a staled item is removed from the resized media cache, if not provided there is no cleanup. - "RemoteCacheMaxStale": "00:00:03:00" // The time before a staled item is removed from the remote media cache, if not provided there is no cleanup. + "RemoteCacheMaxStale": "00:00:02:00" // The time before a staled item is removed from the remote media cache, if not provided there is no cleanup. }, // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media.AmazonS3/#configuration to configure media storage in Amazon S3 Storage. //"OrchardCore_Media_AmazonS3": { diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs index 8a42b578004..4b9c59b704f 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs @@ -50,8 +50,6 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke return; } - var maxStale = _cacheMaxStale.Value; - // Ensure that a remote media cache has been registered. if (serviceProvider.GetService() is null) { @@ -60,7 +58,7 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke // The min write time for an item to be retained in the cache, // without having to get the item info from the remote store. - var minWriteTimeUtc = DateTimeOffset.UtcNow - maxStale; + var minWriteTimeUtc = DateTimeOffset.UtcNow - _cacheMaxStale.Value; try { // Lookup for all cache directories. @@ -69,7 +67,7 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke { // Check if the directory is retained. var directoryInfo = new DirectoryInfo(directory); - if (directoryInfo.LastWriteTimeUtc > minWriteTimeUtc) + if (!directoryInfo.Exists || directoryInfo.LastWriteTimeUtc > minWriteTimeUtc) { continue; } @@ -90,7 +88,7 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke { // Check if the file is retained. var fileInfo = new FileInfo(file); - if (fileInfo.LastWriteTimeUtc > minWriteTimeUtc) + if (!fileInfo.Exists || fileInfo.LastWriteTimeUtc > minWriteTimeUtc) { continue; } @@ -99,7 +97,9 @@ public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToke // Check if the remote media doesn't exist or was updated. var entry = await _mediaFileStore.GetFileInfoAsync(path); - if (entry is null || entry.LastModifiedUtc > (fileInfo.LastWriteTimeUtc + maxStale)) + if (entry is null || + (entry.LastModifiedUtc > fileInfo.LastWriteTimeUtc && + entry.LastModifiedUtc < minWriteTimeUtc)) { File.Delete(fileInfo.FullName); } diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs index 52ccc4377da..38c0489c5ea 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs @@ -57,7 +57,7 @@ public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken canc { // Check if the file is retained. var fileInfo = new FileInfo(file); - if (fileInfo.LastWriteTimeUtc > minWriteTimeUtc) + if (!fileInfo.Exists || fileInfo.LastWriteTimeUtc > minWriteTimeUtc) { continue; } @@ -72,7 +72,7 @@ public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken canc Directory.Delete(parent.FullName); parent = parent.Parent; - if (parent.EnumerateFileSystemInfos().Any()) + if (!parent.Exists || parent.EnumerateFileSystemInfos().Any()) { break; } From b1bfc8f7557be95041f190707415e75d8a5a048a Mon Sep 17 00:00:00 2001 From: jtkech Date: Thu, 19 Oct 2023 08:02:49 +0200 Subject: [PATCH 10/13] Remove testing code --- src/OrchardCore.Cms.Web/appsettings.json | 20 +++++++++---------- .../MediaImageSharpConfiguration.cs | 6 ++---- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/OrchardCore.Cms.Web/appsettings.json b/src/OrchardCore.Cms.Web/appsettings.json index f2c869b0360..d0f1b3f1a5a 100644 --- a/src/OrchardCore.Cms.Web/appsettings.json +++ b/src/OrchardCore.Cms.Web/appsettings.json @@ -37,7 +37,7 @@ // "Extensions": "nohtml+advanced" //}, // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media/#configuration to configure media. - "OrchardCore_Media": { + //"OrchardCore_Media": { // "SupportedSizes": [ 16, 32, 50, 100, 160, 240, 480, 600, 1024, 2048 ], // "MaxBrowserCacheDays": 30, // "MaxCacheDays": 365, @@ -50,9 +50,9 @@ // "ContentSecurityPolicy": "default-src 'self'; style-src 'unsafe-inline'" // "MaxUploadChunkSize": 104857600, // "TemporaryFileLifetime": "01:00:00", - "ResizedCacheMaxStale": "00:00:02:00", // The time before a staled item is removed from the resized media cache, if not provided there is no cleanup. - "RemoteCacheMaxStale": "00:00:02:00" // The time before a staled item is removed from the remote media cache, if not provided there is no cleanup. - }, + // "ResizedCacheMaxStale": "00:00:02:00", // The time before a staled item is removed from the resized media cache, if not provided there is no cleanup. + // "RemoteCacheMaxStale": "00:00:02:00" // The time before a staled item is removed from the remote media cache, if not provided there is no cleanup. + //}, // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media.AmazonS3/#configuration to configure media storage in Amazon S3 Storage. //"OrchardCore_Media_AmazonS3": { // "Region": "eu-central-1", @@ -68,14 +68,14 @@ // "BucketName": "" //}, // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media.Azure/#configuration to configure media storage in Azure Blob Storage. - "OrchardCore_Media_Azure": - { - "ConnectionString": "UseDevelopmentStorage=true", // Set to your Azure Storage account connection string. - "ContainerName": "StorageContainer", // Set to the Azure Blob container name. Templatable, refer docs. + //"OrchardCore_Media_Azure": + //{ + // "ConnectionString": "", // Set to your Azure Storage account connection string. + // "ContainerName": "somecontainer", // Set to the Azure Blob container name. Templatable, refer docs. // "BasePath": "some/base/path", // Optionally, set to a path to store media in a subdirectory inside your container. Templatable, refer docs. - "CreateContainer": true, // Activates an event to create the container if it does not already exist. + // "CreateContainer": true, // Activates an event to create the container if it does not already exist. // "RemoveContainer": true // Whether the 'Container' is deleted if the tenant is removed, false by default. - } + //}, // See https://stackexchange.github.io/StackExchange.Redis/Configuration.html //"OrchardCore_Redis": { // "Configuration": "192.168.99.100:6379,allowAdmin=true", // Redis Configuration string. diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs b/src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs index b142bd30b59..e9341fcf037 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Processing/MediaImageSharpConfiguration.cs @@ -23,10 +23,8 @@ public MediaImageSharpConfiguration(IOptions mediaOptions) public void Configure(ImageSharpMiddlewareOptions options) { options.Configuration = Configuration.Default; - //options.BrowserMaxAge = TimeSpan.FromDays(_mediaOptions.MaxBrowserCacheDays); - //options.CacheMaxAge = TimeSpan.FromDays(_mediaOptions.MaxCacheDays); - options.BrowserMaxAge = TimeSpan.FromSeconds(10); - options.CacheMaxAge = TimeSpan.FromSeconds(30); + options.BrowserMaxAge = TimeSpan.FromDays(_mediaOptions.MaxBrowserCacheDays); + options.CacheMaxAge = TimeSpan.FromDays(_mediaOptions.MaxCacheDays); options.CacheHashLength = 12; options.OnParseCommandsAsync = context => { From d8c2744b525b5302b0cffe69b013379dd14e19e6 Mon Sep 17 00:00:00 2001 From: jtkech Date: Thu, 19 Oct 2023 08:15:20 +0200 Subject: [PATCH 11/13] Revert testing changes --- .../Properties/serviceDependencies.json | 8 -------- .../Properties/serviceDependencies.local.json | 9 --------- src/OrchardCore.Cms.Web/appsettings.json | 2 +- 3 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 src/OrchardCore.Cms.Web/Properties/serviceDependencies.json delete mode 100644 src/OrchardCore.Cms.Web/Properties/serviceDependencies.local.json diff --git a/src/OrchardCore.Cms.Web/Properties/serviceDependencies.json b/src/OrchardCore.Cms.Web/Properties/serviceDependencies.json deleted file mode 100644 index 3f9031befba..00000000000 --- a/src/OrchardCore.Cms.Web/Properties/serviceDependencies.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "dependencies": { - "storage1": { - "type": "storage", - "connectionId": "StorageConnectionString" - } - } -} \ No newline at end of file diff --git a/src/OrchardCore.Cms.Web/Properties/serviceDependencies.local.json b/src/OrchardCore.Cms.Web/Properties/serviceDependencies.local.json deleted file mode 100644 index 752df81bbd8..00000000000 --- a/src/OrchardCore.Cms.Web/Properties/serviceDependencies.local.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "dependencies": { - "storage1": { - "secretStore": null, - "type": "storage.emulator", - "connectionId": "StorageConnectionString" - } - } -} \ No newline at end of file diff --git a/src/OrchardCore.Cms.Web/appsettings.json b/src/OrchardCore.Cms.Web/appsettings.json index d0f1b3f1a5a..0e3eba857d2 100644 --- a/src/OrchardCore.Cms.Web/appsettings.json +++ b/src/OrchardCore.Cms.Web/appsettings.json @@ -38,7 +38,7 @@ //}, // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media/#configuration to configure media. //"OrchardCore_Media": { - // "SupportedSizes": [ 16, 32, 50, 100, 160, 240, 480, 600, 1024, 2048 ], + // "SupportedSizes": [ 16, 32, 50, 100, 160, 240, 480, 600, 1024, 2048 ], // "MaxBrowserCacheDays": 30, // "MaxCacheDays": 365, // "MaxFileSize": 30000000, From 0674c0573bce3060e6c5c542af5c1eb5418c48e6 Mon Sep 17 00:00:00 2001 From: jtkech Date: Thu, 19 Oct 2023 08:40:10 +0200 Subject: [PATCH 12/13] Tweak period of 2 background tasks --- src/OrchardCore.Cms.Web/appsettings.json | 6 +++--- .../Services/RemoteMediaCacheBackgroundTask.cs | 2 +- .../Services/ResizedMediaCacheBackgroundTask.cs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/OrchardCore.Cms.Web/appsettings.json b/src/OrchardCore.Cms.Web/appsettings.json index 0e3eba857d2..3c623ac80b2 100644 --- a/src/OrchardCore.Cms.Web/appsettings.json +++ b/src/OrchardCore.Cms.Web/appsettings.json @@ -47,11 +47,11 @@ // "AssetsPath": "Media", // "UseTokenizedQueryString": true, // "AllowedFileExtensions": [".jpg",".jpeg",".png",".gif",".ico",".svg",".webp",".pdf",".doc",".docx",".ppt",".pptx",".pps",".ppsx",".odt",".xls",".xlsx",".psd",".mp3",".m4a",".ogg",".wav",".mp4",".m4v",".mov",".wmv",".avi",".mpg",".ogv",".3gp"], - // "ContentSecurityPolicy": "default-src 'self'; style-src 'unsafe-inline'" + // "ContentSecurityPolicy": "default-src 'self'; style-src 'unsafe-inline'", // "MaxUploadChunkSize": 104857600, // "TemporaryFileLifetime": "01:00:00", - // "ResizedCacheMaxStale": "00:00:02:00", // The time before a staled item is removed from the resized media cache, if not provided there is no cleanup. - // "RemoteCacheMaxStale": "00:00:02:00" // The time before a staled item is removed from the remote media cache, if not provided there is no cleanup. + // "ResizedCacheMaxStale": "01:00:00", // The time before a staled item is removed from the resized media cache, if not provided there is no cleanup. + // "RemoteCacheMaxStale": "01:00:00" // The time before a staled item is removed from the remote media cache, if not provided there is no cleanup. //}, // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media.AmazonS3/#configuration to configure media storage in Amazon S3 Storage. //"OrchardCore_Media_AmazonS3": { diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs index 4b9c59b704f..56c9ed49121 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/RemoteMediaCacheBackgroundTask.cs @@ -13,7 +13,7 @@ namespace OrchardCore.Media.Services; -[BackgroundTask(Schedule = "* * * * *", Description = "'Remote media cache cleanup.")] +[BackgroundTask(Schedule = "30 0 * * *", Description = "'Remote media cache cleanup.")] public class RemoteMediaCacheBackgroundTask : IBackgroundTask { private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true }; diff --git a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs index 38c0489c5ea..7c532a0ce3b 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media/Services/ResizedMediaCacheBackgroundTask.cs @@ -13,7 +13,7 @@ namespace OrchardCore.Media.Services; -[BackgroundTask(Schedule = "* * * * *", Description = "'Resized media cache cleanup.")] +[BackgroundTask(Schedule = "0 0 * * *", Description = "'Resized media cache cleanup.")] public class ResizedMediaCacheBackgroundTask : IBackgroundTask { private static readonly EnumerationOptions _enumerationOptions = new() { RecurseSubdirectories = true }; From e5df98839725a7c642a4b7ab1c3a426db094d837 Mon Sep 17 00:00:00 2001 From: jtkech Date: Thu, 19 Oct 2023 08:58:57 +0200 Subject: [PATCH 13/13] group cache configurations --- src/OrchardCore.Cms.Web/appsettings.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/OrchardCore.Cms.Web/appsettings.json b/src/OrchardCore.Cms.Web/appsettings.json index 3c623ac80b2..175cf4508c7 100644 --- a/src/OrchardCore.Cms.Web/appsettings.json +++ b/src/OrchardCore.Cms.Web/appsettings.json @@ -41,6 +41,8 @@ // "SupportedSizes": [ 16, 32, 50, 100, 160, 240, 480, 600, 1024, 2048 ], // "MaxBrowserCacheDays": 30, // "MaxCacheDays": 365, + // "ResizedCacheMaxStale": "01:00:00", // The time before a staled item is removed from the resized media cache, if not provided there is no cleanup. + // "RemoteCacheMaxStale": "01:00:00", // The time before a staled item is removed from the remote media cache, if not provided there is no cleanup. // "MaxFileSize": 30000000, // "CdnBaseUrl": "https://your-cdn.com", // "AssetsRequestPath": "/media", @@ -49,9 +51,7 @@ // "AllowedFileExtensions": [".jpg",".jpeg",".png",".gif",".ico",".svg",".webp",".pdf",".doc",".docx",".ppt",".pptx",".pps",".ppsx",".odt",".xls",".xlsx",".psd",".mp3",".m4a",".ogg",".wav",".mp4",".m4v",".mov",".wmv",".avi",".mpg",".ogv",".3gp"], // "ContentSecurityPolicy": "default-src 'self'; style-src 'unsafe-inline'", // "MaxUploadChunkSize": 104857600, - // "TemporaryFileLifetime": "01:00:00", - // "ResizedCacheMaxStale": "01:00:00", // The time before a staled item is removed from the resized media cache, if not provided there is no cleanup. - // "RemoteCacheMaxStale": "01:00:00" // The time before a staled item is removed from the remote media cache, if not provided there is no cleanup. + // "TemporaryFileLifetime": "01:00:00" //}, // See https://docs.orchardcore.net/en/latest/docs/reference/modules/Media.AmazonS3/#configuration to configure media storage in Amazon S3 Storage. //"OrchardCore_Media_AmazonS3": {