From 4ce44fbdbaf092f76051287e384f05f317393c51 Mon Sep 17 00:00:00 2001 From: Andrew McClenaghan Date: Sun, 8 Sep 2019 10:22:30 +1000 Subject: [PATCH] Initial work on adding AWS S3 Provider --- ImageSharp.Web.sln | 15 ++ .../ImageSharp.Web.Providers.AWS.csproj | 57 ++++++++ .../Providers/AWSS3StorageImageProvider.cs | 137 ++++++++++++++++++ .../AWSS3StorageImageProviderOptions.cs | 17 +++ .../Resolvers/AWSS3FileSystemResolver.cs | 47 ++++++ src/Shared/AssemblyInfo.Common.cs | 3 +- 6 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 src/ImageSharp.Web.Providers.AWS/ImageSharp.Web.Providers.AWS.csproj create mode 100644 src/ImageSharp.Web.Providers.AWS/Providers/AWSS3StorageImageProvider.cs create mode 100644 src/ImageSharp.Web.Providers.AWS/Providers/AWSS3StorageImageProviderOptions.cs create mode 100644 src/ImageSharp.Web.Providers.AWS/Resolvers/AWSS3FileSystemResolver.cs diff --git a/ImageSharp.Web.sln b/ImageSharp.Web.sln index 3d626c8b..cf0dc3a6 100644 --- a/ImageSharp.Web.sln +++ b/ImageSharp.Web.sln @@ -41,6 +41,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ImageSharp.Web.Benchmarks", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ImageSharp.Web.Providers.Azure", "src\ImageSharp.Web.Providers.Azure\ImageSharp.Web.Providers.Azure.csproj", "{E2A545EC-B909-4EAD-B95F-397F68588BE3}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ImageSharp.Web.Providers.AWS", "src\ImageSharp.Web.Providers.AWS\ImageSharp.Web.Providers.AWS.csproj", "{E631D300-ACD5-40EA-A6BB-08E22092EC76}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -111,6 +113,18 @@ Global {E2A545EC-B909-4EAD-B95F-397F68588BE3}.Release|x64.Build.0 = Release|Any CPU {E2A545EC-B909-4EAD-B95F-397F68588BE3}.Release|x86.ActiveCfg = Release|Any CPU {E2A545EC-B909-4EAD-B95F-397F68588BE3}.Release|x86.Build.0 = Release|Any CPU + {E631D300-ACD5-40EA-A6BB-08E22092EC76}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E631D300-ACD5-40EA-A6BB-08E22092EC76}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E631D300-ACD5-40EA-A6BB-08E22092EC76}.Debug|x64.ActiveCfg = Debug|Any CPU + {E631D300-ACD5-40EA-A6BB-08E22092EC76}.Debug|x64.Build.0 = Debug|Any CPU + {E631D300-ACD5-40EA-A6BB-08E22092EC76}.Debug|x86.ActiveCfg = Debug|Any CPU + {E631D300-ACD5-40EA-A6BB-08E22092EC76}.Debug|x86.Build.0 = Debug|Any CPU + {E631D300-ACD5-40EA-A6BB-08E22092EC76}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E631D300-ACD5-40EA-A6BB-08E22092EC76}.Release|Any CPU.Build.0 = Release|Any CPU + {E631D300-ACD5-40EA-A6BB-08E22092EC76}.Release|x64.ActiveCfg = Release|Any CPU + {E631D300-ACD5-40EA-A6BB-08E22092EC76}.Release|x64.Build.0 = Release|Any CPU + {E631D300-ACD5-40EA-A6BB-08E22092EC76}.Release|x86.ActiveCfg = Release|Any CPU + {E631D300-ACD5-40EA-A6BB-08E22092EC76}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -122,6 +136,7 @@ Global {BD73DBBD-6859-44C2-99FC-84148A6239A5} = {EA4178FC-11E3-496A-B630-0680B35E0AF8} {0B15E490-7821-42DF-86A5-4DEAE921DE59} = {56801022-D71A-4FBE-BC5B-CBA08E2284EC} {E2A545EC-B909-4EAD-B95F-397F68588BE3} = {815C0625-CD3D-440F-9F80-2D83856AB7AE} + {E631D300-ACD5-40EA-A6BB-08E22092EC76} = {815C0625-CD3D-440F-9F80-2D83856AB7AE} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {C5B38B65-A19E-4359-859C-5B2205429BD1} diff --git a/src/ImageSharp.Web.Providers.AWS/ImageSharp.Web.Providers.AWS.csproj b/src/ImageSharp.Web.Providers.AWS/ImageSharp.Web.Providers.AWS.csproj new file mode 100644 index 00000000..56e46098 --- /dev/null +++ b/src/ImageSharp.Web.Providers.AWS/ImageSharp.Web.Providers.AWS.csproj @@ -0,0 +1,57 @@ + + + + SixLabors.ImageSharp.Web.Providers.AWS + Six Labors and contributors + Six Labors + Copyright (c) Six Labors and contributors. + SixLabors.ImageSharp.Web.Providers.AWS + A provider for resolving images via AWS S3. + en + + netstandard2.0 + 7.3 + + true + SixLabors.ImageSharp.Web.Providers.AWS + SixLabors.ImageSharp.Web.Providers.AWS + Image Middleware Resize Crop Gif Jpg Jpeg Bitmap Png Azure + https://raw.githubusercontent.com/SixLabors/Branding/master/icons/imagesharp/sixlabors.imagesharp.128.png + https://github.com/SixLabors/ImageSharp.Web + http://www.apache.org/licenses/LICENSE-2.0 + git + https://github.com/SixLabors/ImageSharp.Web + + full + portable + + + + + + All + + + + + + + + + + + + + ..\..\shared-infrastructure\SixLabors.ruleset + SixLabors.ImageSharp.Web + + + + + + + + true + + + diff --git a/src/ImageSharp.Web.Providers.AWS/Providers/AWSS3StorageImageProvider.cs b/src/ImageSharp.Web.Providers.AWS/Providers/AWSS3StorageImageProvider.cs new file mode 100644 index 00000000..bf82c69d --- /dev/null +++ b/src/ImageSharp.Web.Providers.AWS/Providers/AWSS3StorageImageProvider.cs @@ -0,0 +1,137 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.Util; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using SixLabors.ImageSharp.Web.Resolvers; + +namespace SixLabors.ImageSharp.Web.Providers +{ + /// + /// Returns images stored in AWS S3. + /// + public class AWSS3StorageImageProvider : IImageProvider + { + /// + /// Character array to remove from paths. + /// + private static readonly char[] SlashChars = { '\\', '/' }; + + private readonly IAmazonS3 amazonS3Client; + private readonly ILogger logger; + private readonly AWSS3StorageImageProviderOptions storageOptions; + private Func match; + + /// + /// Initializes a new instance of the class. + /// + /// Amazon S3 client + /// Microsoft.Extensions.Logging ILogger + /// The S3 storage options + public AWSS3StorageImageProvider(IAmazonS3 amazonS3Client, ILogger logger, IOptions storageOptions) + { + Guard.NotNull(storageOptions, nameof(storageOptions)); + + this.amazonS3Client = amazonS3Client; + this.logger = logger; + this.storageOptions = storageOptions.Value; + } + + /// + public Func Match + { + get => this.match ?? this.IsMatch; + set => this.match = value; + } + + /// + public bool IsValidRequest(HttpContext context) + { + var displayUrl = context.Request.Path; + return Path.GetExtension(displayUrl).EndsWith(".jpg") || Path.GetExtension(displayUrl).EndsWith(".png"); + } + + /// + public async Task GetAsync(HttpContext context) + { + PathString displayUrl = context.Request.Path; + this.logger.LogDebug("Getting image for {ImageUri}", displayUrl); + string imageId = Path.GetFileNameWithoutExtension(displayUrl); + this.logger.LogDebug("Image id is {ImageId}", imageId); + + string imagePath = $"{imageId}.png"; + + bool imageExists = await this.KeyExists(this.storageOptions.BucketName, imagePath); + + if (!imageExists) + { + this.logger.LogDebug("No image found for {ImageId}", imageId); + return null; + } + + this.logger.LogDebug("Found image {ImageId}", imageId); + + return new AWSS3FileSystemResolver(this.amazonS3Client, this.storageOptions.BucketName, imagePath); + } + + private bool IsMatch(HttpContext context) + { + string path = context.Request.Path.Value.TrimStart(SlashChars); + return path.StartsWith(this.storageOptions.BucketName, StringComparison.OrdinalIgnoreCase); + } + + // ref https://github.com/aws/aws-sdk-net/blob/master/sdk/src/Services/S3/Custom/_bcl/IO/S3FileInfo.cs#L118 + private async Task KeyExists(string bucketName, string key, CancellationToken cancellationToken = default(CancellationToken)) + { + this.logger.LogDebug("Checking for the existence of key {Key} in bucket {BucketName}", key, bucketName); + + try + { + var request = new GetObjectMetadataRequest + { + BucketName = bucketName, + Key = key + }; + + ((Amazon.Runtime.Internal.IAmazonWebServiceRequest)request) + .AddBeforeRequestHandler(FileIORequestEventHandler); + + // If the object doesn't exist then a "NotFound" will be thrown + await this.amazonS3Client.GetObjectMetadataAsync(request, cancellationToken).ConfigureAwait(false); + return true; + } + catch (AmazonS3Exception e) + { + if (string.Equals(e.ErrorCode, "NoSuchBucket")) + { + return false; + } + + if (string.Equals(e.ErrorCode, "NotFound")) + { + return false; + } + + throw; + } + } + + private static void FileIORequestEventHandler(object sender, RequestEventArgs args) + { + if (args is WebServiceRequestEventArgs wsArgs) + { + string currentUserAgent = wsArgs.Headers[AWSSDKUtils.UserAgentHeader]; + wsArgs.Headers[AWSSDKUtils.UserAgentHeader] = currentUserAgent + " FileIO"; + } + } + } +} diff --git a/src/ImageSharp.Web.Providers.AWS/Providers/AWSS3StorageImageProviderOptions.cs b/src/ImageSharp.Web.Providers.AWS/Providers/AWSS3StorageImageProviderOptions.cs new file mode 100644 index 00000000..cb10377e --- /dev/null +++ b/src/ImageSharp.Web.Providers.AWS/Providers/AWSS3StorageImageProviderOptions.cs @@ -0,0 +1,17 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the Apache License, Version 2.0. + +namespace SixLabors.ImageSharp.Web.Providers +{ + /// + /// Configuration options for the provider. + /// + public class AWSS3StorageImageProviderOptions + { + /// + /// Gets or sets the bucket name. + /// Must conform to AWS S3 bucket naming guidelines. + /// + public string BucketName { get; set; } + } +} diff --git a/src/ImageSharp.Web.Providers.AWS/Resolvers/AWSS3FileSystemResolver.cs b/src/ImageSharp.Web.Providers.AWS/Resolvers/AWSS3FileSystemResolver.cs new file mode 100644 index 00000000..870e03a6 --- /dev/null +++ b/src/ImageSharp.Web.Providers.AWS/Resolvers/AWSS3FileSystemResolver.cs @@ -0,0 +1,47 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the Apache License, Version 2.0. + +using System.IO; +using System.Threading.Tasks; +using Amazon.S3; +using Amazon.S3.Model; + +namespace SixLabors.ImageSharp.Web.Resolvers +{ + /// + /// Provides means to manage image buffers within the AWS S3 file system. + /// + public class AWSS3FileSystemResolver : IImageResolver + { + private readonly IAmazonS3 amazonS3; + private readonly string bucketName; + private readonly string imagePath; + + /// + /// Initializes a new instance of the class. + /// + /// Amazon S3 Client + /// Bucket Name for where the files are + /// S3 Key + public AWSS3FileSystemResolver(IAmazonS3 amazonS3, string bucketName, string imagePath) + { + this.amazonS3 = amazonS3; + this.bucketName = bucketName; + this.imagePath = imagePath; + } + + /// + public async Task GetMetaDataAsync() + { + GetObjectMetadataResponse metadata = await this.amazonS3.GetObjectMetadataAsync(this.bucketName, this.imagePath); + return new ImageMetadata(metadata.LastModified); + } + + /// + public async Task OpenReadAsync() + { + GetObjectResponse s3Object = await this.amazonS3.GetObjectAsync(this.bucketName, this.imagePath); + return s3Object.ResponseStream; + } + } +} diff --git a/src/Shared/AssemblyInfo.Common.cs b/src/Shared/AssemblyInfo.Common.cs index f92d14e6..b06fe178 100644 --- a/src/Shared/AssemblyInfo.Common.cs +++ b/src/Shared/AssemblyInfo.Common.cs @@ -6,4 +6,5 @@ // Ensure the internals can be built and tested. [assembly: InternalsVisibleTo("SixLabors.ImageSharp.Web.Tests")] [assembly: InternalsVisibleTo("SixLabors.ImageSharp.Web.Providers.Azure")] -[assembly: InternalsVisibleTo("ImageSharp.Web.Benchmarks")] \ No newline at end of file +[assembly: InternalsVisibleTo("SixLabors.ImageSharp.Web.Providers.AWS")] +[assembly: InternalsVisibleTo("ImageSharp.Web.Benchmarks")]