Embedded license file ingestion for Gallery (#6580)

Enabling accepting license metadata. License files are not yet allowed.

src/NuGetGallery.Core/CoreConstants.cs

@@ -9,6 +9,7 @@ public static class CoreConstants
 
         public const string PackageFileSavePathTemplate = "{0}.{1}{2}";
         public const string PackageFileBackupSavePathTemplate = "{0}/{1}/{2}.{3}";
+        public const string PackageContentFileSavePathTemplate = "{0}/{1}";
 
         public const string NuGetPackageFileExtension = ".nupkg";
         public const string CertificateFileExtension = ".cer";
@@ -18,6 +19,7 @@ public static class CoreConstants
         public const string PackageContentType = "binary/octet-stream";
         public const string OctetStreamContentType = "application/octet-stream";
         public const string TextContentType = "text/plain";
+        public const string MarkdownContentType = "text/markdown"; // rfc7763
         public const string CertificateContentType = "application/pkix-cert";
         public const string JsonContentType = "application/json";
 
@@ -29,6 +31,7 @@ public static class CoreConstants
         public const string PackageBackupsFolderName = "package-backups";
         public const string PackageReadMesFolderName = "readmes";
         public const string PackagesFolderName = "packages";
+        public const string PackagesContentFolderName = "packages-content";
         public const string UploadsFolderName = "uploads";
         public const string ValidationFolderName = "validation";
         public const string RevalidationFolderName = "revalidation";
@@ -39,5 +42,7 @@ public static class CoreConstants
         public const string SymbolPackageBackupsFolderName = "symbol-package-backups";
 
         public const string UploadTracingKeyHeaderName = "upload-id";
+
+        public const string LicenseFileName = "license";
     }
 }

src/NuGetGallery.Core/EmbeddedLicenseFileType.cs

@@ -0,0 +1,26 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGetGallery
+{
+    /// <summary>
+    /// Specifies the type of the license file used in the package
+    /// </summary>
+    public enum EmbeddedLicenseFileType
+    {
+        /// <summary>
+        /// Indicates that package has no license file embedded.
+        /// </summary>
+        Absent = 0,
+
+        /// <summary>
+        /// Indicates that embedded license file is plain text.
+        /// </summary>
+        PlainText = 1,
+
+        /// <summary>
+        /// Indicates that embedded license file is markdown.
+        /// </summary>
+        Markdown = 2,
+    }
+}

src/NuGetGallery.Core/NuGetGallery.Core.csproj

@@ -203,7 +203,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="NuGet.Services.Entities">
-      <Version>2.31.0</Version>
+      <Version>2.32.0-agr-license-2170862</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Messaging">
       <Version>2.31.0</Version>
@@ -239,7 +239,7 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="NuGet.Packaging">
-      <Version>4.8.0</Version>
+      <Version>5.0.0-preview1.5665</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Validation">
       <Version>2.31.0</Version>

src/NuGetGallery.Core/Packaging/PackageMetadata.cs

@@ -51,7 +51,8 @@ public PackageMetadata(
             IEnumerable<FrameworkSpecificGroup> frameworkGroups,
             IEnumerable<NuGet.Packaging.Core.PackageType> packageTypes,
             NuGetVersion minClientVersion,
-            RepositoryMetadata repositoryMetadata)
+            RepositoryMetadata repositoryMetadata,
+            LicenseMetadata licenseMetadata = null)
         {
             _metadata = new Dictionary<string, string>(metadata, StringComparer.OrdinalIgnoreCase);
             _dependencyGroups = dependencyGroups.ToList().AsReadOnly();
@@ -67,6 +68,8 @@ public PackageMetadata(
                 RepositoryUrl = repoUrl;
                 RepositoryType = repositoryMetadata.Type;
             }
+
+            LicenseMetadata = licenseMetadata;
         }
 
         private void SetPropertiesFromMetadata()
@@ -123,6 +126,12 @@ private void SetPropertiesFromMetadata()
         public string Language { get; private set; }
         public NuGetVersion MinClientVersion { get; set; }
 
+        /// <summary>
+        /// Contains license metadata taken from the 'license' node of the nuspec file.
+        /// Null if no 'license' node present.
+        /// </summary>
+        public LicenseMetadata LicenseMetadata { get; }
+
         public string GetValueFromMetadata(string key)
         {
             return GetValue(key, (string)null);
@@ -244,7 +253,8 @@ public static PackageMetadata FromNuspecReader(NuspecReader nuspecReader, bool s
                 nuspecReader.GetFrameworkReferenceGroups(),
                 nuspecReader.GetPackageTypes(),
                 nuspecReader.GetMinClientVersion(),
-                nuspecReader.GetRepositoryMetadata());
+                nuspecReader.GetRepositoryMetadata(),
+                nuspecReader.GetLicenseMetadata());
         }
 
         private class StrictNuspecReader : NuspecReader

src/NuGetGallery.Core/Services/CloudBlobCoreFileStorageService.cs

@@ -293,8 +293,19 @@ private static string Log(AccessCondition accessCondition)
             return "(none)";
         }
 
-        public async Task SaveFileAsync(string folderName, string fileName, Stream file, bool overwrite = true)
+        public Task SaveFileAsync(string folderName, string fileName, Stream file, bool overwrite = true)
         {
+            var contentType = GetContentType(folderName);
+            return SaveFileAsync(folderName, fileName, contentType, file, overwrite);
+        }
+
+        public async Task SaveFileAsync(string folderName, string fileName, string contentType, Stream file, bool overwrite = true)
+        {
+            if (contentType == null)
+            {
+                throw new ArgumentNullException(nameof(contentType));
+            }
+
             ICloudBlobContainer container = await GetContainerAsync(folderName);
             var blob = container.GetBlobReference(fileName);
 
@@ -313,7 +324,7 @@ public async Task SaveFileAsync(string folderName, string fileName, Stream file,
                     ex);
             }
 
-            blob.Properties.ContentType = GetContentType(folderName);
+            blob.Properties.ContentType = contentType;
             blob.Properties.CacheControl = GetCacheControl(folderName);
             await blob.SetPropertiesAsync();
         }
@@ -575,6 +586,9 @@ private static string GetContentType(string folderName)
                 case CoreConstants.UserCertificatesFolderName:
                     return CoreConstants.CertificateContentType;
 
+                case CoreConstants.PackagesContentFolderName:
+                    return CoreConstants.OctetStreamContentType;
+
                 default:
                     throw new InvalidOperationException(
                         String.Format(CultureInfo.CurrentCulture, "The folder name {0} is not supported.", folderName));
@@ -587,6 +601,7 @@ private static string GetCacheControl(string folderName)
             {
                 case CoreConstants.PackagesFolderName:
                 case CoreConstants.SymbolPackagesFolderName:
+                case CoreConstants.PackagesContentFolderName:
                     return CoreConstants.DefaultCacheControl;
 
                 case CoreConstants.PackageBackupsFolderName:

src/NuGetGallery.Core/Services/CorePackageFileService.cs

@@ -13,6 +13,8 @@ namespace NuGetGallery
 {
     public class CorePackageFileService : ICorePackageFileService
     {
+        private const string LicenseFileName = "license";
+
         private readonly ICoreFileStorageService _fileStorageService;
         private readonly IFileMetadataService _metadata;
 
@@ -199,6 +201,76 @@ public async Task StorePackageFileInBackupLocationAsync(Package package, Stream
             }
         }
 
+        public Task SaveLicenseFileAsync(Package package, Stream licenseFile)
+        {
+            if (package == null)
+            {
+                throw new ArgumentNullException(nameof(package));
+            }
+
+            if (licenseFile == null)
+            {
+                throw new ArgumentNullException(nameof(licenseFile));
+            }
+
+            if (package.EmbeddedLicenseType == EmbeddedLicenseFileType.Absent)
+            {
+                throw new ArgumentException("Package must have an embedded license", nameof(package));
+            }
+
+            var fileName = BuildLicenseFileName(package);
+
+            // Gallery will generally ignore the content type on license files and will use the value from the DB,
+            // but we'll be nice and try to specify correct content type for them.
+            var contentType = package.EmbeddedLicenseType == EmbeddedLicenseFileType.Markdown
+                ? CoreConstants.MarkdownContentType
+                : CoreConstants.TextContentType;
+
+            return _fileStorageService.SaveFileAsync(_metadata.PackageContentFolderName, fileName, contentType, licenseFile, overwrite: true);
+        }
+
+        public Task<Stream> DownloadLicenseFileAsync(Package package)
+        {
+            var fileName = BuildLicenseFileName(package);
+            return _fileStorageService.GetFileAsync(_metadata.PackageContentFolderName, fileName);
+        }
+
+        public Task DeleteLicenseFileAsync(string id, string version)
+        {
+            if (id == null)
+            {
+                throw new ArgumentNullException(nameof(id));
+            }
+
+            if (string.IsNullOrWhiteSpace(id))
+            {
+                throw new ArgumentException($"{nameof(id)} cannot be empty", nameof(id));
+            }
+
+            if (version == null)
+            {
+                throw new ArgumentNullException(nameof(version));
+            }
+
+            if (string.IsNullOrWhiteSpace(version))
+            {
+                throw new ArgumentException($"{nameof(version)} cannot be empty", nameof(version));
+            }
+
+            var normalizedVersion = NuGetVersionFormatter.Normalize(version);
+            var fileName = BuildLicenseFileName(id, normalizedVersion);
+
+            return _fileStorageService.DeleteFileAsync(_metadata.PackageContentFolderName, fileName);
+        }
+
+        private string LicensePathTemplate => $"{_metadata.PackageContentPathTemplate}/{LicenseFileName}";
+
+        private string BuildLicenseFileName(Package package)
+            => BuildFileName(package, LicensePathTemplate, string.Empty);
+
+        private string BuildLicenseFileName(string id, string version)
+            => BuildFileName(id, version, LicensePathTemplate, string.Empty);
+
         private static string BuildBackupFileName(string id, string version, string hash, string extension, string fileBackupSavePathTemplate)
         {
             if (id == null)

src/NuGetGallery.Core/Services/ICoreFileStorageService.cs

@@ -53,6 +53,21 @@ Task<Uri> GetPriviledgedFileUriAsync(
 
         Task SaveFileAsync(string folderName, string fileName, Stream file, bool overwrite = true);
 
+        /// <summary>
+        /// Saves the file. If storage supports setting the content type for the file,
+        /// it will be set to the specified value
+        /// </summary>
+        /// <param name="folderName">The folder that will contain the file.</param>
+        /// <param name="fileName">The name of the file.</param>
+        /// <param name="contentType">The content type to set for the saved file if storage supports it.</param>
+        /// <param name="file">The content that should be saved.</param>
+        /// <param name="overwrite">Indicates whether file should be overwritten if exists.</param>
+        /// <exception cref="FileAlreadyExistsException">
+        /// Thrown when <paramref name="overwrite"/> is false and file already exists
+        /// in destination.
+        /// </exception>
+        Task SaveFileAsync(string folderName, string fileName, string contentType, Stream file, bool overwrite = true);
+
         /// <summary>
         /// Saves the file. An exception should be thrown if the access condition is not met.
         /// </summary>

src/NuGetGallery.Core/Services/ICorePackageFileService.cs

@@ -20,11 +20,21 @@ public interface ICorePackageFileService
         /// </summary>
         Task SavePackageFileAsync(Package package, Stream packageFile, bool overwrite);
 
+        /// <summary>
+        /// Saves the license file to the public container for package content.
+        /// </summary>
+        Task SaveLicenseFileAsync(Package package, Stream licenseFile);
+
         /// <summary>
         /// Downloads the package from the file storage and reads it into a stream.
         /// </summary>
         Task<Stream> DownloadPackageFileAsync(Package package);
 
+        /// <summary>
+        /// Downloads previously saved license file for a specified package.
+        /// </summary>
+        Task<Stream> DownloadLicenseFileAsync(Package package);
+
         /// <summary>
         /// Generates the URL for the specified package in the public container for available packages.
         /// </summary>
@@ -88,6 +98,14 @@ public interface ICorePackageFileService
         /// <param name="version">The package version. This value is case-insensitive and need not be normalized.</param>
         Task DeletePackageFileAsync(string id, string version);
 
+        /// <summary>
+        /// Deletes the license file for the package from the publicly available storage for the package content.
+        /// </summary>
+        /// <param name="id">The package ID. This value is case-insensitive.</param>
+        /// <param name="version">The package version. This value is case-insensitive and need not be normalized.</param>
+        /// <returns></returns>
+        Task DeleteLicenseFileAsync(string id, string version);
+
         /// <summary>
         /// Copies the contents of the package represented by the stream into the file storage backup location.
         /// </summary>

src/NuGetGallery.Core/Services/IFileMetadataService.cs

@@ -10,6 +10,16 @@ public interface IFileMetadataService
         /// </summary>
         string FileFolderName { get; }
 
+        /// <summary>
+        /// The name of the public folder where bits of package content can be extracted to.
+        /// </summary>
+        string PackageContentFolderName { get; }
+
+        /// <summary>
+        /// The template for the path to save user content. File name will be appended to it.
+        /// </summary>
+        string PackageContentPathTemplate { get; }
+
         /// <summary>
         /// The save file path template. For example <see cref="CoreConstants.PackageFileSavePathTemplate"/>
         /// </summary>

src/NuGetGallery.Core/Services/PackageFileServiceMetadata.cs

@@ -7,6 +7,10 @@ public class PackageFileMetadataService : IFileMetadataService
     {
         public string FileFolderName => CoreConstants.PackagesFolderName;
 
+        public string PackageContentFolderName => CoreConstants.PackagesContentFolderName;
+
+        public string PackageContentPathTemplate => CoreConstants.PackageContentFileSavePathTemplate;
+
         public string FileSavePathTemplate => CoreConstants.PackageFileSavePathTemplate;
 
         public string FileExtension => CoreConstants.NuGetPackageFileExtension;

src/NuGetGallery.Core/Services/SymbolPackageFileServiceMetadata.cs

@@ -1,12 +1,17 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
+
 namespace NuGetGallery
 {
     public class SymbolPackageFileMetadataService : IFileMetadataService
     {
         public string FileFolderName => CoreConstants.SymbolPackagesFolderName;
 
+        public string PackageContentFolderName => throw new NotImplementedException();
+        public string PackageContentPathTemplate => throw new NotImplementedException();
+
         public string FileSavePathTemplate => CoreConstants.PackageFileSavePathTemplate;
 
         public string FileExtension => CoreConstants.NuGetSymbolPackageFileExtension;

src/NuGetGallery/Configuration/AppConfiguration.cs

@@ -357,5 +357,11 @@ public string ExternalBrandingMessage
 
         [DefaultValue(false)]
         public bool RejectPackagesWithLicense { get; set; }
+
+        [DefaultValue(false)]
+        public bool BlockLegacyLicenseUrl { get; set; }
+
+        [DefaultValue(true)]
+        public bool AllowLicenselessPackages { get; set; }
     }
 }

src/NuGetGallery/Configuration/IAppConfiguration.cs

@@ -370,5 +370,16 @@ public interface IAppConfiguration : IMessageServiceConfiguration
         /// Flag that indicates whether packages with `license` node in them should be rejected.
         /// </summary>
         bool RejectPackagesWithLicense { get; set; }
+
+        /// <summary>
+        /// Indicates whether packages that specify the license the "old" way (with a "licenseUrl" node only) should be rejected.
+        /// </summary>
+        bool BlockLegacyLicenseUrl { get; set; }
+
+        /// <summary>
+        /// Indicates whether packages that don't specify any license information (no license URL, no license expression,
+        /// no embedded license) are allowed into Gallery.
+        /// </summary>
+        bool AllowLicenselessPackages { get; set; }
     }
 }

src/NuGetGallery/GalleryConstants.cs

@@ -106,6 +106,8 @@ public static class GalleryConstants
 
         public const string GitRepository = "git";
 
+        public const string LicenseDeprecationUrl = "https://aka.ms/deprecateLicenseUrl";
+
         public static class ContentNames
         {
             public static readonly string ReadOnly = "ReadOnly";