Implement SE-209 Swift Language Version API Update

This is the main implementation of the proposal. It adds a new runtime
and the proposed API changes.

<rdar://problem/38721967> Provide support for Swift 4.2 in SwiftPM
https://bugs.swift.org/browse/SR-7464

Sources/PackageDescription4/CXXLanguageStandard.swift → Sources/PackageDescription4/LanguageStandardSettings.swift

@@ -38,6 +38,38 @@ public enum CXXLanguageStandard: String {
     case gnucxx1z = "gnu++1z"
 }
 
+#if PACKAGE_DESCRIPTION_4_2
+/// Represents the version of the Swift language that should be used for
+/// compiling Swift sources in the package.
+public enum SwiftVersion {
+    case v3
+    case v4
+    case v4_2
+
+    /// User-defined value of Swift version.
+    ///
+    /// The value is passed as-is to Swift compiler's `-swift-version` flag.
+    case version(String)
+}
+
+extension SwiftVersion {
+    func toString() -> String {
+        let value: String
+        switch self {
+        case .v3:
+            value = "3"
+        case .v4:
+            value = "4"
+        case .v4_2:
+            value = "4.2"
+        case .version(let v):
+            value = v
+        }
+        return value
+    }
+}
+#endif
+
 extension CLanguageStandard {
     func toJSON() -> JSON {
         return .string(self.rawValue)

Sources/PackageDescription4/Package.swift

@@ -64,6 +64,9 @@ public final class Package {
   #if PACKAGE_DESCRIPTION_4
     /// The list of swift versions, this package is compatible with.
     public var swiftLanguageVersions: [Int]?
+  #elseif PACKAGE_DESCRIPTION_4_2
+    /// The list of swift versions, this package is compatible with.
+    public var swiftLanguageVersions: [SwiftVersion]?
   #else
     // The public API is an Int array but we will use String internally to
     // allow storing different types in the module when its loaded into SwiftPM.
@@ -78,6 +81,31 @@ public final class Package {
     /// The C++ language standard to use for all C++ targets in this package.
     public var cxxLanguageStandard: CXXLanguageStandard?
 
+  #if PACKAGE_DESCRIPTION_4_2
+    /// Construct a package.
+    public init(
+        name: String,
+        pkgConfig: String? = nil,
+        providers: [SystemPackageProvider]? = nil,
+        products: [Product] = [],
+        dependencies: [Dependency] = [],
+        targets: [Target] = [],
+        swiftLanguageVersions: [SwiftVersion]? = nil,
+        cLanguageStandard: CLanguageStandard? = nil,
+        cxxLanguageStandard: CXXLanguageStandard? = nil
+    ) {
+        self.name = name
+        self.pkgConfig = pkgConfig
+        self.providers = providers
+        self.products = products
+        self.dependencies = dependencies
+        self.targets = targets
+        self.swiftLanguageVersions = swiftLanguageVersions
+        self.cLanguageStandard = cLanguageStandard
+        self.cxxLanguageStandard = cxxLanguageStandard
+        registerExitHandler()
+    }
+  #else
     /// Construct a package.
     public init(
         name: String,
@@ -100,13 +128,19 @@ public final class Package {
       #if PACKAGE_DESCRIPTION_4
         /// The list of swift versions, this package is compatible with.
         self.swiftLanguageVersions = swiftLanguageVersions
+      #elseif PACKAGE_DESCRIPTION_4_2
+        self.swiftLanguageVersions = swiftLanguageVersions?.map(String.init).map(SwiftVersion.version)
       #else
         self.swiftLanguageVersions = swiftLanguageVersions?.map(String.init)
       #endif
 
         self.cLanguageStandard = cLanguageStandard
         self.cxxLanguageStandard = cxxLanguageStandard
+        registerExitHandler()
+    }
+  #endif
 
+    private func registerExitHandler() {
         // Add custom exit handler to cause package to be dumped at exit, if
         // requested.
         //
@@ -191,6 +225,8 @@ extension Package {
         let swiftLanguageVersionsString: [String]?
       #if PACKAGE_DESCRIPTION_4
         swiftLanguageVersionsString = self.swiftLanguageVersions?.map(String.init)
+      #elseif PACKAGE_DESCRIPTION_4_2
+        swiftLanguageVersionsString = self.swiftLanguageVersions?.map({ $0.toString() })
       #else
         swiftLanguageVersionsString = self.swiftLanguageVersions
       #endif

Sources/PackageLoading/ManifestLoader.swift

@@ -56,7 +56,23 @@ public extension ManifestResourceProvider {
 extension ToolsVersion {
     /// Returns the manifest version for this tools version.
     public var manifestVersion: ManifestVersion {
-        return major == 3 ? .v3 : .v4
+        // FIXME: This works for now but we may want to do something better here
+        // if we're going to have a lot of manifest versions. We can make
+        // ManifestVersion a proper version type and then automatically
+        // determine the best version from the available versions.
+        //
+        // Return manifest version 3 if major component of tools version is 3.
+        if major == 3 {
+            return .v3
+        }
+
+        // If the tools version is less than 4.2, return manifest version 4.
+        if major == 4 && minor < 2 {
+            return .v4
+        }
+
+        // Return 4.2 for otherwise.
+        return .v4_2
     }
 }
 
@@ -199,16 +215,20 @@ public final class ManifestLoader: ManifestLoaderProtocol {
                 package: .v3(pd.package),
                 legacyProducts: pd.products,
                 version: version,
-                interpreterFlags: parseResult.interpreterFlags)
+                interpreterFlags: parseResult.interpreterFlags,
+                manifestVersion: manifestVersion
+            )
 
-        case .v4:
+        case .v4, .v4_2:
             let package = try loadPackageDescription4(json, baseURL: baseURL)
             manifest = Manifest(
                 path: inputPath,
                 url: baseURL,
                 package: .v4(package),
                 version: version,
-                interpreterFlags: parseResult.interpreterFlags)
+                interpreterFlags: parseResult.interpreterFlags,
+                manifestVersion: manifestVersion
+            )
         }
 
         return manifest
@@ -298,7 +318,7 @@ public final class ManifestLoader: ManifestLoaderProtocol {
     ) -> [String] {
         var cmd = [String]()
         let runtimePath = self.runtimePath(for: manifestVersion)
-        cmd += ["-swift-version", String(manifestVersion.rawValue)]
+        cmd += ["-swift-version", manifestVersion.swiftLanguageVersion.rawValue]
         cmd += ["-I", runtimePath.asString]
       #if os(macOS)
         cmd += ["-target", "x86_64-apple-macosx10.10"]
@@ -311,7 +331,7 @@ public final class ManifestLoader: ManifestLoaderProtocol {
 
     /// Returns the runtime path given the manifest version and path to libDir.
     private func runtimePath(for version: ManifestVersion) -> AbsolutePath {
-        return resources.libDir.appending(component: String(version.rawValue))
+        return resources.libDir.appending(component: version.rawValue)
     }
 }
 

Sources/PackageLoading/PackageBuilder.swift

@@ -819,14 +819,24 @@ public final class PackageBuilder {
                     package: manifest.name, required: swiftLanguageVersions, current: .currentToolsVersion)
             }
             computedSwiftVersion = swiftVersion
-        } else if isVersion3Manifest {
+        } else {
             // Otherwise, use the version depending on the manifest version.
             //
-            // FIXME: We need to store the tools version in the Manifest so we can
-            // compute the right language version for this package.
-            computedSwiftVersion = .v3
-        } else {
-            computedSwiftVersion = .v4
+            // FIXME: This is not very scalable. We need to store the tools
+            // version in the manifest and then use that to compute the right
+            // Swift version instead of relying on the manifest version.  The
+            // manifest version is just the version that was used to load the
+            // manifest and shouldn't contribute to what Swift version is
+            // chosen. For e.g., we might have a new manifest version 4.3, but
+            // the language version should still be 4.2.
+            switch manifest.manifestVersion {
+            case .v3:
+                computedSwiftVersion = .v3
+            case .v4:
+                computedSwiftVersion = .v4
+            case .v4_2:
+                computedSwiftVersion = .v4_2
+            }
         }
         _swiftVersion = computedSwiftVersion
         return computedSwiftVersion

Sources/PackageModel/Manifest.swift

@@ -17,6 +17,22 @@ import Utility
 public enum ManifestVersion: String {
     case v3 = "3"
     case v4 = "4"
+    case v4_2 = "4_2"
+
+    /// The Swift language version to use when parsing the manifest file.
+    public var swiftLanguageVersion: SwiftLanguageVersion {
+        switch self {
+        case .v3: return .v3
+        case .v4: return .v4
+
+        // To temporarily keep SwiftPM's tests compatible with Swift 4.1.
+      #if swift(>=4.1.50)
+        case .v4_2: return .v4_2
+      #else
+        case .v4_2: return .v4
+      #endif
+        }
+    }
 }
 
 /**
@@ -82,12 +98,7 @@ public final class Manifest: ObjectIdentifierProtocol, CustomStringConvertible {
     }
 
     /// The manifest version.
-    public var manifestVersion: ManifestVersion {
-        switch package {
-        case .v3: return .v3
-        case .v4: return .v4
-        }
-    }
+    public let manifestVersion: ManifestVersion
 
     /// The flags that were used to interprete the manifest.
     public let interpreterFlags: [String]
@@ -98,7 +109,8 @@ public final class Manifest: ObjectIdentifierProtocol, CustomStringConvertible {
         package: RawPackage,
         legacyProducts: [PackageDescription.Product] = [],
         version: Version?,
-        interpreterFlags: [String] = []
+        interpreterFlags: [String] = [],
+        manifestVersion: ManifestVersion
     ) {
         if case .v4 = package {
             precondition(legacyProducts.isEmpty, "Legacy products are not supported in v4 manifest.")
@@ -109,6 +121,7 @@ public final class Manifest: ObjectIdentifierProtocol, CustomStringConvertible {
         self.legacyProducts = legacyProducts
         self.version = version
         self.interpreterFlags = interpreterFlags
+        self.manifestVersion = manifestVersion
     }
 
     public var description: String {

Sources/PackageModel/Target.swift

@@ -76,8 +76,11 @@ public class SwiftTarget: Target {
             return target.type == .test
         }).flatMap({ $0 as? SwiftTarget })
 
-        // FIXME: We need to select the latest version that can satisfy the current tools version.
-        self.swiftVersion = swiftTestTarget?.swiftVersion ?? .v4
+        // FIXME: This is not very correct but doesn't matter much in practice.
+        // We need to select the latest Swift language version that can
+        // satisfy the current tools version but there is not a good way to
+        // do that currently.
+        self.swiftVersion = swiftTestTarget?.swiftVersion ?? SwiftLanguageVersion(string: String(ToolsVersion.currentToolsVersion.major)) ?? .v4
         let sources = Sources(paths: [linuxMain], root: linuxMain.parentDirectory)
         super.init(name: name, type: .executable, sources: sources, dependencies: dependencies)
     }

Sources/TestSupport/MockDependencyGraph.swift

@@ -146,7 +146,8 @@ public struct MockManifestGraph {
             package: .v3(PackageDescription.Package(
                 name: "Root",
                 dependencies: MockManifestGraph.createDependencies(repos: repos, dependencies: rootDeps))),
-            version: nil
+            version: nil,
+            manifestVersion: .v3
         )
 
         // Create the manifests from mock packages.
@@ -160,7 +161,9 @@ public struct MockManifestGraph {
                     dependencies: MockManifestGraph.createDependencies(
                         repos: repos,
                         dependencies: package.dependencies))),
-                version: package.version)
+                version: package.version,
+                manifestVersion: .v3
+            )
             return (MockManifestLoader.Key(url: url, version: package.version), manifest)
         }))
         // Add the root manifest.

Sources/TestSupport/misc.swift

@@ -214,7 +214,8 @@ public func loadMockPackageGraph(
             path: AbsolutePath(url).appending(component: Manifest.filename),
             url: url,
             package: .v3(package),
-            version: "1.0.0"
+            version: "1.0.0",
+            manifestVersion: .v3
         )
         if url == root {
             rootManifest = manifest
@@ -239,7 +240,8 @@ public func loadMockPackageGraph4(
             path: AbsolutePath(url).appending(component: Manifest.filename),
             url: url,
             package: .v4(package),
-            version: "1.0.0"
+            version: "1.0.0",
+            manifestVersion: .v4
         )
         if url == root {
             rootManifest = manifest

Tests/FunctionalTests/ToolsVersionTests.swift

@@ -78,7 +78,7 @@ class ToolsVersionTests: XCTestCase {
                     """
             }
             _ = try SwiftPMProduct.SwiftPackage.execute(
-                ["tools-version", "--set-current"], packagePath: primaryPath).chomp()
+                ["tools-version", "--set", "4.0"], packagePath: primaryPath).chomp()
 
             // Build the primary package.
             _ = try SwiftPMProduct.SwiftBuild.execute([], packagePath: primaryPath)
@@ -109,15 +109,15 @@ class ToolsVersionTests: XCTestCase {
                     """
             }
             _ = try SwiftPMProduct.SwiftPackage.execute(
-                ["tools-version", "--set-current"], packagePath: primaryPath).chomp()
+                ["tools-version", "--set", "4.0"], packagePath: primaryPath).chomp()
 
             do {
                 _ = try SwiftPMProduct.SwiftBuild.execute([], packagePath: primaryPath)
                 XCTFail()
             } catch SwiftPMProductError.executionFailure(_, _, let stderr) {
                 XCTAssertTrue(
                     stderr.contains("package 'Primary' not compatible with current tools version (") &&
-                    stderr.contains("); it supports: 1000\n"))
+                    stderr.contains("); it supports: 1000\n"), stderr)
             }
 
              try fs.writeFileContents(primaryPath.appending(component: "Package.swift")) {
@@ -131,7 +131,7 @@ class ToolsVersionTests: XCTestCase {
                     """
              }
              _ = try SwiftPMProduct.SwiftPackage.execute(
-                 ["tools-version", "--set-current"], packagePath: primaryPath).chomp()
+                 ["tools-version", "--set", "4.0"], packagePath: primaryPath).chomp()
              _ = try SwiftPMProduct.SwiftBuild.execute([], packagePath: primaryPath)
         }
     }

Tests/PackageGraphPerformanceTests/PackageGraphPerfTests.swift

@@ -41,7 +41,8 @@ class PackageGraphPerfTests: XCTestCasePerf {
                 path: AbsolutePath(url).appending(component: Manifest.filename),
                 url: url,
                 package: .v3(package),
-                version: "1.0.0"
+                version: "1.0.0",
+                manifestVersion: .v3
             )
             if pkg == 1 {
                 rootManifests = [manifest]

Tests/PackageGraphTests/RepositoryPackageContainerProviderTests.swift

@@ -187,7 +187,8 @@ class RepositoryPackageContainerProviderTests: XCTestCase {
                                 .Package(url: "B", majorVersion: 2)
                             ]
                         )),
-                        version: v1
+                        version: v1,
+                        manifestVersion: .v3
                     )
                 ])
             let repoB = MockRepository(
@@ -198,7 +199,8 @@ class RepositoryPackageContainerProviderTests: XCTestCase {
                         url: "B",
                         package: .v3(PackageDescription.Package(
                             name: "Bar")),
-                        version: v2
+                        version: v2,
+                        manifestVersion: .v3
                     )
                 ])
             let resolver = MockDependencyResolver(repositories: repoA, repoB)