Auto merge of #4836 - bundler:seg-resolve-for-specific-platforms, r=i…

…ndirect

Resolve for specific platforms

Closes #4295.

This will require adding a bunch of tests, as well as figuring out how to put this new behavior behind a feature flag (thus fixing all of the existing tests).

lib/bundler/definition.rb

@@ -69,7 +69,8 @@ def initialize(lockfile, dependencies, sources, unlock, ruby_version = nil, opti
       if lockfile && File.exist?(lockfile)
         @lockfile_contents = Bundler.read_file(lockfile)
         @locked_gems = LockfileParser.new(@lockfile_contents)
-        @platforms = @locked_gems.platforms
+        @locked_platforms = @locked_gems.platforms
+        @platforms = @locked_platforms.dup
         @locked_bundler_version = @locked_gems.bundler_version
         @locked_ruby_version = @locked_gems.ruby_version
 
@@ -90,6 +91,7 @@ def initialize(lockfile, dependencies, sources, unlock, ruby_version = nil, opti
         @locked_deps    = []
         @locked_specs   = SpecSet.new([])
         @locked_sources = []
+        @locked_platforms = []
       end
 
       @unlock[:gems] ||= []
@@ -105,8 +107,9 @@ def initialize(lockfile, dependencies, sources, unlock, ruby_version = nil, opti
 
       @gem_version_promoter = create_gem_version_promoter
 
-      current_platform = Bundler.rubygems.platforms.map {|p| generic(p) }.compact.last
-      add_platform(current_platform)
+      current_platform = Bundler.rubygems.platforms.last
+      add_platform(current_platform) if Bundler.settings[:specific_platform]
+      add_platform(generic(current_platform))
 
       @path_changes = converge_paths
       eager_unlock = expand_dependencies(@unlock[:gems])
@@ -403,6 +406,11 @@ def ensure_equivalent_gemfile_and_lockfile(explicit_flag = false)
       deleted = []
       changed = []
 
+      new_platforms = @platforms - @locked_platforms
+      deleted_platforms = @locked_platforms - @platforms
+      added.concat new_platforms.map {|p| "* platform: #{p}" }
+      deleted.concat deleted_platforms.map {|p| "* platform: #{p}" }
+
       gemfile_sources = sources.lock_sources
 
       new_sources = gemfile_sources - @locked_sources

lib/bundler/dependency.rb

@@ -52,7 +52,7 @@ class Dependency < Gem::Dependency
       :x64_mingw_20 => Gem::Platform::X64_MINGW,
       :x64_mingw_21 => Gem::Platform::X64_MINGW,
       :x64_mingw_22 => Gem::Platform::X64_MINGW,
-      :x64_mingw_23 => Gem::Platform::X64_MINGW
+      :x64_mingw_23 => Gem::Platform::X64_MINGW,
     }.freeze
 
     REVERSE_PLATFORM_MAP = {}.tap do |reverse_platform_map|

lib/bundler/gem_helpers.rb

@@ -28,5 +28,73 @@ def generic_local_platform
       generic(Gem::Platform.local)
     end
     module_function :generic_local_platform
+
+    def platform_specificity_match(spec_platform, user_platform)
+      spec_platform = Gem::Platform.new(spec_platform)
+      return PlatformMatch::EXACT_MATCH if spec_platform == user_platform
+      return PlatformMatch::WORST_MATCH if spec_platform.nil? || spec_platform == Gem::Platform::RUBY || user_platform == Gem::Platform::RUBY
+
+      PlatformMatch.new(
+        PlatformMatch.os_match(spec_platform, user_platform),
+        PlatformMatch.cpu_match(spec_platform, user_platform),
+        PlatformMatch.platform_version_match(spec_platform, user_platform)
+      )
+    end
+    module_function :platform_specificity_match
+
+    def select_best_platform_match(specs, platform)
+      specs.select {|spec| spec.match_platform(platform) }.
+        min_by {|spec| platform_specificity_match(spec.platform, platform) }
+    end
+    module_function :select_best_platform_match
+
+    PlatformMatch = Struct.new(:os_match, :cpu_match, :platform_version_match)
+    class PlatformMatch
+      def <=>(other)
+        return nil unless other.is_a?(PlatformMatch)
+
+        m = os_match <=> other.os_match
+        return m unless m.zero?
+
+        m = cpu_match <=> other.cpu_match
+        return m unless m.zero?
+
+        m = platform_version_match <=> other.platform_version_match
+        m
+      end
+
+      EXACT_MATCH = new(-1, -1, -1).freeze
+      WORST_MATCH = new(1_000_000, 1_000_000, 1_000_000).freeze
+
+      def self.os_match(spec_platform, user_platform)
+        if spec_platform.os == user_platform.os
+          0
+        else
+          1
+        end
+      end
+
+      def self.cpu_match(spec_platform, user_platform)
+        if spec_platform.cpu == user_platform.cpu
+          0
+        elsif spec_platform.cpu == "arm" && user_platform.cpu.to_s.start_with?("arm")
+          0
+        elsif spec_platform.cpu.nil? || spec_platform.cpu == "universal"
+          1
+        else
+          2
+        end
+      end
+
+      def self.platform_version_match(spec_platform, user_platform)
+        if spec_platform.version == user_platform.version
+          0
+        elsif spec_platform.version.nil?
+          1
+        else
+          2
+        end
+      end
+    end
   end
 end

lib/bundler/lazy_specification.rb

@@ -55,7 +55,8 @@ def to_lock
     end
 
     def __materialize__
-      @specification = source.specs.search(Gem::Dependency.new(name, version)).last
+      search_object = Bundler.settings[:specific_platform] ? self : Dependency.new(name, version)
+      @specification = source.specs.search(search_object).last
     end
 
     def respond_to?(*args)

lib/bundler/match_platform.rb

@@ -8,7 +8,8 @@ module MatchPlatform
     def match_platform(p)
       Gem::Platform::RUBY == platform ||
         platform.nil? || p == platform ||
-        generic(Gem::Platform.new(platform)) === p
+        generic(Gem::Platform.new(platform)) === p ||
+        Gem::Platform.new(platform) === p
     end
   end
 end

lib/bundler/resolver.rb

@@ -66,44 +66,33 @@ def message
       end
     end
 
-    ALL = Bundler::Dependency::PLATFORM_MAP.values.uniq.freeze
-
     class SpecGroup < Array
       include GemHelpers
 
-      attr_reader :activated, :required_by
+      attr_reader :activated
 
       def initialize(a)
         super
         @required_by  = []
         @activated    = []
         @dependencies = nil
-        @specs        = {}
-
-        ALL.each do |p|
-          @specs[p] = reverse.find {|s| s.match_platform(p) }
+        @specs        = Hash.new do |specs, platform|
+          specs[platform] = select_best_platform_match(self, platform)
         end
       end
 
       def initialize_copy(o)
         super
-        @required_by = o.required_by.dup
-        @activated   = o.activated.dup
+        @activated = o.activated.dup
       end
 
       def to_specs
-        specs = {}
-
-        @activated.each do |p|
+        @activated.map do |p|
           next unless s = @specs[p]
-          platform = generic(Gem::Platform.new(s.platform))
-          next if specs[platform]
-
-          lazy_spec = LazySpecification.new(name, version, platform, source)
+          lazy_spec = LazySpecification.new(name, version, s.platform, source)
           lazy_spec.dependencies.replace s.dependencies
-          specs[platform] = lazy_spec
-        end
-        specs.values
+          lazy_spec
+        end.compact
       end
 
       def activate_platform!(platform)
@@ -150,17 +139,15 @@ def platforms_for_dependency_named(dependency)
     private
 
       def __dependencies
-        @dependencies ||= begin
-          dependencies = {}
-          ALL.each do |p|
-            next unless spec = @specs[p]
-            dependencies[p] = []
+        @dependencies = Hash.new do |dependencies, platform|
+          dependencies[platform] = []
+          if spec = @specs[platform]
             spec.dependencies.each do |dep|
               next if dep.type == :development
-              dependencies[p] << DepProxy.new(dep, p)
+              dependencies[platform] << DepProxy.new(dep, platform)
             end
           end
-          dependencies
+          dependencies[platform]
         end
       end
     end

lib/bundler/runtime.rb

@@ -140,7 +140,8 @@ def cache(custom_path = nil)
 
       Bundler.ui.info "Updating files in #{Bundler.settings.app_cache_path}"
 
-      specs.each do |spec|
+      specs_to_cache = Bundler.settings[:cache_all_platforms] ? @definition.resolve.materialized_for_all_platforms : specs
+      specs_to_cache.each do |spec|
         next if spec.name == "bundler"
         next if spec.source.is_a?(Source::Gemspec)
         spec.source.send(:fetch_gem, spec) if Bundler.settings[:cache_all_platforms] && spec.source.respond_to?(:fetch_gem, true)

lib/bundler/source.rb

@@ -14,6 +14,7 @@ def unmet_deps
 
     def version_message(spec)
       message = "#{spec.name} #{spec.version}"
+      message += " (#{spec.platform})" if spec.platform != Gem::Platform::RUBY
 
       if Bundler.locked_gems
         locked_spec = Bundler.locked_gems.specs.find {|s| s.name == spec.name }

lib/bundler/spec_set.rb

@@ -24,12 +24,13 @@ def for(dependencies, skip = [], check = false, match_current_platform = false)
         dep = deps.shift
         next if handled[dep] || skip.include?(dep.name)
 
-        spec = lookup[dep.name].find do |s|
-          if match_current_platform
-            Gem::Platform.match(s.platform)
-          else
-            s.match_platform(dep.__platform)
+        spec = if match_current_platform
+          Bundler.rubygems.platforms.reverse_each do |pl|
+            match = GemHelpers.select_best_platform_match(lookup[dep.name], pl)
+            break match if match
           end
+        else
+          GemHelpers.select_best_platform_match(lookup[dep.name], dep.__platform)
         end
 
         handled[dep] = true
@@ -99,6 +100,20 @@ def materialize(deps, missing_specs = nil)
       SpecSet.new(materialized.compact)
     end
 
+    # Materialize for all the specs in the spec set, regardless of what platform they're for
+    # This is in contrast to how for does platform filtering (and specifically different from how `materialize` calls `for` only for the current platform)
+    # @return [Array<Gem::Specification>]
+    def materialized_for_all_platforms
+      names = @specs.map(&:name).uniq
+      @specs.map do |s|
+        next s unless s.is_a?(LazySpecification)
+        s.source.dependency_names = names if s.source.respond_to?(:dependency_names=)
+        spec = s.__materialize__
+        raise GemNotFound, "Could not find #{s.full_name} in any of the sources" unless spec
+        spec
+      end
+    end
+
     def merge(set)
       arr = sorted.dup
       set.each do |s|
@@ -133,10 +148,7 @@ def extract_circular_gems(error)
     def lookup
       @lookup ||= begin
         lookup = Hash.new {|h, k| h[k] = [] }
-        specs = @specs.sort_by do |s|
-          s.platform.to_s == "ruby" ? "\0" : s.platform.to_s
-        end
-        specs.reverse_each do |s|
+        Index.sort_specs(@specs).reverse_each do |s|
           lookup[s.name] << s
         end
         lookup

spec/bundler/source_spec.rb

@@ -22,7 +22,7 @@ class ExampleSource < Bundler::Source
   end
 
   describe "#version_message" do
-    let(:spec) { double(:spec, :name => "nokogiri", :version => ">= 1.6") }
+    let(:spec) { double(:spec, :name => "nokogiri", :version => ">= 1.6", :platform => rb) }
 
     shared_examples_for "the lockfile specs are not relevant" do
       it "should return a string with the spec name and version" do

spec/install/gemfile/specific_platform_spec.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+require "spec_helper"
+
+describe "bundle install with specific_platform enabled" do
+  before do
+    bundle "config specific_platform true"
+
+    build_repo2 do
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.5.1")
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.5.1") {|s| s.platform = "x86_64-linux" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.5.1") {|s| s.platform = "x86-mingw32" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.5.1") {|s| s.platform = "x86-linux" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.5.1") {|s| s.platform = "x64-mingw32" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.5.1") {|s| s.platform = "universal-darwin" }
+
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.5") {|s| s.platform = "x86_64-linux" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.5") {|s| s.platform = "x86-linux" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.5") {|s| s.platform = "x64-mingw32" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.5") {|s| s.platform = "x86-mingw32" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.5")
+
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.4") {|s| s.platform = "universal-darwin" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.4") {|s| s.platform = "x86_64-linux" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.4") {|s| s.platform = "x86-mingw32" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.4") {|s| s.platform = "x86-linux" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.4") {|s| s.platform = "x64-mingw32" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.4")
+
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.3")
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.3") {|s| s.platform = "x86_64-linux" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.3") {|s| s.platform = "x86-mingw32" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.3") {|s| s.platform = "x86-linux" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.3") {|s| s.platform = "x64-mingw32" }
+      build_gem("google-protobuf", "3.0.0.alpha.5.0.3") {|s| s.platform = "universal-darwin" }
+
+      build_gem("google-protobuf", "3.0.0.alpha.4.0")
+      build_gem("google-protobuf", "3.0.0.alpha.3.1.pre")
+      build_gem("google-protobuf", "3.0.0.alpha.3")
+      build_gem("google-protobuf", "3.0.0.alpha.2.0")
+      build_gem("google-protobuf", "3.0.0.alpha.1.1")
+      build_gem("google-protobuf", "3.0.0.alpha.1.0")
+    end
+  end
+
+  let(:google_protobuf) { <<-G }
+    source "file:#{gem_repo2}"
+    gem "google-protobuf"
+  G
+
+  context "when on a darwin machine" do
+    before { simulate_platform "x86_64-darwin-15" }
+
+    it "locks to both the specific darwin platform and ruby" do
+      install_gemfile!(google_protobuf)
+      expect(the_bundle.locked_gems.platforms).to eq([pl("ruby"), pl("x86_64-darwin-15")])
+      expect(the_bundle).to include_gem("google-protobuf 3.0.0.alpha.5.0.5.1 universal-darwin")
+      expect(the_bundle.locked_gems.specs.map(&:full_name)).to eq(%w(
+        google-protobuf-3.0.0.alpha.5.0.5.1
+        google-protobuf-3.0.0.alpha.5.0.5.1-universal-darwin
+      ))
+    end
+
+    it "caches both the universal-darwin and ruby gems when --all-platforms is passed" do
+      gemfile(google_protobuf)
+      bundle! "package --all-platforms"
+      expect([cached_gem("google-protobuf-3.0.0.alpha.5.0.5.1"), cached_gem("google-protobuf-3.0.0.alpha.5.0.5.1-universal-darwin")]).
+        to all(exist)
+    end
+
+    context "when adding a platform via lock --add_platform" do
+      it "adds the foreign platform" do
+        install_gemfile!(google_protobuf)
+        bundle! "lock --add-platform=#{x64_mingw}"
+
+        expect(the_bundle.locked_gems.platforms).to eq([rb, x64_mingw, pl("x86_64-darwin-15")])
+        expect(the_bundle.locked_gems.specs.map(&:full_name)).to eq(%w(
+          google-protobuf-3.0.0.alpha.5.0.5.1
+          google-protobuf-3.0.0.alpha.5.0.5.1-universal-darwin
+          google-protobuf-3.0.0.alpha.5.0.5.1-x64-mingw32
+        ))
+      end
+
+      it "falls back on plain ruby when that version doesnt have a platform-specific gem" do
+        install_gemfile!(google_protobuf)
+        bundle! "lock --add-platform=#{java}"
+
+        expect(the_bundle.locked_gems.platforms).to eq([java, rb, pl("x86_64-darwin-15")])
+        expect(the_bundle.locked_gems.specs.map(&:full_name)).to eq(%w(
+          google-protobuf-3.0.0.alpha.5.0.5.1
+          google-protobuf-3.0.0.alpha.5.0.5.1-universal-darwin
+        ))
+      end
+    end
+  end
+end