(PDK-1096, PDK-1097, PDK-1098) Add puppet-dev flag to validate and test unit

lib/pdk/cli.rb

@@ -55,6 +55,12 @@ def self.puppet_version_options(dsl)
     dsl.option nil, 'pe-version', _('Puppet Enterprise version to run tests or validations against.'), argument: :required
   end
 
+  def self.puppet_dev_option(dsl)
+    dsl.option nil,
+               'puppet-dev',
+               _('When specified, PDK will validate or test against the current Puppet source from github.com. To use this option, you must have network access to https://github.com.')
+  end
+
   @base_cmd = Cri::Command.define do
     name 'pdk'
     usage _('pdk command [options]')

lib/pdk/cli/test/unit.rb

@@ -8,6 +8,7 @@ module PDK::CLI
     summary _('Run unit tests.')
 
     PDK::CLI.puppet_version_options(self)
+    PDK::CLI.puppet_dev_option(self)
     flag nil, :list, _('List all available unit test files.')
     flag nil, :parallel, _('Run unit tests in parallel.')
     flag :v, :verbose, _('More verbose --list output. Displays a list of examples in each unit test file.')
@@ -67,6 +68,7 @@ module PDK::CLI
 
         # Ensure that the bundled gems are up to date and correct Ruby is activated before running tests.
         puppet_env = PDK::CLI::Util.puppet_from_opts_or_env(opts)
+        PDK::Util::PuppetVersion.fetch_puppet_dev if opts.key?(:'puppet-dev')
         PDK::Util::RubyVersion.use(puppet_env[:ruby_version])
         PDK::Util::Bundler.ensure_bundle!(puppet_env[:gemset])
 

lib/pdk/cli/util.rb

@@ -83,12 +83,15 @@ def module_version_check
       module_function :module_version_check
 
       def puppet_from_opts_or_env(opts)
+        use_puppet_dev = (opts || {})[:'puppet-dev'] || ENV['PDK_PUPPET_DEV']
         desired_puppet_version = (opts || {})[:'puppet-version'] || ENV['PDK_PUPPET_VERSION']
         desired_pe_version = (opts || {})[:'pe-version'] || ENV['PDK_PE_VERSION']
 
         begin
           puppet_env =
-            if desired_puppet_version
+            if use_puppet_dev
+              PDK::Util::PuppetVersion.puppet_dev_env
+            elsif desired_puppet_version
               PDK::Util::PuppetVersion.find_gem_for(desired_puppet_version)
             elsif desired_pe_version
               PDK::Util::PuppetVersion.from_pe_version(desired_pe_version)
@@ -132,6 +135,21 @@ def validate_puppet_version_opts(opts)
         pe_ver_specs << '--pe-version option' if opts[:'pe-version']
         pe_ver_specs << 'PDK_PE_VERSION environment variable' if ENV['PDK_PE_VERSION'] && !ENV['PDK_PE_VERSION'].empty?
 
+        puppet_dev_specs = []
+        puppet_dev_specs << '--puppet-dev flag' if opts[:'puppet-dev']
+        puppet_dev_specs << 'PDK_PUPPET_DEV environment variable' if ENV['PDK_PUPPET_DEV'] && !ENV['PDK_PUPPET_DEV'].empty?
+
+        puppet_dev_specs.each do |pup_dev_spec|
+          [puppet_ver_specs, pe_ver_specs].each do |offending|
+            next if offending.empty?
+
+            raise PDK::CLI::ExitWithError, _('You cannot specify a %{first} and %{second} at the same time') % {
+              first: pup_dev_spec,
+              second: offending.first,
+            }
+          end
+        end
+
         puppet_ver_specs.each do |pup_ver_spec|
           next if pe_ver_specs.empty?
 
@@ -143,7 +161,15 @@ def validate_puppet_version_opts(opts)
           }
         end
 
-        if puppet_ver_specs.size == 2
+        if puppet_dev_specs.size == 2
+          warning_str = 'Puppet dev flag from command line: "--puppet-dev" '
+          warning_str += 'overrides value from environment: "PDK_PUPPET_DEV=true". You should not specify both.'
+
+          PDK.logger.warn(_(warning_str) % {
+            pup_ver_opt: opts[:'puppet-dev'],
+            pup_ver_env: ENV['PDK_PUPPET_DEV'],
+          })
+        elsif puppet_ver_specs.size == 2
           warning_str = 'Puppet version option from command line: "--puppet-version=%{pup_ver_opt}" '
           warning_str += 'overrides value from environment: "PDK_PUPPET_VERSION=%{pup_ver_env}". You should not specify both.'
 

lib/pdk/cli/validate.rb

@@ -14,6 +14,7 @@ module PDK::CLI
     )
 
     PDK::CLI.puppet_version_options(self)
+    PDK::CLI.puppet_dev_option(self)
     flag nil, :list, _('List all available validators.')
     flag :a, 'auto-correct', _('Automatically correct problems where possible.')
     flag nil, :parallel, _('Run validations in parallel.')
@@ -87,6 +88,7 @@ module PDK::CLI
 
       # Ensure that the bundled gems are up to date and correct Ruby is activated before running any validations.
       puppet_env = PDK::CLI::Util.puppet_from_opts_or_env(opts)
+      PDK::Util::PuppetVersion.fetch_puppet_dev if opts.key?(:'puppet-dev')
       PDK::Util::RubyVersion.use(puppet_env[:ruby_version])
       PDK::Util::Bundler.ensure_bundle!(puppet_env[:gemset])
 

lib/pdk/util/puppet_version.rb

@@ -1,12 +1,13 @@
 require 'pdk/util'
+require 'pdk/util/git'
 
 module PDK
   module Util
     class PuppetVersion
       class << self
         extend Forwardable
 
-        def_delegators :instance, :find_gem_for, :from_pe_version, :from_module_metadata, :latest_available
+        def_delegators :instance, :puppet_dev_env, :puppet_dev_path, :fetch_puppet_dev, :find_gem_for, :from_pe_version, :from_module_metadata, :latest_available
 
         attr_writer :instance
 
@@ -16,6 +17,19 @@ def instance
       end
 
       PE_VERSIONS_URL = 'https://forgeapi.puppet.com/private/versions/pe'.freeze
+      DEFAULT_PUPPET_DEV_URL = 'https://github.com/puppetlabs/puppet'.freeze
+      DEFAULT_PUPPET_DEV_BRANCH = 'master'.freeze
+
+      def puppet_dev_env
+        {
+          gem_version: 'file://%{path}' % { path: puppet_dev_path },
+          ruby_version: PDK::Util::RubyVersion.default_ruby_version,
+        }
+      end
+
+      def puppet_dev_path
+        File.join(PDK::Util.cachedir, 'src', 'puppet')
+      end
 
       def latest_available
         latest = find_gem(Gem::Requirement.create('>= 0'))
@@ -27,6 +41,25 @@ def latest_available
         latest
       end
 
+      def fetch_puppet_dev
+        unless PDK::Util::Git.remote_repo? puppet_dev_path
+          FileUtils.mkdir_p puppet_dev_path
+          clone_result = PDK::Util::Git.git('clone', DEFAULT_PUPPET_DEV_URL, puppet_dev_path)
+          return if clone_result[:exit_code].zero?
+
+          PDK.logger.error clone_result[:stdout]
+          PDK.logger.error clone_result[:stderr]
+          raise PDK::CLI::FatalError, _("Unable to clone git repository at '%{repo}'.") % { repo: DEFAULT_PUPPET_DEV_URL }
+        end
+
+        fetch_result = PDK::Util::Git.git('--git-dir', File.join(puppet_dev_path, '.git'), 'pull', '--ff-only')
+        return if fetch_result[:exit_code].zero?
+
+        PDK.logger.error fetch_result[:stdout]
+        PDK.logger.error fetch_result[:stderr]
+        raise PDK::CLI::FatalError, _("Unable to pull updates for git repository at '%{cachedir}'.") % { repo: puppet_dev_path }
+      end
+
       def find_gem_for(version_str)
         version = parse_specified_version(version_str)
 

spec/acceptance/version_changer_spec.rb

@@ -25,5 +25,10 @@
         its(:content) { is_expected.to match(%r{^\s+puppet \(#{Regexp.escape(puppet_version)}(\)|-)}im) }
       end
     end
+
+    describe command('pdk validate --puppet-dev') do
+      its(:stderr) { is_expected.to match(%r{Using Puppet file://}i) }
+      its(:exit_status) { is_expected.to eq(0) }
+    end
   end
 end

spec/unit/pdk/cli/test/unit_spec.rb

@@ -150,6 +150,38 @@
     end
   end
 
+  context 'with --puppet-dev' do
+    let(:puppet_env) do
+      {
+        ruby_version: '2.4.3',
+        gemset: { puppet: 'file://path/to/puppet' },
+      }
+    end
+
+    before(:each) do
+      allow(PDK::CLI::Util).to receive(:puppet_from_opts_or_env).with(hash_including(:'puppet-dev' => true)).and_return(puppet_env)
+      allow(PDK::Util::PuppetVersion).to receive(:fetch_puppet_dev).and_return(nil)
+      allow(PDK::Test::Unit).to receive(:invoke).and_return(0)
+      allow(PDK::CLI::Util).to receive(:module_version_check)
+    end
+
+    it 'activates puppet github source' do
+      expect(PDK::Util::Bundler).to receive(:ensure_bundle!).with(puppet_env[:gemset])
+
+      expect {
+        test_unit_cmd.run_this(['--puppet-dev'])
+      }.to exit_zero
+    end
+
+    it 'activates resolved ruby version' do
+      expect(PDK::Util::RubyVersion).to receive(:use).with(puppet_env[:ruby_version])
+
+      expect {
+        test_unit_cmd.run_this(['--puppet-dev'])
+      }.to exit_zero
+    end
+  end
+
   context 'when --puppet-version and --pe-version are specified' do
     it 'exits with an error' do
       expect(logger).to receive(:error).with(a_string_matching(%r{cannot specify.*--pe-version.*and.*--puppet-version}i))

spec/unit/pdk/cli/util_spec.rb

@@ -178,6 +178,51 @@
       { ruby_version: ruby_version, gem_version: Gem::Version.new(puppet_version) }
     end
 
+    context 'when puppet-dev has been set' do
+      let(:options) { { :'puppet-dev' => true } }
+      let(:ruby_version) { '2.4.4' }
+      let(:puppet_version) { 'path/to/puppet' }
+
+      let(:version_result) do
+        {
+          gem_version:  puppet_version,
+          ruby_version: ruby_version,
+        }
+      end
+
+      before(:each) do
+        allow(PDK::Util::PuppetVersion).to receive(:puppet_dev_path).and_return(puppet_version)
+        allow(PDK::Util::PuppetVersion).to receive(:puppet_dev_env).and_return(version_result)
+      end
+
+      it_behaves_like 'it returns a puppet environment'
+    end
+
+    context 'when PDK_PUPPET_DEV has been set' do
+      let(:options) { {} }
+      let(:ruby_version) { '2.4.4' }
+      let(:puppet_version) { 'path/to/puppet' }
+
+      let(:version_result) do
+        {
+          gem_version:  puppet_version,
+          ruby_version: ruby_version,
+        }
+      end
+
+      before(:each) do
+        allow(PDK::Util::PuppetVersion).to receive(:puppet_dev_path).and_return(puppet_version)
+        allow(PDK::Util::PuppetVersion).to receive(:puppet_dev_env).and_return(version_result)
+        ENV['PDK_PUPPET_DEV'] = 'true'
+      end
+
+      after(:each) do
+        ENV.delete('PDK_PUPPET_DEV')
+      end
+
+      it_behaves_like 'it returns a puppet environment'
+    end
+
     context 'when puppet-version has been set' do
       let(:options) { { :'puppet-version' => '4.10.10' } }
       let(:ruby_version) { '2.1.9' }
@@ -305,6 +350,76 @@
     let(:cli_pe_version) { '2016.2' }
     let(:env_pe_version) { '2017.3' }
 
+    context 'when --puppet-dev is set' do
+      let(:options) { { :'puppet-dev' => true } }
+
+      it 'is silent' do
+        expect(logger).not_to receive(:warn)
+
+        expect { validate_puppet_version_opts }.not_to raise_error
+      end
+
+      context 'when PDK_PUPPET_DEV is also set' do
+        before(:each) do
+          ENV['PDK_PUPPET_DEV'] = 'true'
+        end
+
+        after(:each) do
+          ENV.delete('PDK_PUPPET_DEV')
+        end
+
+        it 'warns about option precedence' do
+          expect(logger).to receive(:warn).with(%r{dev flag.*overrides.*environment}i)
+
+          validate_puppet_version_opts
+        end
+      end
+
+      context 'when --puppet-version is also set' do
+        let(:options) { { :'puppet-version' => cli_puppet_version, :'puppet-dev' => true } }
+
+        it 'exits with error' do
+          expect { validate_puppet_version_opts }.to raise_error(PDK::CLI::ExitWithError, %r{cannot specify.*flag.*and.*option}i)
+        end
+      end
+
+      context 'when PDK_PUPPET_VERSION is also set' do
+        before(:each) do
+          ENV['PDK_PUPPET_VERSION'] = env_puppet_version
+        end
+
+        after(:each) do
+          ENV.delete('PDK_PUPPET_VERSION')
+        end
+
+        it 'exits with error' do
+          expect { validate_puppet_version_opts }.to raise_error(PDK::CLI::ExitWithError, %r{cannot specify.*flag.*and.*environment}i)
+        end
+      end
+
+      context 'when --pe-version is also set' do
+        let(:options) { { :'pe-version' => cli_pe_version, :'puppet-dev' => true } }
+
+        it 'exits with error' do
+          expect { validate_puppet_version_opts }.to raise_error(PDK::CLI::ExitWithError, %r{cannot specify.*flag.*and.*option}i)
+        end
+      end
+
+      context 'when PDK_PE_VERSION is also set' do
+        before(:each) do
+          ENV['PDK_PE_VERSION'] = env_pe_version
+        end
+
+        after(:each) do
+          ENV.delete('PDK_PE_VERSION')
+        end
+
+        it 'exits with error' do
+          expect { validate_puppet_version_opts }.to raise_error(PDK::CLI::ExitWithError, %r{cannot specify.*flag.*and.*environment}i)
+        end
+      end
+    end
+
     context 'when --puppet-version is set' do
       let(:options) { { :'puppet-version' => cli_puppet_version } }
 

spec/unit/pdk/cli/validate_spec.rb

@@ -152,6 +152,56 @@
     end
   end
 
+  context 'with --puppet-dev' do
+    let(:puppet_env) do
+      {
+        ruby_version: '2.4.3',
+        gemset: { puppet: 'file://path/to/puppet' },
+      }
+    end
+
+    before(:each) do
+      allow(PDK::CLI::Util).to receive(:puppet_from_opts_or_env).with(hash_including(:'puppet-dev' => true)).and_return(puppet_env)
+      allow(PDK::Util::PuppetVersion).to receive(:fetch_puppet_dev).and_return(nil)
+    end
+
+    it 'activates puppet github source' do
+      expect(PDK::Util::Bundler).to receive(:ensure_bundle!).with(puppet_env[:gemset])
+
+      expect {
+        PDK::CLI.run(['validate', '--puppet-dev'])
+      }.to exit_zero
+    end
+
+    it 'activates resolved ruby version' do
+      expect(PDK::Util::RubyVersion).to receive(:use).with(puppet_env[:ruby_version])
+
+      expect {
+        PDK::CLI.run(['validate', '--puppet-dev'])
+      }.to exit_zero
+    end
+  end
+
+  context 'with both --puppet-version and --puppet-dev' do
+    it 'exits with an error' do
+      expect(logger).to receive(:error).with(a_string_matching(%r{cannot specify.*--puppet-dev.*and.*--puppet-version}i))
+
+      expect {
+        PDK::CLI.run(%w[validate --puppet-version 4.10.10 --puppet-dev])
+      }.to exit_nonzero
+    end
+  end
+
+  context 'with both --pe-version and --puppet-dev' do
+    it 'exits with an error' do
+      expect(logger).to receive(:error).with(a_string_matching(%r{cannot specify.*--puppet-dev.*and.*--pe-version}i))
+
+      expect {
+        PDK::CLI.run(%w[validate --pe-version 2018.1 --puppet-dev])
+      }.to exit_nonzero
+    end
+  end
+
   context 'with both --puppet-version and --pe-version' do
     it 'exits with an error' do
       expect(logger).to receive(:error).with(a_string_matching(%r{cannot specify.*--pe-version.*and.*--puppet-version}i))