(PDK-985) Split targets into chunks of 1000

lib/pdk/cli/exec_group.rb

@@ -18,7 +18,7 @@ def initialize(message, opts = {})
           @spinner.auto_spin
         end
 
-        @threads = []
+        @threads_or_procs = []
         @exit_codes = []
       end
 
@@ -29,15 +29,15 @@ def parallel?
       def register(&block)
         raise PDK::CLI::FatalError, 'No block registered' unless block_given?
 
-        @threads << if parallel?
-                      Thread.new do
-                        GettextSetup.initialize(File.absolute_path('../../../locales', File.dirname(__FILE__)))
-                        GettextSetup.negotiate_locale!(GettextSetup.candidate_locales)
-                        @exit_codes << yield
-                      end
-                    else
-                      block
-                    end
+        @threads_or_procs << if parallel?
+                               Thread.new do
+                                 GettextSetup.initialize(File.absolute_path('../../../locales', File.dirname(__FILE__)))
+                                 GettextSetup.negotiate_locale!(GettextSetup.candidate_locales)
+                                 @exit_codes << yield
+                               end
+                             else
+                               block
+                             end
       end
 
       def add_spinner(message, opts = {})
@@ -47,9 +47,9 @@ def add_spinner(message, opts = {})
 
       def exit_code
         if parallel?
-          @threads.each(&:join)
+          @threads_or_procs.each(&:join)
         else
-          @exit_codes = @threads.map(&:call)
+          @exit_codes = @threads_or_procs.map(&:call)
         end
 
         exit_code = @exit_codes.max

lib/pdk/validate/base_validator.rb

@@ -115,7 +115,14 @@ def self.invoke(report, options = {})
         # targets array in another array. This is so we can loop through the
         # invokes with the same logic, regardless of which invoke style is
         # needed.
-        targets = (self::INVOKE_STYLE == :per_target) ? targets.combination(1).to_a : Array[targets]
+        #
+        if self::INVOKE_STYLE == :per_target
+          targets = targets.combination(1).to_a
+        else
+          targets = targets.each_slice(1000).to_a
+          options[:split_exec] = PDK::CLI::ExecGroup.new(spinner_text(targets), parallel: false)
+        end
+
         exit_codes = []
 
         targets.each do |invokation_targets|
@@ -124,22 +131,32 @@ def self.invoke(report, options = {})
 
           command = PDK::CLI::Exec::Command.new(*cmd_argv).tap do |c|
             c.context = :module
-            exec_group = options[:exec_group]
-            if exec_group
-              sub_spinner = exec_group.add_spinner(spinner_text(invokation_targets))
-              c.register_spinner(sub_spinner)
-            else
-              c.add_spinner(spinner_text(invokation_targets))
+            unless options[:split_exec]
+              exec_group = options[:exec_group]
+              if exec_group
+                sub_spinner = exec_group.add_spinner(spinner_text(invokation_targets))
+                c.register_spinner(sub_spinner)
+              else
+                c.add_spinner(spinner_text(invokation_targets))
+              end
             end
           end
 
-          result = command.execute!
-          exit_codes << result[:exit_code]
+          if options[:split_exec]
+            options[:split_exec].register do
+              result = command.execute!
+              parse_output(report, result, invokation_targets)
+              result[:exit_code]
+            end
+          else
+            result = command.execute!
+            exit_codes << result[:exit_code]
 
-          parse_output(report, result, invokation_targets)
+            parse_output(report, result, invokation_targets)
+          end
         end
 
-        exit_codes.max
+        options.key?(:split_exec) ? options[:split_exec].exit_code : exit_codes.max
       end
     end
   end

spec/unit/pdk/validate/base_validator_spec.rb

@@ -9,6 +9,43 @@
     end
   end
 
+  describe '.invoke' do
+    before(:each) do
+      allow(described_class).to receive(:parse_targets).and_return([(1..1001).map(&:to_s), [], []])
+      allow(described_class).to receive(:cmd).and_return('dummy_cmd')
+      allow(PDK::Util::Bundler).to receive(:ensure_binstubs!).with('dummy_cmd')
+      allow(described_class).to receive(:parse_output)
+    end
+
+    let(:dummy_exec) do
+      instance_double(PDK::CLI::Exec::Command, :context= => nil, :execute! => { exit_code: 0 })
+    end
+
+    after(:each) do
+      described_class.invoke(instance_double(PDK::Report))
+    end
+
+    context 'when validating less than 1000 targets' do
+      before(:each) do
+        allow(described_class).to receive(:parse_targets).and_return([(1..999).map(&:to_s), [], []])
+      end
+
+      it 'executes the validator once' do
+        expect(PDK::CLI::Exec::Command).to receive(:new).and_return(dummy_exec).once
+      end
+    end
+
+    context 'when validating more than 1000 targets' do
+      before(:each) do
+        allow(described_class).to receive(:parse_targets).and_return([(1..1001).map(&:to_s), [], []])
+      end
+
+      it 'executes the validator for each block of up to 1000 targets' do
+        expect(PDK::CLI::Exec::Command).to receive(:new).and_return(dummy_exec).twice
+      end
+    end
+  end
+
   describe '.parse_targets' do
     subject(:target_files) { described_class.parse_targets(targets: targets) }