(PDK-1053) Print validator output on parse_output failure

Gemfile

@@ -28,6 +28,7 @@ end
 group :test do
   gem 'coveralls'
   gem 'license_finder', '~> 3.0.4'
+  gem 'parser', '~> 2.5.1.2'
   gem 'rake', '~> 10.0'
   gem 'rspec', '~> 3.0'
   gem 'rspec-xsd'

lib/pdk/validate.rb

@@ -7,5 +7,7 @@ module Validate
     def self.validators
       @validators ||= [MetadataValidator, PuppetValidator, RubyValidator].freeze
     end
+
+    class ParseOutputError < StandardError; end
   end
 end

lib/pdk/validate/base_validator.rb

@@ -147,14 +147,22 @@ def self.invoke(report, options = {})
           if options[:split_exec]
             options[:split_exec].register do
               result = command.execute!
-              parse_output(report, result, invokation_targets)
+              begin
+                parse_output(report, result, invokation_targets)
+              rescue PDK::Validate::ParseOutputError => e
+                $stderr.puts e.message
+              end
               result[:exit_code]
             end
           else
             result = command.execute!
             exit_codes << result[:exit_code]
 
-            parse_output(report, result, invokation_targets)
+            begin
+              parse_output(report, result, invokation_targets)
+            rescue PDK::Validate::ParseOutputError => e
+              $stderr.puts e.message
+            end
           end
         end
 

lib/pdk/validate/metadata/metadata_json_lint.rb

@@ -49,14 +49,7 @@ def self.parse_output(report, result, targets)
           begin
             json_data = JSON.parse(result[:stdout])
           rescue JSON::ParserError
-            report.add_event(
-              file:     targets.first,
-              source:   name,
-              state:    :error,
-              severity: :error,
-              message:  result[:stdout],
-            )
-            return
+            raise PDK::Validate::ParseOutputError, result[:stdout]
           end
         end
 

lib/pdk/validate/puppet/puppet_lint.rb

@@ -34,7 +34,7 @@ def self.parse_output(report, result, targets)
         begin
           json_data = JSON.parse(result[:stdout]).flatten
         rescue JSON::ParserError
-          json_data = []
+          raise PDK::Validate::ParseOutputError, result[:stdout]
         end
 
         # puppet-lint does not include files without problems in its JSON

lib/pdk/validate/ruby/rubocop.rb

@@ -40,7 +40,7 @@ def self.parse_output(report, result, _targets)
         begin
           json_data = JSON.parse(result[:stdout])
         rescue JSON::ParserError
-          json_data = {}
+          raise PDK::Validate::ParseOutputError, result[:stdout]
         end
 
         return unless json_data.key?('files')

spec/unit/pdk/validate/base_validator_spec.rb

@@ -33,6 +33,18 @@
       it 'executes the validator once' do
         expect(PDK::CLI::Exec::Command).to receive(:new).and_return(dummy_exec).once
       end
+
+      context 'if the output fails to parse' do
+        before(:each) do
+          allow(described_class).to receive(:parse_output)
+            .with(any_args).and_raise(PDK::Validate::ParseOutputError, 'test')
+          allow(PDK::CLI::Exec::Command).to receive(:new).and_return(dummy_exec)
+        end
+
+        it 'prints the validator output to STDERR' do
+          expect($stderr).to receive(:puts).with('test')
+        end
+      end
     end
 
     context 'when validating more than 1000 targets' do
@@ -43,6 +55,18 @@
       it 'executes the validator for each block of up to 1000 targets' do
         expect(PDK::CLI::Exec::Command).to receive(:new).and_return(dummy_exec).twice
       end
+
+      context 'if the output fails to parse' do
+        before(:each) do
+          allow(described_class).to receive(:parse_output)
+            .with(any_args).and_raise(PDK::Validate::ParseOutputError, 'test').twice
+          allow(PDK::CLI::Exec::Command).to receive(:new).and_return(dummy_exec).twice
+        end
+
+        it 'prints the validator output to STDERR' do
+          expect($stderr).to receive(:puts).with('test').twice
+        end
+      end
     end
   end
 

spec/unit/pdk/validate/metadata_json_lint_spec.rb

@@ -105,15 +105,9 @@
       let(:metadata_json_lint_output) { 'some unhandled error' }
 
       it 'adds an error event for the target to the report' do
-        expect(report).to receive(:add_event).with(
-          file:     targets.first,
-          source:   described_class.name,
-          state:    :error,
-          severity: :error,
-          message:  metadata_json_lint_output,
-        )
-
-        parse_output
+        expect {
+          parse_output
+        }.to raise_error(PDK::Validate::ParseOutputError, 'some unhandled error')
       end
     end
 

spec/unit/pdk/validate/puppet_lint_spec.rb

@@ -79,12 +79,12 @@ def mock_lint(path, line, column, message, check, kind)
     end
 
     context 'when puppet-lint generates bad JSON' do
-      let(:lint_output) { '' }
+      let(:lint_output) { 'this is not JSON' }
 
       it 'adds no events to the report' do
-        expect(report).not_to receive(:add_event)
-
-        parse_output
+        expect {
+          parse_output
+        }.to raise_error(PDK::Validate::ParseOutputError, 'this is not JSON')
       end
     end
 

spec/unit/pdk/validate/rubocop_spec.rb

@@ -135,12 +135,12 @@ def mock_offense(severity, message, cop_name, corrected, line, column)
     end
 
     context 'when rubocop generates bad JSON' do
-      let(:rubocop_json) { '' }
+      let(:rubocop_json) { 'this is not JSON' }
 
       it 'does not add any events to the report' do
-        expect(report).not_to receive(:add_event)
-
-        parse_output
+        expect {
+          parse_output
+        }.to raise_error(PDK::Validate::ParseOutputError, 'this is not JSON')
       end
     end