Handle selective strict options.

lib/cucumber/cli/configuration.rb

@@ -26,7 +26,7 @@ def parse!(args)
         @args = args
         @options.parse!(args)
         arrange_formats
-        raise("You can't use both --strict and --wip") if strict? && wip?
+        raise("You can't use both --strict and --wip") if strict.strict? && wip?
         set_environment_variables
       end
 
@@ -42,7 +42,7 @@ def seed
         Integer(@options[:seed] || rand(0xFFFF))
       end
 
-      def strict?
+      def strict
         @options[:strict]
       end
 

lib/cucumber/cli/options.rb

@@ -3,6 +3,7 @@
 require 'cucumber/formatter/ansicolor'
 require 'cucumber/glue/registry_and_more'
 require 'cucumber/project_initializer'
+require 'cucumber/core/test/result'
 
 module Cucumber
   module Cli
@@ -119,7 +120,10 @@ def parse!(args) # rubocop:disable Metrics/AbcSize
           opts.on('-q', '--quiet', 'Alias for --no-snippets --no-source.') { shut_up }
           opts.on('--no-duration', "Don't print the duration at the end of the summary") { set_option :duration, false }
           opts.on('-b', '--backtrace', 'Show full backtrace for all errors.') { Cucumber.use_full_backtrace = true }
-          opts.on('-S', '--strict', 'Fail if there are any undefined or pending steps.') { set_option :strict }
+          opts.on('-S', '--[no-]strict', *strict_msg) { |setting| set_strict(setting) }
+          opts.on('--[no-]strict-undefined', 'Fail if there are any undefined results.') { |setting| set_strict(setting, :undefined) }
+          opts.on('--[no-]strict-pending', 'Fail if there are any pending results.') { |setting| set_strict(setting, :pending) }
+          opts.on('--[no-]strict-flaky', 'Fail if there are any flaky results.') { |setting| set_strict(setting, :flaky) }
           opts.on('-w', '--wip', 'Fail if there are any passing scenarios.') { set_option :wip }
           opts.on('-v', '--verbose', 'Show the files and features loaded.') { set_option :verbose }
           opts.on('-g', '--guess', 'Guess best match for Ambiguous steps.') { set_option :guess }
@@ -254,6 +258,13 @@ def name_msg
         ]
       end
 
+      def strict_msg
+        [
+          'Fail if there are any strict affected results ',
+          '(that is undefined, pending or flaky results).'
+        ]
+      end
+
       def parse_formats(v)
         formatter, *formatter_options = v.split(',')
         options_hash = Hash[formatter_options.map { |s| s.split('=') }]
@@ -409,6 +420,10 @@ def shut_up
         @options[:duration] = false
       end
 
+      def set_strict(setting, type = nil)
+        @options[:strict].set_strict(setting, type)
+      end
+
       def stdout_formats
         @options[:formats].select { |_, _, output| output == @out_stream }
       end
@@ -477,7 +492,7 @@ def reverse_merge(other_options)
         @options[:source] &= other_options[:source]
         @options[:snippets] &= other_options[:snippets]
         @options[:duration] &= other_options[:duration]
-        @options[:strict] |= other_options[:strict]
+        @options[:strict] = other_options[:strict].merge!(@options[:strict])
         @options[:dry_run] |= other_options[:dry_run]
 
         @profiles += other_options.profiles
@@ -545,7 +560,7 @@ def to_code_keywords_string(list)
 
       def default_options
         {
-          :strict       => false,
+          :strict       => Cucumber::Core::Test::Result::StrictConfiguration.new,
           :require      => [],
           :dry_run      => false,
           :formats      => [],

lib/cucumber/configuration.rb

@@ -3,6 +3,7 @@
 require 'cucumber/cli/rerun_file'
 require 'cucumber/events'
 require 'cucumber/core/event_bus'
+require 'cucumber/core/test/result'
 require 'forwardable'
 require 'cucumber'
 
@@ -71,7 +72,7 @@ def guess?
       @options[:guess]
     end
 
-    def strict?
+    def strict
       @options[:strict]
     end
 
@@ -243,7 +244,7 @@ def default_options
       {
         :autoload_code_paths => ['features/support', 'features/step_definitions'],
         :filters             => [],
-        :strict              => false,
+        :strict              => Cucumber::Core::Test::Result::StrictConfiguration.new,
         :require             => [],
         :dry_run             => false,
         :fail_fast           => false,

lib/cucumber/formatter/console_issues.rb

@@ -12,9 +12,9 @@ def initialize(config)
         @config.on_event(:test_case_finished) do |event|
           if event.test_case != @previous_test_case
             @previous_test_case = event.test_case
-            @issues[event.result.to_sym] << event.test_case unless event.result.ok?(@config.strict?)
+            @issues[event.result.to_sym] << event.test_case unless event.result.ok?(@config.strict)
           elsif event.result.passed?
-            @issues[:flaky] << event.test_case unless Core::Test::Result::Flaky.ok?(@config.strict?)
+            @issues[:flaky] << event.test_case unless Core::Test::Result::Flaky.ok?(@config.strict)
             @issues[:failed].delete(event.test_case)
           end
         end

lib/cucumber/formatter/fail_fast.rb

@@ -10,7 +10,7 @@ class FailFast
       def initialize(configuration)
         configuration.on_event :test_case_finished do |event|
           _test_case, result = *event.attributes
-          Cucumber.wants_to_quit = true unless result.ok?(configuration.strict?)
+          Cucumber.wants_to_quit = true unless result.ok?(configuration.strict)
         end
       end
 

lib/cucumber/formatter/junit.rb

@@ -52,7 +52,7 @@ def on_test_step_finished(event)
         test_step, result = *event.attributes
         return if @failing_step_source
 
-        @failing_step_source = test_step.source.last unless result.ok?(@config.strict?)
+        @failing_step_source = test_step.source.last unless result.ok?(@config.strict)
       end
 
       def on_test_case_finished(event)
@@ -102,7 +102,7 @@ def end_feature(feature_data)
 
       def create_output_string(test_case, scenario, result, row_name)
         output = "#{test_case.keyword}: #{scenario}\n\n"
-        return output if result.ok?(@config.strict?)
+        return output if result.ok?(@config.strict)
         if test_case.keyword == 'Scenario'
           output += @failing_step_source.keyword.to_s unless hook?(@failing_step_source)
           output += "#{@failing_step_source.name}\n"
@@ -123,7 +123,7 @@ def build_testcase(result, scenario_designation, output)
         name = scenario_designation
 
         @current_feature_data[:builder].testcase(:classname => classname, :name => name, :time => format('%.6f', duration)) do
-          if !result.passed? && result.ok?(@config.strict?)
+          if !result.passed? && result.ok?(@config.strict)
             @current_feature_data[:builder].skipped
             @current_feature_data[:skipped] += 1
           elsif !result.passed?

lib/cucumber/formatter/legacy_api/adapter.rb

@@ -989,7 +989,7 @@ def describe_exception_to(formatter)
 
           def step_exception(step, configuration)
             return filtered_step_exception(step) if @exception
-            return nil unless @status == :undefined && configuration.strict?
+            return nil unless @status == :undefined && configuration.strict.strict?(:undefined)
             @exception = Cucumber::Undefined.from(@result, step.name)
             @exception.backtrace << step.backtrace_line
             filtered_step_exception(step)

lib/cucumber/formatter/rerun.rb

@@ -12,7 +12,7 @@ def initialize(config)
         @failures = {}
         config.on_event :test_case_finished do |event|
           test_case, result = *event.attributes
-          next if result.ok?(@config.strict?)
+          next if result.ok?(@config.strict)
           @failures[test_case.location.file] ||= []
           @failures[test_case.location.file] << test_case.location.line
         end

lib/cucumber/runtime.rb

@@ -229,7 +229,7 @@ def failure?
       if @configuration.wip?
         summary_report.test_cases.total_passed > 0
       else
-        !summary_report.ok?(@configuration.strict?)
+        !summary_report.ok?(@configuration.strict)
       end
     end
     public :failure?

spec/cucumber/cli/configuration_spec.rb

@@ -71,7 +71,19 @@ def reset_config
 
         config.parse!(%w{--profile bongo})
 
-        expect(config.options[:strict]).to be true
+        expect(config.options[:strict].strict?(:undefined)).to be true
+        expect(config.options[:strict].strict?(:pending)).to be true
+        expect(config.options[:strict].strict?(:flaky)).to be true
+      end
+
+      it 'allows --strict from a profile to be selectively overridden' do
+        given_cucumber_yml_defined_as({'bongo' => '--strict'})
+
+        config.parse!(%w{--profile bongo --no-strict-flaky})
+
+        expect(config.options[:strict].strict?(:undefined)).to be true
+        expect(config.options[:strict].strict?(:pending)).to be true
+        expect(config.options[:strict].strict?(:flaky)).to be false
       end
 
       it 'parses ERB syntax in the cucumber.yml file' do

spec/cucumber/formatter/fail_fast_spec.rb

@@ -75,7 +75,7 @@ module Cucumber::Formatter
       end
 
       context 'in strict mode' do
-        let(:configuration) { Cucumber::Configuration.new strict: true }
+        let(:configuration) { Cucumber::Configuration.new strict: Cucumber::Core::Test::Result::StrictConfiguration.new([:undefined]) }
 
         it 'sets Cucumber.wants_to_quit' do
           execute [@gherkin], [StandardStepActions.new], configuration.event_bus

spec/cucumber/formatter/legacy_api/adapter_spec.rb

@@ -2126,7 +2126,7 @@ def find_around_hooks(test_case)
       end
 
       context 'in strict mode' do
-        let(:runtime) { Runtime.new strict: true }
+        let(:runtime) { Runtime.new strict: Cucumber::Core::Test::Result::StrictConfiguration.new([:undefined]) }
 
         it 'passes an exception to the formatter for undefined steps' do
           expect( formatter ).to receive(:exception) do |exception|