diff --git a/lib/cucumber/cli/configuration.rb b/lib/cucumber/cli/configuration.rb index 7410336f61..9b4bd177c8 100644 --- a/lib/cucumber/cli/configuration.rb +++ b/lib/cucumber/cli/configuration.rb @@ -26,7 +26,7 @@ def parse!(args) @args = args @options.parse!(args) arrange_formats - raise("You can't use both --strict and --wip") if strict? && wip? + raise("You can't use both --strict and --wip") if strict.strict? && wip? set_environment_variables end @@ -42,7 +42,7 @@ def seed Integer(@options[:seed] || rand(0xFFFF)) end - def strict? + def strict @options[:strict] end diff --git a/lib/cucumber/cli/options.rb b/lib/cucumber/cli/options.rb index 40b658ec39..7334179b2a 100644 --- a/lib/cucumber/cli/options.rb +++ b/lib/cucumber/cli/options.rb @@ -3,6 +3,7 @@ require 'cucumber/formatter/ansicolor' require 'cucumber/glue/registry_and_more' require 'cucumber/project_initializer' +require 'cucumber/core/test/result' module Cucumber module Cli @@ -119,7 +120,10 @@ def parse!(args) # rubocop:disable Metrics/AbcSize opts.on('-q', '--quiet', 'Alias for --no-snippets --no-source.') { shut_up } opts.on('--no-duration', "Don't print the duration at the end of the summary") { set_option :duration, false } opts.on('-b', '--backtrace', 'Show full backtrace for all errors.') { Cucumber.use_full_backtrace = true } - opts.on('-S', '--strict', 'Fail if there are any undefined or pending steps.') { set_option :strict } + opts.on('-S', '--[no-]strict', *strict_msg) { |setting| set_strict(setting) } + opts.on('--[no-]strict-undefined', 'Fail if there are any undefined results.') { |setting| set_strict(setting, :undefined) } + opts.on('--[no-]strict-pending', 'Fail if there are any pending results.') { |setting| set_strict(setting, :pending) } + opts.on('--[no-]strict-flaky', 'Fail if there are any flaky results.') { |setting| set_strict(setting, :flaky) } opts.on('-w', '--wip', 'Fail if there are any passing scenarios.') { set_option :wip } opts.on('-v', '--verbose', 'Show the files and features loaded.') { set_option :verbose } opts.on('-g', '--guess', 'Guess best match for Ambiguous steps.') { set_option :guess } @@ -254,6 +258,13 @@ def name_msg ] end + def strict_msg + [ + 'Fail if there are any strict affected results ', + '(that is undefined, pending or flaky results).' + ] + end + def parse_formats(v) formatter, *formatter_options = v.split(',') options_hash = Hash[formatter_options.map { |s| s.split('=') }] @@ -409,6 +420,10 @@ def shut_up @options[:duration] = false end + def set_strict(setting, type = nil) + @options[:strict].set_strict(setting, type) + end + def stdout_formats @options[:formats].select { |_, _, output| output == @out_stream } end @@ -477,7 +492,7 @@ def reverse_merge(other_options) @options[:source] &= other_options[:source] @options[:snippets] &= other_options[:snippets] @options[:duration] &= other_options[:duration] - @options[:strict] |= other_options[:strict] + @options[:strict] = other_options[:strict].merge!(@options[:strict]) @options[:dry_run] |= other_options[:dry_run] @profiles += other_options.profiles @@ -545,7 +560,7 @@ def to_code_keywords_string(list) def default_options { - :strict => false, + :strict => Cucumber::Core::Test::Result::StrictConfiguration.new, :require => [], :dry_run => false, :formats => [], diff --git a/lib/cucumber/configuration.rb b/lib/cucumber/configuration.rb index 7b550531b4..5b9f996794 100644 --- a/lib/cucumber/configuration.rb +++ b/lib/cucumber/configuration.rb @@ -3,6 +3,7 @@ require 'cucumber/cli/rerun_file' require 'cucumber/events' require 'cucumber/core/event_bus' +require 'cucumber/core/test/result' require 'forwardable' require 'cucumber' @@ -71,7 +72,7 @@ def guess? @options[:guess] end - def strict? + def strict @options[:strict] end @@ -243,7 +244,7 @@ def default_options { :autoload_code_paths => ['features/support', 'features/step_definitions'], :filters => [], - :strict => false, + :strict => Cucumber::Core::Test::Result::StrictConfiguration.new, :require => [], :dry_run => false, :fail_fast => false, diff --git a/lib/cucumber/formatter/console_issues.rb b/lib/cucumber/formatter/console_issues.rb index 2d4196d9e1..166b5f4988 100644 --- a/lib/cucumber/formatter/console_issues.rb +++ b/lib/cucumber/formatter/console_issues.rb @@ -12,9 +12,9 @@ def initialize(config) @config.on_event(:test_case_finished) do |event| if event.test_case != @previous_test_case @previous_test_case = event.test_case - @issues[event.result.to_sym] << event.test_case unless event.result.ok?(@config.strict?) + @issues[event.result.to_sym] << event.test_case unless event.result.ok?(@config.strict) elsif event.result.passed? - @issues[:flaky] << event.test_case unless Core::Test::Result::Flaky.ok?(@config.strict?) + @issues[:flaky] << event.test_case unless Core::Test::Result::Flaky.ok?(@config.strict) @issues[:failed].delete(event.test_case) end end diff --git a/lib/cucumber/formatter/fail_fast.rb b/lib/cucumber/formatter/fail_fast.rb index e00038af3e..1b6e8cde3f 100644 --- a/lib/cucumber/formatter/fail_fast.rb +++ b/lib/cucumber/formatter/fail_fast.rb @@ -10,7 +10,7 @@ class FailFast def initialize(configuration) configuration.on_event :test_case_finished do |event| _test_case, result = *event.attributes - Cucumber.wants_to_quit = true unless result.ok?(configuration.strict?) + Cucumber.wants_to_quit = true unless result.ok?(configuration.strict) end end diff --git a/lib/cucumber/formatter/junit.rb b/lib/cucumber/formatter/junit.rb index 7ee4f991a2..812510c335 100644 --- a/lib/cucumber/formatter/junit.rb +++ b/lib/cucumber/formatter/junit.rb @@ -52,7 +52,7 @@ def on_test_step_finished(event) test_step, result = *event.attributes return if @failing_step_source - @failing_step_source = test_step.source.last unless result.ok?(@config.strict?) + @failing_step_source = test_step.source.last unless result.ok?(@config.strict) end def on_test_case_finished(event) @@ -102,7 +102,7 @@ def end_feature(feature_data) def create_output_string(test_case, scenario, result, row_name) output = "#{test_case.keyword}: #{scenario}\n\n" - return output if result.ok?(@config.strict?) + return output if result.ok?(@config.strict) if test_case.keyword == 'Scenario' output += @failing_step_source.keyword.to_s unless hook?(@failing_step_source) output += "#{@failing_step_source.name}\n" @@ -123,7 +123,7 @@ def build_testcase(result, scenario_designation, output) name = scenario_designation @current_feature_data[:builder].testcase(:classname => classname, :name => name, :time => format('%.6f', duration)) do - if !result.passed? && result.ok?(@config.strict?) + if !result.passed? && result.ok?(@config.strict) @current_feature_data[:builder].skipped @current_feature_data[:skipped] += 1 elsif !result.passed? diff --git a/lib/cucumber/formatter/legacy_api/adapter.rb b/lib/cucumber/formatter/legacy_api/adapter.rb index 76b59ea9c6..a63413fc02 100644 --- a/lib/cucumber/formatter/legacy_api/adapter.rb +++ b/lib/cucumber/formatter/legacy_api/adapter.rb @@ -989,7 +989,7 @@ def describe_exception_to(formatter) def step_exception(step, configuration) return filtered_step_exception(step) if @exception - return nil unless @status == :undefined && configuration.strict? + return nil unless @status == :undefined && configuration.strict.strict?(:undefined) @exception = Cucumber::Undefined.from(@result, step.name) @exception.backtrace << step.backtrace_line filtered_step_exception(step) diff --git a/lib/cucumber/formatter/rerun.rb b/lib/cucumber/formatter/rerun.rb index 442ba27b64..501e98b845 100644 --- a/lib/cucumber/formatter/rerun.rb +++ b/lib/cucumber/formatter/rerun.rb @@ -12,7 +12,7 @@ def initialize(config) @failures = {} config.on_event :test_case_finished do |event| test_case, result = *event.attributes - next if result.ok?(@config.strict?) + next if result.ok?(@config.strict) @failures[test_case.location.file] ||= [] @failures[test_case.location.file] << test_case.location.line end diff --git a/lib/cucumber/runtime.rb b/lib/cucumber/runtime.rb index 40c9f268dc..5a0083ace7 100644 --- a/lib/cucumber/runtime.rb +++ b/lib/cucumber/runtime.rb @@ -229,7 +229,7 @@ def failure? if @configuration.wip? summary_report.test_cases.total_passed > 0 else - !summary_report.ok?(@configuration.strict?) + !summary_report.ok?(@configuration.strict) end end public :failure? diff --git a/spec/cucumber/cli/configuration_spec.rb b/spec/cucumber/cli/configuration_spec.rb index 777638a876..eba5daa99d 100644 --- a/spec/cucumber/cli/configuration_spec.rb +++ b/spec/cucumber/cli/configuration_spec.rb @@ -71,7 +71,19 @@ def reset_config config.parse!(%w{--profile bongo}) - expect(config.options[:strict]).to be true + expect(config.options[:strict].strict?(:undefined)).to be true + expect(config.options[:strict].strict?(:pending)).to be true + expect(config.options[:strict].strict?(:flaky)).to be true + end + + it 'allows --strict from a profile to be selectively overridden' do + given_cucumber_yml_defined_as({'bongo' => '--strict'}) + + config.parse!(%w{--profile bongo --no-strict-flaky}) + + expect(config.options[:strict].strict?(:undefined)).to be true + expect(config.options[:strict].strict?(:pending)).to be true + expect(config.options[:strict].strict?(:flaky)).to be false end it 'parses ERB syntax in the cucumber.yml file' do diff --git a/spec/cucumber/formatter/fail_fast_spec.rb b/spec/cucumber/formatter/fail_fast_spec.rb index 48a4bc1d97..487ad0fbbb 100644 --- a/spec/cucumber/formatter/fail_fast_spec.rb +++ b/spec/cucumber/formatter/fail_fast_spec.rb @@ -75,7 +75,7 @@ module Cucumber::Formatter end context 'in strict mode' do - let(:configuration) { Cucumber::Configuration.new strict: true } + let(:configuration) { Cucumber::Configuration.new strict: Cucumber::Core::Test::Result::StrictConfiguration.new([:undefined]) } it 'sets Cucumber.wants_to_quit' do execute [@gherkin], [StandardStepActions.new], configuration.event_bus diff --git a/spec/cucumber/formatter/legacy_api/adapter_spec.rb b/spec/cucumber/formatter/legacy_api/adapter_spec.rb index e3b2824afd..da5543816e 100644 --- a/spec/cucumber/formatter/legacy_api/adapter_spec.rb +++ b/spec/cucumber/formatter/legacy_api/adapter_spec.rb @@ -2126,7 +2126,7 @@ def find_around_hooks(test_case) end context 'in strict mode' do - let(:runtime) { Runtime.new strict: true } + let(:runtime) { Runtime.new strict: Cucumber::Core::Test::Result::StrictConfiguration.new([:undefined]) } it 'passes an exception to the formatter for undefined steps' do expect( formatter ).to receive(:exception) do |exception|