diff --git a/.travis.yml b/.travis.yml index 4ac1063150..97f9e4766d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,14 +12,11 @@ rvm: branches: only: - master + - resolve-issue-882 - v1.3.x-bugfix before_install: - gem update bundler notifications: - email: - - cukes-devs@googlegroups.com - webhooks: - urls: # gitter - - https://webhooks.gitter.im/e/dc010332f9d40fcc21c4 + email: false diff --git a/features/docs/cli/retry_failing_tests.feature b/features/docs/cli/retry_failing_tests.feature new file mode 100644 index 0000000000..1df3a1ece4 --- /dev/null +++ b/features/docs/cli/retry_failing_tests.feature @@ -0,0 +1,32 @@ +@wip +Feature: Retry failing tests + + Retry gives you a way to get through flaky tests that usually pass after a few runs. + This gives a development team a way forward other than disabling a valuable test. + + - Specify max retry count in option + - Output information to the screen + - Output retry information in test report + + Questions: + use a tag for flaky tests? Global option to retry any test that fails? + + Background: + Given a scenario "Flakey" that fails once, then passes + And a scenario "Shakey" that fails twice, then passes + And a scenario "Solid" that passes + And a scenario "No Dice" that fails + + Scenario: + When I run `cucumber -q --retry 1` + Then it should fail with: + """ + 4 scenarios (2 passed, 2 failed) + """ + + Scenario: + When I run `cucumber -q --retry 2` + Then it should pass with: + """ + 4 scenarios (3 passed, 1 failed) + """ \ No newline at end of file diff --git a/features/lib/step_definitions/cucumber_steps.rb b/features/lib/step_definitions/cucumber_steps.rb index 3a70db6450..3cec638e61 100644 --- a/features/lib/step_definitions/cucumber_steps.rb +++ b/features/lib/step_definitions/cucumber_steps.rb @@ -32,6 +32,34 @@ create_step_definition { string } end +Given /^a scenario "([^\"]*)" that passes$/ do |name| + write_file "features/#{name}.feature", + <<-FEATURE + Feature: #{name} + Scenario: #{name} + Given it passes + FEATURE + + write_file "features/step_definitions/#{name}_steps.rb", + <<-STEPS + Given(/^it passes$/) { expect(true).to be true } + STEPS +end + +Given /^a scenario "([^\"]*)" that fails$/ do |name| + write_file "features/#{name}.feature", + <<-FEATURE + Feature: #{name} + Scenario: #{name} + Given it fails + FEATURE + + write_file "features/step_definitions/#{name}_steps.rb", + <<-STEPS + Given(/^it fails$/) { expect(false).to be true } + STEPS +end + When /^I run the feature with the (\w+) formatter$/ do |formatter| expect(features.length).to eq 1 run_feature features.first, formatter diff --git a/features/lib/step_definitions/retry_steps.rb b/features/lib/step_definitions/retry_steps.rb new file mode 100644 index 0000000000..88d0abbe68 --- /dev/null +++ b/features/lib/step_definitions/retry_steps.rb @@ -0,0 +1,35 @@ +Given /^a scenario "([^\"]*)" that fails once, then passes$/ do |name| + write_file "features/#{name}.feature", + <<-FEATURE + Feature: #{name} + Scenario: #{name} + Given it fails once, then passes + FEATURE + + write_file "features/step_defnitions/#{name}_steps.rb", + <<-STEPS + Given(/^it fails once, then passes$/) do + $#{name.downcase} ||= 0 + $#{name.downcase} += 1 + expect($#{name.downcase}).to eql 2 + end + STEPS +end + +Given /^a scenario "([^\"]*)" that fails twice, then passes$/ do |name| + write_file "features/#{name}.feature", + <<-FEATURE + Feature: #{name} + Scenario: #{name} + Given it fails twice, then passes + FEATURE + + write_file "features/step_definitions/#{name}_steps.rb", + <<-STEPS + Given(/^it fails twice, then passes$/) do + $#{name.downcase} ||= 0 + $#{name.downcase} += 1 + expect($#{name.downcase}).to eql 3 + end + STEPS +end diff --git a/lib/cucumber/cli/configuration.rb b/lib/cucumber/cli/configuration.rb index dff04cc9df..cbd793529c 100644 --- a/lib/cucumber/cli/configuration.rb +++ b/lib/cucumber/cli/configuration.rb @@ -67,6 +67,10 @@ def fail_fast? !!@options[:fail_fast] end + def retry_attempts + @options[:retry] + end + def snippet_type @options[:snippet_type] || :regexp end diff --git a/lib/cucumber/cli/options.rb b/lib/cucumber/cli/options.rb index 18e8956934..79ddd1f870 100644 --- a/lib/cucumber/cli/options.rb +++ b/lib/cucumber/cli/options.rb @@ -44,9 +44,10 @@ class Options PROFILE_LONG_FLAG = '--profile' NO_PROFILE_LONG_FLAG = '--no-profile' FAIL_FAST_FLAG = '--fail-fast' + RETRY_FLAG = '--retry' OPTIONS_WITH_ARGS = ['-r', '--require', '--i18n', '-f', '--format', '-o', '--out', '-t', '--tags', '-n', '--name', '-e', '--exclude', - PROFILE_SHORT_FLAG, PROFILE_LONG_FLAG, + PROFILE_SHORT_FLAG, PROFILE_LONG_FLAG, RETRY_FLAG, '-l', '--lines', '--port', '-I', '--snippet-type'] ORDER_TYPES = %w{defined random} @@ -188,6 +189,9 @@ def parse!(args) "Disables all profile loading to avoid using the 'default' profile.") do |v| @disable_profile_loading = true end + opts.on("#{RETRY_FLAG} ATTEMPTS", "Specify the number of times to retry failing tests (default: 0)") do |v| + @options[:retry] = v.to_i + end opts.on("-c", "--[no-]color", "Whether or not to use ANSI color in the output. Cucumber decides", "based on your platform and the output destination if not specified.") do |v| @@ -450,7 +454,8 @@ def default_options :diff_enabled => true, :snippets => true, :source => true, - :duration => true + :duration => true, + :retry => 0 } end end diff --git a/lib/cucumber/configuration.rb b/lib/cucumber/configuration.rb index 1dd28086de..5ffdd5e556 100644 --- a/lib/cucumber/configuration.rb +++ b/lib/cucumber/configuration.rb @@ -65,6 +65,10 @@ def fail_fast? @options[:fail_fast] end + def retry_attempts + @options[:retry] + end + def guess? @options[:guess] end diff --git a/lib/cucumber/filters/retry.rb b/lib/cucumber/filters/retry.rb new file mode 100644 index 0000000000..3017bd7727 --- /dev/null +++ b/lib/cucumber/filters/retry.rb @@ -0,0 +1,32 @@ +require 'cucumber/core/filter' +require 'cucumber/running_test_case' +require 'cucumber/events/bus' +require 'cucumber/events/after_test_case' + +module Cucumber + module Filters + class Retry < Core::Filter.new(:configuration) + + def test_case(test_case) + configuration.on_event(:after_test_case) do |event| + next unless retry_required?(test_case, event) + + test_case_counts[test_case] += 1 + event.test_case.describe_to(receiver) + end + + super + end + + private + + def retry_required?(test_case, event) + event.test_case == test_case && event.result.failed? && test_case_counts[test_case] < configuration.retry_attempts + end + + def test_case_counts + @test_case_counts ||= Hash.new {|h,k| h[k] = 0 } + end + end + end +end diff --git a/spec/cucumber/cli/configuration_spec.rb b/spec/cucumber/cli/configuration_spec.rb index 3b072f1c1e..bef7954ad9 100644 --- a/spec/cucumber/cli/configuration_spec.rb +++ b/spec/cucumber/cli/configuration_spec.rb @@ -423,6 +423,13 @@ def reset_config expect(config.snippet_type).to eq :regexp end end + + describe "#retry_attempts" do + it "returns the specified number of retries" do + config.parse!(['--retry=3']) + expect(config.retry_attempts).to eql 3 + end + end end end end diff --git a/spec/cucumber/cli/options_spec.rb b/spec/cucumber/cli/options_spec.rb index 9b136170ec..65fbbbe195 100644 --- a/spec/cucumber/cli/options_spec.rb +++ b/spec/cucumber/cli/options_spec.rb @@ -348,6 +348,20 @@ def after_parsing(args) end end + context '--retry ATTEMPTS' do + it 'is 0 by default' do + after_parsing("") do + expect(options[:retry]).to eql 0 + end + end + + it 'sets the options[:retry] value' do + after_parsing("--retry 4") do + expect(options[:retry]).to eql 4 + end + end + end + it "assigns any extra arguments as paths to features" do after_parsing('-f pretty my_feature.feature my_other_features') do expect(options[:paths]).to eq ['my_feature.feature', 'my_other_features'] diff --git a/spec/cucumber/filters/retry_spec.rb b/spec/cucumber/filters/retry_spec.rb new file mode 100644 index 0000000000..f9e94823a1 --- /dev/null +++ b/spec/cucumber/filters/retry_spec.rb @@ -0,0 +1,79 @@ +require 'cucumber' +require 'cucumber/filters/retry' +require 'cucumber/core/gherkin/writer' +require 'cucumber/configuration' +require 'cucumber/core/test/case' +require 'cucumber/core' +require 'cucumber/events' + +describe Cucumber::Filters::Retry do + include Cucumber::Core::Gherkin::Writer + include Cucumber::Core + include Cucumber::Events + + let(:configuration) { Cucumber::Configuration.new(:retry => 2) } + let(:test_case) { Cucumber::Core::Test::Case.new([double('test steps')], double('source').as_null_object) } + let(:receiver) { double('receiver').as_null_object } + let(:filter) { Cucumber::Filters::Retry.new(configuration, receiver) } + let(:fail) { Cucumber::Events::AfterTestCase.new(test_case, double('result', :failed? => true, :ok? => false)) } + let(:pass) { Cucumber::Events::AfterTestCase.new(test_case, double('result', :failed? => false, :ok? => true)) } + + it { is_expected.to respond_to(:test_case) } + it { is_expected.to respond_to(:with_receiver) } + it { is_expected.to respond_to(:done) } + + context "general" do + before(:each) do + filter.with_receiver(receiver) + end + + it "registers the :after_test_case event" do + expect(configuration).to receive(:on_event).with(:after_test_case) + filter.test_case(test_case) + end + end + + context "passing test case" do + it "describes the test case once" do + expect(test_case).to receive(:describe_to).with(receiver) + filter.test_case(test_case) + configuration.notify(pass) + end + end + + context "failing test case" do + it "describes the test case the specified number of times" do + expect(receiver).to receive(:test_case) {|test_case| + configuration.notify(fail) + }.exactly(3).times + + filter.test_case(test_case) + end + end + + context "flaky test cases" do + + context "a little flaky" do + it "describes the test case twice" do + results = [fail, pass] + expect(receiver).to receive(:test_case) {|test_case| + configuration.notify(results.shift) + }.exactly(2).times + + filter.test_case(test_case) + end + end + + context "really flaky" do + it "describes the test case 3 times" do + results = [fail, fail, pass] + + expect(receiver).to receive(:test_case) {|test_case| + configuration.notify(results.shift) + }.exactly(3).times + + filter.test_case(test_case) + end + end + end +end