Add --retry option to retry failed tests as part of the same run (#920)

* Add feature file from PR 895 to start work on retry functionality

* Initial spec for --retry option

* Add another test for --retry flag, make new tests pass

* Squash some commits

Redo the commit I just reverted but with the file I intended to include the first time

* Rebase onto master

* Allow for --retry option in configuration

Add retry_attempts method to configuration base class

Add Retry formatter and specs for same

Remove files attempting to implement --retry with a formatter - bad idea

Add wip tag to prevent --retry features from killing the build

Actually run Travis build against active branch

Fix tiny grammatical error. Closes #914.

* Update scenarios to look like @mattwynne's suggestion

* Initial specs for retry filter - they test if it works at all, basically

Change Retry filter to have attribute :configuration and first pass at setting up a listener

Reorganize spec file

Finish adding tests for re-running flaky test cases - all but one pass

Fix spec that said to test what happens when a test case passes but actually didn't make sure it did pass

* Add consistently failing scenario to test cases

Define steps for the retry feature

* Correct # of retries

* Refactor test case

* Make method private

* Extract retry_required? method

* Move steps for retry filter into other file

* Correct block style

.travis.yml

@@ -12,14 +12,11 @@ rvm:
 branches:
   only:
     - master
+    - resolve-issue-882
     - v1.3.x-bugfix
 
 before_install:
   - gem update bundler
 
 notifications:
-  email:
-    - [email protected]
-  webhooks:
-    urls: # gitter
-      - https://webhooks.gitter.im/e/dc010332f9d40fcc21c4
+  email: false

features/docs/cli/retry_failing_tests.feature

@@ -0,0 +1,32 @@
+@wip
+Feature: Retry failing tests
+
+  Retry gives you a way to get through flaky tests that usually pass after a few runs.
+  This gives a development team a way forward other than disabling a valuable test.
+
+  - Specify max retry count in option
+  - Output information to the screen
+  - Output retry information in test report
+
+  Questions:
+  use a tag for flaky tests?  Global option to retry any test that fails?
+
+  Background:
+    Given a scenario "Flakey" that fails once, then passes
+    And a scenario "Shakey" that fails twice, then passes
+    And a scenario "Solid" that passes
+    And a scenario "No Dice" that fails
+
+  Scenario:
+    When I run `cucumber -q --retry 1`
+    Then it should fail with:
+      """
+      4 scenarios (2 passed, 2 failed)
+      """
+
+  Scenario:
+    When I run `cucumber -q --retry 2`
+    Then it should pass with:
+      """
+      4 scenarios (3 passed, 1 failed)
+      """

features/lib/step_definitions/cucumber_steps.rb

@@ -32,6 +32,34 @@
   create_step_definition { string }
 end
 
+Given /^a scenario "([^\"]*)" that passes$/ do |name|
+  write_file "features/#{name}.feature",
+  <<-FEATURE
+  Feature: #{name}
+    Scenario: #{name}
+      Given it passes
+  FEATURE
+
+  write_file "features/step_definitions/#{name}_steps.rb",
+  <<-STEPS
+  Given(/^it passes$/) { expect(true).to be true }
+  STEPS
+end
+
+Given /^a scenario "([^\"]*)" that fails$/ do |name|
+  write_file "features/#{name}.feature",
+  <<-FEATURE
+  Feature: #{name}
+    Scenario: #{name}
+      Given it fails
+  FEATURE
+
+  write_file "features/step_definitions/#{name}_steps.rb",
+  <<-STEPS
+  Given(/^it fails$/) { expect(false).to be true }
+  STEPS
+end
+
 When /^I run the feature with the (\w+) formatter$/ do |formatter|
   expect(features.length).to eq 1
   run_feature features.first, formatter

features/lib/step_definitions/retry_steps.rb

@@ -0,0 +1,35 @@
+Given /^a scenario "([^\"]*)" that fails once, then passes$/ do |name|
+  write_file "features/#{name}.feature",
+  <<-FEATURE
+  Feature: #{name}
+    Scenario: #{name}
+      Given it fails once, then passes
+  FEATURE
+
+  write_file "features/step_defnitions/#{name}_steps.rb",
+  <<-STEPS
+  Given(/^it fails once, then passes$/) do
+    $#{name.downcase} ||= 0
+    $#{name.downcase} += 1
+    expect($#{name.downcase}).to eql 2
+  end
+  STEPS
+end
+
+Given /^a scenario "([^\"]*)" that fails twice, then passes$/ do |name|
+  write_file "features/#{name}.feature",
+  <<-FEATURE
+  Feature: #{name}
+    Scenario: #{name}
+      Given it fails twice, then passes
+  FEATURE
+
+  write_file "features/step_definitions/#{name}_steps.rb",
+  <<-STEPS
+  Given(/^it fails twice, then passes$/) do
+    $#{name.downcase} ||= 0
+    $#{name.downcase} += 1
+    expect($#{name.downcase}).to eql 3
+  end
+  STEPS
+end

lib/cucumber/cli/configuration.rb

@@ -67,6 +67,10 @@ def fail_fast?
         !!@options[:fail_fast]
       end
 
+      def retry_attempts
+        @options[:retry]
+      end
+
       def snippet_type
         @options[:snippet_type] || :regexp
       end

lib/cucumber/cli/options.rb

@@ -44,9 +44,10 @@ class Options
       PROFILE_LONG_FLAG = '--profile'
       NO_PROFILE_LONG_FLAG = '--no-profile'
       FAIL_FAST_FLAG = '--fail-fast'
+      RETRY_FLAG = '--retry'
       OPTIONS_WITH_ARGS = ['-r', '--require', '--i18n', '-f', '--format', '-o', '--out',
                                   '-t', '--tags', '-n', '--name', '-e', '--exclude',
-                                  PROFILE_SHORT_FLAG, PROFILE_LONG_FLAG,
+                                  PROFILE_SHORT_FLAG, PROFILE_LONG_FLAG, RETRY_FLAG,
                                   '-l', '--lines', '--port',
                                   '-I', '--snippet-type']
       ORDER_TYPES = %w{defined random}
@@ -188,6 +189,9 @@ def parse!(args)
             "Disables all profile loading to avoid using the 'default' profile.") do |v|
             @disable_profile_loading = true
           end
+          opts.on("#{RETRY_FLAG} ATTEMPTS", "Specify the number of times to retry failing tests (default: 0)") do |v|
+            @options[:retry] = v.to_i
+          end
           opts.on("-c", "--[no-]color",
             "Whether or not to use ANSI color in the output. Cucumber decides",
             "based on your platform and the output destination if not specified.") do |v|
@@ -450,7 +454,8 @@ def default_options
           :diff_enabled => true,
           :snippets     => true,
           :source       => true,
-          :duration     => true
+          :duration     => true,
+          :retry        => 0
         }
       end
     end

lib/cucumber/configuration.rb

@@ -65,6 +65,10 @@ def fail_fast?
       @options[:fail_fast]
     end
 
+    def retry_attempts
+      @options[:retry]
+    end
+
     def guess?
       @options[:guess]
     end

lib/cucumber/filters/retry.rb

@@ -0,0 +1,32 @@
+require 'cucumber/core/filter'
+require 'cucumber/running_test_case'
+require 'cucumber/events/bus'
+require 'cucumber/events/after_test_case'
+
+module Cucumber
+  module Filters
+    class Retry < Core::Filter.new(:configuration)
+
+      def test_case(test_case)
+        configuration.on_event(:after_test_case) do |event|
+          next unless retry_required?(test_case, event)
+
+          test_case_counts[test_case] += 1
+          event.test_case.describe_to(receiver)
+        end
+
+        super
+      end
+
+      private
+
+      def retry_required?(test_case, event)
+        event.test_case == test_case && event.result.failed? && test_case_counts[test_case] < configuration.retry_attempts
+      end
+
+      def test_case_counts
+        @test_case_counts ||= Hash.new {|h,k| h[k] = 0 }
+      end
+    end
+  end
+end

spec/cucumber/cli/configuration_spec.rb

@@ -423,6 +423,13 @@ def reset_config
         expect(config.snippet_type).to eq :regexp
       end
     end
+
+    describe "#retry_attempts" do 
+      it "returns the specified number of retries" do 
+        config.parse!(['--retry=3'])
+        expect(config.retry_attempts).to eql 3
+      end
+    end
   end
 end
 end

spec/cucumber/cli/options_spec.rb

@@ -348,6 +348,20 @@ def after_parsing(args)
           end
         end
 
+        context '--retry ATTEMPTS' do
+          it 'is 0 by default' do 
+            after_parsing("") do 
+              expect(options[:retry]).to eql 0
+            end
+          end
+
+          it 'sets the options[:retry] value' do 
+            after_parsing("--retry 4") do 
+              expect(options[:retry]).to eql 4
+            end
+          end
+        end
+
         it "assigns any extra arguments as paths to features" do
           after_parsing('-f pretty my_feature.feature my_other_features') do
             expect(options[:paths]).to eq ['my_feature.feature', 'my_other_features']

spec/cucumber/filters/retry_spec.rb

@@ -0,0 +1,79 @@
+require 'cucumber'
+require 'cucumber/filters/retry'
+require 'cucumber/core/gherkin/writer'
+require 'cucumber/configuration'
+require 'cucumber/core/test/case'
+require 'cucumber/core'
+require 'cucumber/events'
+
+describe Cucumber::Filters::Retry do
+  include Cucumber::Core::Gherkin::Writer
+  include Cucumber::Core
+  include Cucumber::Events
+
+  let(:configuration) { Cucumber::Configuration.new(:retry => 2) }
+  let(:test_case) { Cucumber::Core::Test::Case.new([double('test steps')], double('source').as_null_object) }
+  let(:receiver) { double('receiver').as_null_object }
+  let(:filter) { Cucumber::Filters::Retry.new(configuration, receiver) }
+  let(:fail) { Cucumber::Events::AfterTestCase.new(test_case, double('result', :failed? => true, :ok? => false)) }
+  let(:pass) { Cucumber::Events::AfterTestCase.new(test_case, double('result', :failed? => false, :ok? => true)) }
+
+  it { is_expected.to respond_to(:test_case) }
+  it { is_expected.to respond_to(:with_receiver) }
+  it { is_expected.to respond_to(:done) }
+
+  context "general" do
+    before(:each) do
+      filter.with_receiver(receiver)
+    end
+
+    it "registers the :after_test_case event" do
+      expect(configuration).to receive(:on_event).with(:after_test_case)
+      filter.test_case(test_case)
+    end
+  end
+
+  context "passing test case" do
+    it "describes the test case once" do
+      expect(test_case).to receive(:describe_to).with(receiver)
+      filter.test_case(test_case)
+      configuration.notify(pass)
+    end
+  end
+
+  context "failing test case" do
+    it "describes the test case the specified number of times" do
+      expect(receiver).to receive(:test_case) {|test_case|
+        configuration.notify(fail)
+      }.exactly(3).times
+
+      filter.test_case(test_case)
+    end
+  end
+
+  context "flaky test cases" do
+
+    context "a little flaky" do
+      it "describes the test case twice" do
+        results = [fail, pass]
+        expect(receiver).to receive(:test_case) {|test_case|
+          configuration.notify(results.shift)
+        }.exactly(2).times
+
+        filter.test_case(test_case)
+      end
+    end
+
+    context "really flaky" do
+      it "describes the test case 3 times" do
+        results = [fail, fail, pass]
+
+        expect(receiver).to receive(:test_case) {|test_case|
+          configuration.notify(results.shift)
+        }.exactly(3).times
+
+        filter.test_case(test_case)
+      end
+    end
+  end
+end