diff --git a/.travis.yml b/.travis.yml index 6d26088eef..97f9e4766d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ rvm: branches: only: - master + - resolve-issue-882 - v1.3.x-bugfix before_install: diff --git a/features/docs/cli/retry_failing_tests.feature b/features/docs/cli/retry_failing_tests.feature index 232fbdb5c6..debc7ef18b 100644 --- a/features/docs/cli/retry_failing_tests.feature +++ b/features/docs/cli/retry_failing_tests.feature @@ -1,6 +1,7 @@ +@wip Feature: Retry failing tests - Retry gives you a way to get through flaky tests usually pass after a few runs. + Retry gives you a way to get through flaky tests that usually pass after a few runs. This gives a development team a way forward other than disabling a valuable test. - Specify max retry count in option @@ -11,10 +12,17 @@ Feature: Retry failing tests use a tag for flaky tests? Global option to retry any test that fails? Background: - Given a flaky step definition + Given a file named "features/step_definitions/flaky_steps.rb" with: + """ + Given(/^a flaky step$/) do + $answer ||= 0 + $answer += 1 + expect(@answer).to eq 2 + end + """ - Scenario: run a flaky test - Given a file named "features/fail.feature" with: + Scenario: Run a flaky test + Given a file named "features/flaky.feature" with: """ Feature: Fails @@ -26,47 +34,83 @@ Feature: Retry failing tests Then it should fail with: """ Feature: Fails - + Scenario: flaky test Given a flaky step - (RuntimeError) - ./features/step_definitions/steps.rb:4:in `flaky_pass' - ./features/step_definitions/steps.rb:1:in `/^a flaky step$/' - features/fail.feature:4:in `Given a flaky step' - + + expected: 1 + got: 0 + + (compared using ==) + (RSpec::Expectations::ExpectationNotMetError) + ./features/step_definitions/flaky_steps.rb:3:in `/^a flaky step$/' + features/flaky.feature:4:in `Given a flaky step' + Failing Scenarios: cucumber features/fail.feature:3 - + 1 scenario (1 failed) 1 step (1 failed) - """ - @wip - Scenario: rerun a flaky test - Given a file named "features/fail.feature" with: + + Scenario: Retry a flaky test + Given a file named "features/flaky.feature" with: """ - Feature: Fails + Feature: Passes Scenario: flaky test Given a flaky step """ - When I run `cucumber features --rerun_flakes=1 -q` + When I run `cucumber features --retry=1 -q` Then it should pass with: """ - Feature: Fails + Feature: Passes Scenario: flaky test Given a flaky step - (RuntimeError) - ./features/step_definitions/steps.rb:4:in `flaky_pass' - ./features/step_definitions/steps.rb:1:in `/^a flaky step$/' - features/fail.feature:4:in `Given a flaky step' - - Failing Scenarios: - cucumber features/fail.feature:3 - 1 scenario (1 passed, 1 flake) + 1 scenario (1 passed) 1 step (1 passed) + """ + + Scenario: One failure, one flake + Given a file named "features/step_definitions/failing_steps.rb" with: + """ + Given(/^a failing step$/) do + expect(true).not_to be true + end + """ + And a file named "features/flaky.feature" with: + """ + Feature: Fails + + Scenario: Flaky + Given a flaky step + + Scenario: Failing + Given a failing step + """ + When I run `cucumber features --retry=1 -q` + Then it should fail with: + """ + Feature: Fails + + Scenario: Flaky + Given a flaky step + (RuntimeError) + ./features/step_definitions/steps.rb:4 in `flaky_pass' + ./features/step_definitions/steps.rb:1 in `/^a flaky step$/' + features/fail.feature:4:in `Given a flaky step` + + Scenario: Failing + Given a failing step + (RuntimeError) + ./features/step_definitions/failing_steps.rb:4 in `expect' + ./features/step_definitions/failing_steps.rb:1 in `/^a failing step$/' + features/fail.feature:7:in `Given a failing step' + + 2 scenarios (1 passed, 1 failed) + 2 steps (1 passed, 1 failed) """ \ No newline at end of file diff --git a/lib/cucumber/cli/configuration.rb b/lib/cucumber/cli/configuration.rb index dff04cc9df..cbd793529c 100644 --- a/lib/cucumber/cli/configuration.rb +++ b/lib/cucumber/cli/configuration.rb @@ -67,6 +67,10 @@ def fail_fast? !!@options[:fail_fast] end + def retry_attempts + @options[:retry] + end + def snippet_type @options[:snippet_type] || :regexp end diff --git a/lib/cucumber/configuration.rb b/lib/cucumber/configuration.rb index 1dd28086de..5ffdd5e556 100644 --- a/lib/cucumber/configuration.rb +++ b/lib/cucumber/configuration.rb @@ -65,6 +65,10 @@ def fail_fast? @options[:fail_fast] end + def retry_attempts + @options[:retry] + end + def guess? @options[:guess] end diff --git a/spec/cucumber/cli/configuration_spec.rb b/spec/cucumber/cli/configuration_spec.rb index 3b072f1c1e..bef7954ad9 100644 --- a/spec/cucumber/cli/configuration_spec.rb +++ b/spec/cucumber/cli/configuration_spec.rb @@ -423,6 +423,13 @@ def reset_config expect(config.snippet_type).to eq :regexp end end + + describe "#retry_attempts" do + it "returns the specified number of retries" do + config.parse!(['--retry=3']) + expect(config.retry_attempts).to eql 3 + end + end end end end