From d47f61527fe8af21d06b721f1d0dde8a9ce46258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rasmusson?= Date: Sun, 31 May 2015 16:21:22 +0200 Subject: [PATCH 1/2] Make the Rerun formatter consistent with the exit code The exit code is zero => the Rerun formatter output is empty The exit code is non-zero => the Rerun formatter output is not empty --- .../docs/formatters/rerun_formatter.feature | 68 ++++++++++++++++--- lib/cucumber/formatter/rerun.rb | 3 +- spec/cucumber/formatter/rerun_spec.rb | 6 +- 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/features/docs/formatters/rerun_formatter.feature b/features/docs/formatters/rerun_formatter.feature index 4970a42c4d..3f3c4faca1 100644 --- a/features/docs/formatters/rerun_formatter.feature +++ b/features/docs/formatters/rerun_formatter.feature @@ -2,7 +2,7 @@ Feature: Rerun formatter The rerun formatter writes an output that's perfect for passing to Cucumber when you want to rerun only the - scenarios that have failed. + scenarios that prevented the exit code to be zero. You can save off the rerun output to a file by using it like this: @@ -18,7 +18,28 @@ Feature: Rerun formatter Background: Given the standard step definitions - Scenario: Regular scenarios + Scenario: Exit code is zero + Given a file named "features/mixed.feature" with: + """ + Feature: Mixed + + Scenario: + Given this step is undefined + + Scenario: + Given this step is pending + + Scenario: + Given this step passes + + """ + + When I run `cucumber -f rerun` + Then it should pass with exactly: + """ + """ + + Scenario: Exit code is zero in the dry-run mode Given a file named "features/mixed.feature" with: """ Feature: Mixed @@ -44,13 +65,44 @@ Feature: Rerun formatter Given this step passes """ - When I run `cucumber -f rerun` - Then it should fail with: + When I run `cucumber -f rerun --dry-run` + Then it should pass with exactly: + """ + """ + + Scenario: Exit code is not zero, regular scenario + Given a file named "features/mixed.feature" with: + """ + Feature: Mixed + + Scenario: + Given this step fails + + Scenario: + Given this step is undefined + + Scenario: + Given this step is pending + + Scenario: + Given this step passes + + """ + And a file named "features/all_good.feature" with: + """ + Feature: All good + + Scenario: + Given this step passes + """ + + When I run `cucumber -f rerun --strict` + Then it should fail with exactly: """ features/mixed.feature:3:6:9 """ - Scenario: Scenario outlines + Scenario: Exit code is not zero, scenario outlines For details see https://github.com/cucumber/cucumber/issues/57 Given a file named "features/one_passing_one_failing.feature" with: """ @@ -71,7 +123,7 @@ Feature: Rerun formatter features/one_passing_one_failing.feature:9 """ - Scenario: Failing background + Scenario: Exit code is not zero, failing background Given a file named "features/failing_background.feature" with: """ Feature: Failing background sample @@ -91,7 +143,7 @@ Feature: Rerun formatter features/failing_background.feature:6:9 """ - Scenario: Failing background with scenario outline + Scenario: Exit code is not zero, failing background with scenario outline Given a file named "features/failing_background_outline.feature" with: """ Feature: Failing background sample with scenario outline @@ -113,7 +165,7 @@ Feature: Rerun formatter features/failing_background_outline.feature:11:12 """ - Scenario: Scenario outlines with expand + Scenario: Exit code is not zero, scenario outlines with expand For details see https://github.com/cucumber/cucumber/issues/503 Given a file named "features/one_passing_one_failing.feature" with: diff --git a/lib/cucumber/formatter/rerun.rb b/lib/cucumber/formatter/rerun.rb index 933e0824e9..93c76defc9 100644 --- a/lib/cucumber/formatter/rerun.rb +++ b/lib/cucumber/formatter/rerun.rb @@ -8,10 +8,11 @@ class Rerun def initialize(runtime, path_or_io, options) @io = ensure_io(path_or_io, "rerun") @failures = {} + @options = options end def after_test_case(test_case, result) - return if result.passed? + return if result.passed? || result.skipped? || !@options[:strict] && (result.pending? || result.undefined?) @failures[test_case.location.file] ||= [] @failures[test_case.location.file] << test_case.location.line end diff --git a/spec/cucumber/formatter/rerun_spec.rb b/spec/cucumber/formatter/rerun_spec.rb index e1a250f3ac..c3b85488cf 100644 --- a/spec/cucumber/formatter/rerun_spec.rb +++ b/spec/cucumber/formatter/rerun_spec.rb @@ -44,7 +44,7 @@ def test_case(test_case) end end io = StringIO.new - report = Rerun.new(double, io, double) + report = Rerun.new(double, io, {}) execute [gherkin], report, [WithSteps.new] @@ -79,7 +79,7 @@ def test_case(test_case) end io = StringIO.new - report = Rerun.new(double, io, double) + report = Rerun.new(double, io, {}) execute [foo, bar], report, [WithSteps.new] @@ -98,7 +98,7 @@ def test_case(test_case) end io = StringIO.new - report = Rerun.new(double, io, double) + report = Rerun.new(double, io, {}) execute [gherkin], report, [WithSteps.new] end From fc356b18fa928276f6582335669ba8931216be85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rasmusson?= Date: Sun, 31 May 2015 16:34:59 +0200 Subject: [PATCH 2/2] Use Result#ok? in the Rerun Formatter --- lib/cucumber/formatter/rerun.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cucumber/formatter/rerun.rb b/lib/cucumber/formatter/rerun.rb index 93c76defc9..c05cb26565 100644 --- a/lib/cucumber/formatter/rerun.rb +++ b/lib/cucumber/formatter/rerun.rb @@ -12,7 +12,7 @@ def initialize(runtime, path_or_io, options) end def after_test_case(test_case, result) - return if result.passed? || result.skipped? || !@options[:strict] && (result.pending? || result.undefined?) + return if result.ok?(@options[:strict]) @failures[test_case.location.file] ||= [] @failures[test_case.location.file] << test_case.location.line end