From 230acc31528ee30b37e88fcd3eb7573233b16077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rasmusson?= Date: Sun, 24 May 2015 10:38:59 +0200 Subject: [PATCH 1/4] Add failing test for scenario outlines when not using --expand The current implementation of the JUnit Formatter does not handle several failing scenarios form the same Examples table correctly. Nor does it handle undefined or pending scenarios properly in strict mode. --- features/docs/formatters/junit_formatter.feature | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/features/docs/formatters/junit_formatter.feature b/features/docs/formatters/junit_formatter.feature index 2a440662ff..7da4a4f595 100644 --- a/features/docs/formatters/junit_formatter.feature +++ b/features/docs/formatters/junit_formatter.feature @@ -271,10 +271,14 @@ You *must* specify --out DIR for the junit formatter + ']]> @@ -283,10 +287,13 @@ You *must* specify --out DIR for the junit formatter + ']]> From c1c4196e9bd1c6865b40c0ad2e90e68499aedc02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rasmusson?= Date: Sun, 24 May 2015 10:40:11 +0200 Subject: [PATCH 2/4] Rewrite the JUnit Formatter to the new formatter Api --- .../docs/formatters/junit_formatter.feature | 2 +- lib/cucumber/formatter/junit.rb | 214 +++++++----------- 2 files changed, 88 insertions(+), 128 deletions(-) diff --git a/features/docs/formatters/junit_formatter.feature b/features/docs/formatters/junit_formatter.feature index 7da4a4f595..8daf2a11b5 100644 --- a/features/docs/formatters/junit_formatter.feature +++ b/features/docs/formatters/junit_formatter.feature @@ -201,7 +201,7 @@ Feature: JUnit output formatter Message: ]]> - diff --git a/lib/cucumber/formatter/junit.rb b/lib/cucumber/formatter/junit.rb index 50a0b13500..8c4cc8aed2 100644 --- a/lib/cucumber/formatter/junit.rb +++ b/lib/cucumber/formatter/junit.rb @@ -8,10 +8,6 @@ module Formatter # The formatter used for --format junit class Junit - # TODO: remove coupling to types - AST_SCENARIO_OUTLINE = Core::Ast::ScenarioOutline - AST_EXAMPLE_ROW = LegacyApi::Ast::ExampleTableRow - include Io class UnNamedFeatureError < StandardError @@ -20,15 +16,48 @@ def initialize(feature_file) end end - def initialize(runtime, io, options) + def initialize(_runtime, io, options) @reportdir = ensure_dir(io, "junit") @options = options end - def before_feature(feature) + def before_test_case(test_case) + unless same_feature_as_previous_test_case?(test_case.feature) + end_feature if @current_feature + start_feature(test_case.feature) + end + @failing_step_source = nil + end + + def after_test_step(test_step, result) + return if @failing_step_source + + @failing_step_source = test_step.source.last if result.failed? || (@options[:strict] && (result.pending? || result.undefined?)) + end + + def after_test_case(test_case, result) + test_case_name = NameBuilder.new(test_case) + scenario = test_case_name.scenario_name + scenario_designation = "#{scenario}#{test_case_name.name_suffix}" + output = create_output_string(test_case, scenario, result, test_case_name.row_name) + build_testcase(result, scenario_designation, output) + end + + def done + end_feature if @current_feature + end + + private + + def same_feature_as_previous_test_case?(feature) + @current_feature && @current_feature.file == feature.file && @current_feature.location == feature.location + end + + def start_feature(feature) + raise UnNamedFeatureError.new(feature.file) if feature.name.empty? @current_feature = feature @failures = @errors = @tests = @skipped = 0 - @builder = Builder::XmlMarkup.new( :indent => 2 ) + @builder = Builder::XmlMarkup.new(:indent => 2) @time = 0 # In order to fill out and , we need to # intercept the $stderr and $stdout @@ -36,13 +65,8 @@ def before_feature(feature) @interceptederr = Interceptor::Pipe.wrap(:stderr) end - def before_feature_element(feature_element) - @in_examples = AST_SCENARIO_OUTLINE === feature_element - @steps_start = Time.now - end - - def after_feature(feature) - @testsuite = Builder::XmlMarkup.new( :indent => 2 ) + def end_feature + @testsuite = Builder::XmlMarkup.new(:indent => 2) @testsuite.instruct! @testsuite.testsuite( :failures => @failures, @@ -50,7 +74,7 @@ def after_feature(feature) :skipped => @skipped, :tests => @tests, :time => "%.6f" % @time, - :name => @feature_name ) do + :name => @current_feature.name ) do @testsuite << @builder.target! @testsuite.tag!('system-out') do @testsuite.cdata! strip_control_chars(@interceptedout.buffer.join) @@ -60,110 +84,44 @@ def after_feature(feature) end end - write_file(feature_result_filename(feature.file), @testsuite.target!) + write_file(feature_result_filename(@current_feature.file), @testsuite.target!) Interceptor::Pipe.unwrap! :stdout Interceptor::Pipe.unwrap! :stderr end - def before_background(*args) - @in_background = true - end - - def after_background(*args) - @in_background = false - end - - def feature_name(keyword, name) - raise UnNamedFeatureError.new(@current_feature.file) if name.empty? - lines = name.split(/\r?\n/) - @feature_name = lines[0] - end - - def scenario_name(keyword, name, file_colon_line, source_indent) - @scenario = (name.nil? || name == "") ? "Unnamed scenario" : name.split("\n")[0] - @output = "#{keyword}: #{@scenario}\n\n" - end - - def before_steps(steps) - end - - def after_steps(steps) - return if @in_background || @in_examples - - duration = Time.now - @steps_start - if steps.failed? - steps.each { |step| @output += "#{step.keyword}#{step.name}\n" } - @output += "\nMessage:\n" + def create_output_string(test_case, scenario, result, row_name) + output = "#{test_case.keyword}: #{scenario}\n\n" + return output unless result.failed? || (@options[:strict] && (result.pending? || result.undefined?)) + if test_case.keyword == "Scenario" + output += "#{@failing_step_source.keyword}" unless hook?(@failing_step_source) + output += "#{@failing_step_source.name}\n" + else + output += "Example row: #{row_name}\n" end - build_testcase(duration, steps.status, steps.exception) + output + "\nMessage:\n" end - def before_examples(*args) - @header_row = true - @in_examples = true + def hook?(step) + ["Before hook", "After hook", "AfterStep hook"].include? step.name end - def after_examples(*args) - @in_examples = false - end - - def before_table_row(table_row) - return unless @in_examples - - @table_start = Time.now - end - - def after_table_row(table_row) - return unless @in_examples and AST_EXAMPLE_ROW === table_row - duration = Time.now - @table_start - unless @header_row - name_suffix = " (outline example : #{table_row.name})" - if table_row.failed? - @output += "Example row: #{table_row.name}\n" - @output += "\nMessage:\n" - end - build_testcase(duration, table_row.status, table_row.exception, name_suffix) - end - - @header_row = false if @header_row - end - - def after_test_case(test_case, result) - if @options[:expand] and test_case.keyword == "Scenario Outline" - test_case_name = NameBuilder.new(test_case) - @scenario = test_case_name.outline_name - @output = "#{test_case.keyword}: #{@scenario}\n\n" - @exception = nil - if result.failed? or (@options[:strict] and (result.pending? or result.undefined?)) - if result.failed? - @exception = result.exception - elsif result.backtrace - @exception = result - end - @output += "Example row: #{test_case_name.row_name}\n" - @output += "\nMessage:\n" - end - test_case_result = ResultBuilder.new(result) - build_testcase(test_case_result.test_case_duration, test_case_result.status, @exception, test_case_name.name_suffix) - end - end - - private - - def build_testcase(duration, status, exception = nil, suffix = "") + def build_testcase(result, scenario_designation, output) + duration = ResultBuilder.new(result).test_case_duration @time += duration - classname = @feature_name - name = "#{@scenario}#{suffix}" - pending = [:pending, :undefined].include?(status) && (!@options[:strict]) + classname = @current_feature.name + name = scenario_designation + pending = (result.pending? || result.undefined?) && (!@options[:strict]) @builder.testcase(:classname => classname, :name => name, :time => "%.6f" % duration) do - if status == :skipped || pending + if result.skipped? || pending @builder.skipped @skipped += 1 - elsif status != :passed - @builder.failure(:message => "#{status.to_s} #{name}", :type => status.to_s) do - @builder.cdata! @output + elsif !result.passed? + status = result.to_sym + exception = get_backtrace_object(result) + @builder.failure(:message => "#{status} #{name}", :type => status) do + @builder.cdata! output @builder.cdata!(format_exception(exception)) if exception end @failures += 1 @@ -174,6 +132,16 @@ def build_testcase(duration, status, exception = nil, suffix = "") @tests += 1 end + def get_backtrace_object(result) + if result.failed? + return result.exception + elsif result.backtrace + return result + else + return nil + end + end + def format_exception(exception) (["#{exception.message} (#{exception.class})"] + exception.backtrace).join("\n") end @@ -194,13 +162,15 @@ def write_file(feature_filename, data) def strip_control_chars(cdata) cdata.scan(/[[:print:]\t\n\r]/).join end - + end class NameBuilder - attr_reader :outline_name, :name_suffix, :row_name + attr_reader :scenario_name, :name_suffix, :row_name def initialize(test_case) + @name_suffix = "" + @row_name = "" test_case.describe_source_to self end @@ -208,12 +178,13 @@ def feature(*) self end - def scenario(*) + def scenario(scenario) + @scenario_name = (scenario.name.nil? || scenario.name == "") ? "Unnamed scenario" : scenario.name self end def scenario_outline(outline) - @outline_name = outline.name + @scenario_name = (outline.name.nil? || outline.name == "") ? "Unnamed scenario outline" : outline.name self end @@ -229,37 +200,26 @@ def examples_table_row(row) end class ResultBuilder - attr_reader :status, :test_case_duration + attr_reader :test_case_duration def initialize(result) @test_case_duration = 0 result.describe_to(self) end - def passed - @status = :passed - end + def passed(*) end - def failed - @status = :failed - end + def failed(*) end - def undefined - @status = :undefined - end + def undefined(*) end - def skipped - @status = :skipped - end + def skipped(*) end - def pending(*) - @status = :pending - end + def pending(*) end - def exception(*) - end + def exception(*) end def duration(duration, *) - duration.tap { |duration| @test_case_duration = duration.nanoseconds / 10 ** 9.0 } + duration.tap { |duration| @test_case_duration = duration.nanoseconds / 10**9.0 } end end From fd5e38e01793e599d60fa11b2a3e08adde729389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rasmusson?= Date: Sun, 24 May 2015 11:00:30 +0200 Subject: [PATCH 3/4] Put intercepted stdout and stderr in the testcase element Since Cucumber is Scenario/Test Case oriented it makes more sense to put the intercepted stdout and stderr there, instead in the testsuite element (that represents the Feature). Stdout and stderr elements are already created for the testcase elements, but they have so far always been empty. --- .../docs/formatters/junit_formatter.feature | 104 +++++++++++------- lib/cucumber/formatter/junit.rb | 28 +++-- spec/cucumber/formatter/junit_spec.rb | 10 +- 3 files changed, 84 insertions(+), 58 deletions(-) diff --git a/features/docs/formatters/junit_formatter.feature b/features/docs/formatters/junit_formatter.feature index 8daf2a11b5..b4a7230e64 100644 --- a/features/docs/formatters/junit_formatter.feature +++ b/features/docs/formatters/junit_formatter.feature @@ -71,8 +71,12 @@ Feature: JUnit output formatter - - + + + + + + @@ -86,15 +90,13 @@ Feature: JUnit output formatter ./features/step_definitions/steps.rb:4:in `/^this step fails$/' features/one_passing_one_failing.feature:7:in `Given this step fails']]> - - - + """ @@ -110,8 +112,12 @@ Feature: JUnit output formatter - - + + + + + + @@ -125,15 +131,13 @@ Feature: JUnit output formatter ./features/step_definitions/steps.rb:4:in `/^this step fails$/' features/some_subdirectory/one_passing_one_failing.feature:7:in `Given this step fails']]> - - - + """ @@ -150,20 +154,22 @@ Feature: JUnit output formatter - - + + + + + + - - - + """ @@ -190,8 +196,12 @@ Feature: JUnit output formatter ./features/step_definitions/steps.rb:3:in `/^this step is pending$/' features/pending.feature:4:in `Given this step is pending']]> - - + + + + + + @@ -204,15 +214,13 @@ Feature: JUnit output formatter - - - + """ @@ -248,8 +256,12 @@ You *must* specify --out DIR for the junit formatter - - + + + + + + @@ -264,8 +276,12 @@ You *must* specify --out DIR for the junit formatter features/scenario_outline.feature:9:in `Given this step fails' features/scenario_outline.feature:4:in `Given this step ']]> - - + + + + + + @@ -280,8 +296,12 @@ You *must* specify --out DIR for the junit formatter features/scenario_outline.feature:10:in `Given this step is pending' features/scenario_outline.feature:4:in `Given this step ']]> - - + + + + + + @@ -295,15 +315,13 @@ You *must* specify --out DIR for the junit formatter features/scenario_outline.feature:11:in `Given this step is undefined' features/scenario_outline.feature:4:in `Given this step ']]> - - - + """ @@ -319,8 +337,12 @@ You *must* specify --out DIR for the junit formatter - - + + + + + + @@ -335,8 +357,12 @@ You *must* specify --out DIR for the junit formatter features/scenario_outline.feature:9:in `Given this step fails' features/scenario_outline.feature:4:in `Given this step ']]> - - + + + + + + @@ -351,8 +377,12 @@ You *must* specify --out DIR for the junit formatter features/scenario_outline.feature:10:in `Given this step is pending' features/scenario_outline.feature:4:in `Given this step ']]> - - + + + + + + @@ -366,15 +396,13 @@ You *must* specify --out DIR for the junit formatter features/scenario_outline.feature:11:in `Given this step is undefined' features/scenario_outline.feature:4:in `Given this step ']]> - - - + """ diff --git a/lib/cucumber/formatter/junit.rb b/lib/cucumber/formatter/junit.rb index 8c4cc8aed2..78b5eb4492 100644 --- a/lib/cucumber/formatter/junit.rb +++ b/lib/cucumber/formatter/junit.rb @@ -27,6 +27,10 @@ def before_test_case(test_case) start_feature(test_case.feature) end @failing_step_source = nil + # In order to fill out and , we need to + # intercept the $stderr and $stdout + @interceptedout = Interceptor::Pipe.wrap(:stdout) + @interceptederr = Interceptor::Pipe.wrap(:stderr) end def after_test_step(test_step, result) @@ -41,6 +45,9 @@ def after_test_case(test_case, result) scenario_designation = "#{scenario}#{test_case_name.name_suffix}" output = create_output_string(test_case, scenario, result, test_case_name.row_name) build_testcase(result, scenario_designation, output) + + Interceptor::Pipe.unwrap! :stdout + Interceptor::Pipe.unwrap! :stderr end def done @@ -59,10 +66,6 @@ def start_feature(feature) @failures = @errors = @tests = @skipped = 0 @builder = Builder::XmlMarkup.new(:indent => 2) @time = 0 - # In order to fill out and , we need to - # intercept the $stderr and $stdout - @interceptedout = Interceptor::Pipe.wrap(:stdout) - @interceptederr = Interceptor::Pipe.wrap(:stderr) end def end_feature @@ -76,18 +79,9 @@ def end_feature :time => "%.6f" % @time, :name => @current_feature.name ) do @testsuite << @builder.target! - @testsuite.tag!('system-out') do - @testsuite.cdata! strip_control_chars(@interceptedout.buffer.join) - end - @testsuite.tag!('system-err') do - @testsuite.cdata! strip_control_chars(@interceptederr.buffer.join) - end end write_file(feature_result_filename(@current_feature.file), @testsuite.target!) - - Interceptor::Pipe.unwrap! :stdout - Interceptor::Pipe.unwrap! :stderr end def create_output_string(test_case, scenario, result, row_name) @@ -126,8 +120,12 @@ def build_testcase(result, scenario_designation, output) end @failures += 1 end - @builder.tag!('system-out') - @builder.tag!('system-err') + @builder.tag!('system-out') do + @builder.cdata! strip_control_chars(@interceptedout.buffer.join) + end + @builder.tag!('system-err') do + @builder.cdata! strip_control_chars(@interceptederr.buffer.join) + end end @tests += 1 end diff --git a/spec/cucumber/formatter/junit_spec.rb b/spec/cucumber/formatter/junit_spec.rb index 8d3f46e6b8..5a44562c9d 100644 --- a/spec/cucumber/formatter/junit_spec.rb +++ b/spec/cucumber/formatter/junit_spec.rb @@ -59,7 +59,7 @@ def after_step(step) end end - it { expect(@doc.xpath('//testsuite/system-out').first.content).to match(/\s+boo boo\s+/) } + it { expect(@doc.xpath('//testsuite/testcase/system-out').first.content).to match(/\s+boo boo\s+/) } end describe "a feature with no name" do @@ -92,12 +92,12 @@ def after_step(step) it { expect(@doc.to_s).to match /One passing scenario, one failing scenario/ } - it 'has a root system-out node' do - expect(@doc.xpath('//testsuite/system-out').size).to eq 1 + it 'has not a root system-out node' do + expect(@doc.xpath('//testsuite/system-out').size).to eq 0 end - it 'has a root system-err node' do - expect(@doc.xpath('//testsuite/system-err').size).to eq 1 + it 'has not a root system-err node' do + expect(@doc.xpath('//testsuite/system-err').size).to eq 0 end it 'has a system-out node under ' do From 4f3ebda7645f76fb5de479a5603b36be7c2fef8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rasmusson?= Date: Thu, 11 Jun 2015 08:56:15 +0200 Subject: [PATCH 4/4] Use Result#ok? in the Junit Formatter --- lib/cucumber/formatter/junit.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/cucumber/formatter/junit.rb b/lib/cucumber/formatter/junit.rb index 78b5eb4492..398d8d6e00 100644 --- a/lib/cucumber/formatter/junit.rb +++ b/lib/cucumber/formatter/junit.rb @@ -36,7 +36,7 @@ def before_test_case(test_case) def after_test_step(test_step, result) return if @failing_step_source - @failing_step_source = test_step.source.last if result.failed? || (@options[:strict] && (result.pending? || result.undefined?)) + @failing_step_source = test_step.source.last unless result.ok?(@options[:strict]) end def after_test_case(test_case, result) @@ -86,7 +86,7 @@ def end_feature def create_output_string(test_case, scenario, result, row_name) output = "#{test_case.keyword}: #{scenario}\n\n" - return output unless result.failed? || (@options[:strict] && (result.pending? || result.undefined?)) + return output if result.ok?(@options[:strict]) if test_case.keyword == "Scenario" output += "#{@failing_step_source.keyword}" unless hook?(@failing_step_source) output += "#{@failing_step_source.name}\n" @@ -105,10 +105,9 @@ def build_testcase(result, scenario_designation, output) @time += duration classname = @current_feature.name name = scenario_designation - pending = (result.pending? || result.undefined?) && (!@options[:strict]) @builder.testcase(:classname => classname, :name => name, :time => "%.6f" % duration) do - if result.skipped? || pending + if !result.passed? && result.ok?(@options[:strict]) @builder.skipped @skipped += 1 elsif !result.passed?