feat: include exceptions in test step result messages (#2229)

cucumber · Feb 6, 2023 · 9779d26 · 9779d26
1 parent 88eb48d
commit 9779d26
Show file tree

Hide file tree

Showing 12 changed files with 183 additions and 39 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ Please see [CONTRIBUTING.md](https://github.com/cucumber/cucumber/blob/master/CO
 ## [Unreleased]
 ### Added
 - Affirm support for Node.js 19 [#2230](https://github.com/cucumber/cucumber-js/pull/2230)
+- Include some exception details in the result of a test step for downstream tools [#2229](https://github.com/cucumber/cucumber-js/pull/2229)
 
 ### Fixed
 - Handle invalid characters when generating XML for JUnit formatter [#2228](https://github.com/cucumber/cucumber-js/pull/2228)

diff --git a/compatibility/features/cdata/cdata.ts b/compatibility/features/cdata/cdata.ts
@@ -0,0 +1,9 @@
+import assert from 'assert'
+import { Given } from '../../../src'
+
+Given(
+  'I have {int} <![CDATA[cukes]]> in my belly',
+  function (cukeCount: number) {
+    assert(cukeCount)
+  }
+)
diff --git a/features/fixtures/formatters/failed.message.json.ts b/features/fixtures/formatters/failed.message.json.ts
@@ -142,6 +142,10 @@ module.exports = [
         },
         status: 'FAILED',
         message: 'Error: my error',
+        exception: {
+          type: 'Error',
+          message: 'my error',
+        },
       },
       timestamp: {
         seconds: 0,

diff --git a/features/fixtures/formatters/retried.message.json.ts b/features/fixtures/formatters/retried.message.json.ts
@@ -142,6 +142,10 @@ module.exports = [
         },
         status: 'FAILED',
         message: 'Error: my error',
+        exception: {
+          type: 'Error',
+          message: 'my error',
+        },
       },
       timestamp: {
         seconds: 0,

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -237,7 +237,7 @@
     "yup": "^0.32.11"
   },
   "devDependencies": {
-    "@cucumber/compatibility-kit": "11.0.1",
+    "@cucumber/compatibility-kit": "11.2.0",
     "@cucumber/query": "12.0.1",
     "@microsoft/api-documenter": "7.19.27",
     "@microsoft/api-extractor": "7.33.7",

diff --git a/src/formatter/junit_formatter.ts b/src/formatter/junit_formatter.ts
@@ -36,9 +36,15 @@ interface IJUnitTestCase {
   steps: IJUnitTestStep[]
 }
 
+interface IJUnitTestCaseFailure {
+  type: string
+  message?: string
+  detail: string
+}
+
 interface IJUnitTestCaseResult {
   status: TestStepResultStatus
-  message?: string
+  failure?: IJUnitTestCaseFailure
 }
 
 interface IJUnitTestStep {
@@ -60,16 +66,6 @@ interface IBuildJUnitTestStepOptions {
   testStepResult: messages.TestStepResult
 }
 
-const statusDescriptions: Record<TestStepResultStatus, string> = {
-  UNKNOWN: `A result couldn't be established`,
-  PASSED: 'Everything went fine',
-  SKIPPED: 'The test case was skipped',
-  PENDING: 'A step in the test case is not yet implemented',
-  UNDEFINED: 'A step in the test case is not defined',
-  AMBIGUOUS: 'Multiple definitions match one of the steps in the test case',
-  FAILED: 'A hook or step failed',
-}
-
 export default class JunitFormatter extends Formatter {
   private readonly names: Record<string, string[]> = {}
   public static readonly documentation: string = 'Outputs JUnit report'
@@ -134,8 +130,20 @@ export default class JunitFormatter extends Formatter {
   }
 
   private getTestCaseResult(steps: IJUnitTestStep[]): IJUnitTestCaseResult {
-    const worstResult = getWorstTestStepResult(steps.map((step) => step.result))
-    return worstResult
+    const { status, message, exception } = getWorstTestStepResult(
+      steps.map((step) => step.result)
+    )
+    return {
+      status,
+      failure:
+        message || exception
+          ? {
+              type: exception?.type,
+              message: exception?.message,
+              detail: message,
+            }
+          : undefined,
+    }
   }
 
   private durationToSeconds(duration: Duration): number {
@@ -262,11 +270,11 @@ export default class JunitFormatter extends Formatter {
         xmlTestCase.ele('skipped')
       } else if (test.result.status !== TestStepResultStatus.PASSED) {
         const xmlFailure = xmlTestCase.ele('failure', {
-          type: test.result.status,
-          message: statusDescriptions[test.result.status],
+          type: test.result.failure?.type,
+          message: test.result.failure?.message,
         })
-        if (test.result.message) {
-          xmlFailure.cdata(test.result.message)
+        if (test.result?.failure) {
+          xmlFailure.cdata(test.result.failure.detail)
         }
       }
       xmlTestCase.ele('system-out', {}).cdata(test.systemOutput)

diff --git a/src/formatter/junit_formatter_spec.ts b/src/formatter/junit_formatter_spec.ts
@@ -20,6 +20,10 @@ function getJUnitFormatterSupportCodeLibrary(
       clock.tick(1)
     })
 
+    Given('I have <![CDATA[cukes]]> in my belly', function () {
+      clock.tick(1)
+    })
+
     let willPass = false
     Given('a flaky step', function () {
       clock.tick(1)
@@ -145,7 +149,7 @@ describe('JunitFormatter', () => {
           '<?xml version="1.0"?>\n' +
             '<testsuite failures="1" skipped="0" name="cucumber-js" time="0.001" tests="1">\n' +
             '  <testcase classname="my feature" name="my scenario" time="0.001">\n' +
-            '    <failure type="FAILED" message="A hook or step failed"><![CDATA[error]]></failure>\n' +
+            '    <failure type="Error" message="error"><![CDATA[error]]></failure>\n' +
             '    <system-out><![CDATA[Given a failing step......................................................failed]]></system-out>\n' +
             '  </testcase>\n' +
             '</testsuite>'
@@ -179,7 +183,7 @@ describe('JunitFormatter', () => {
           '<?xml version="1.0"?>\n' +
             '<testsuite failures="1" skipped="0" name="cucumber-js" time="0.001" tests="1">\n' +
             '  <testcase classname="my feature" name="my scenario" time="0.001">\n' +
-            '    <failure type="FAILED" message="A hook or step failed"><![CDATA[Error: include invalid character]]></failure>\n' +
+            '    <failure type="Error" message="Error: include invalid character"><![CDATA[Error: include invalid character]]></failure>\n' +
             '    <system-out><![CDATA[Given a failing step with invalid character...............................failed]]></system-out>\n' +
             '  </testcase>\n' +
             '</testsuite>'
@@ -250,7 +254,7 @@ describe('JunitFormatter', () => {
           '<?xml version="1.0"?>\n' +
             '<testsuite failures="1" skipped="0" name="cucumber-js" time="0.001" tests="1">\n' +
             '  <testcase classname="my feature" name="my scenario" time="0.001">\n' +
-            '    <failure type="PENDING" message="A step in the test case is not yet implemented"/>\n' +
+            '    <failure/>\n' +
             '    <system-out><![CDATA[Given a pending step.....................................................pending]]></system-out>\n' +
             '  </testcase>\n' +
             '</testsuite>'
@@ -322,7 +326,7 @@ describe('JunitFormatter', () => {
           '<?xml version="1.0"?>\n' +
             '<testsuite failures="1" skipped="0" name="cucumber-js" time="0" tests="1">\n' +
             '  <testcase classname="my feature" name="my scenario" time="0">\n' +
-            '    <failure type="UNDEFINED" message="A step in the test case is not defined"/>\n' +
+            '    <failure/>\n' +
             '    <system-out><![CDATA[Given a passing step...................................................undefined]]></system-out>\n' +
             '  </testcase>\n' +
             '</testsuite>'
@@ -409,7 +413,7 @@ describe('JunitFormatter', () => {
           '    <system-out><![CDATA[Given a passing step......................................................passed]]></system-out>\n' +
           '  </testcase>\n' +
           '  <testcase classname="my feature" name="my templated scenario [1]" time="0.001">\n' +
-          '    <failure type="FAILED" message="A hook or step failed"><![CDATA[error]]></failure>\n' +
+          '    <failure type="Error" message="error"><![CDATA[error]]></failure>\n' +
           '    <system-out><![CDATA[Given a failing step......................................................failed]]></system-out>\n' +
           '  </testcase>\n' +
           '</testsuite>'
@@ -518,4 +522,43 @@ describe('JunitFormatter', () => {
       )
     })
   })
+
+  describe('content containing CDATA', () => {
+    it('outputs the feature', async () => {
+      // Arrange
+      const sources = [
+        {
+          data: [
+            'Feature: my feature',
+            '  my feature description',
+            '',
+            '  Scenario: my scenario',
+            '    my scenario description',
+            '',
+            '    Given I have <![CDATA[cukes]]> in my belly',
+          ].join('\n'),
+          uri: 'a.feature',
+        },
+      ]
+
+      const supportCodeLibrary = getJUnitFormatterSupportCodeLibrary(clock)
+
+      // Act
+      const output = await testFormatter({
+        sources,
+        supportCodeLibrary,
+        type: 'junit',
+      })
+
+      // Assert
+      expect(output).xml.to.deep.equal(
+        '<?xml version="1.0"?>\n' +
+          '<testsuite failures="0" skipped="0" name="cucumber-js" time="0.001" tests="1">\n' +
+          '  <testcase classname="my feature" name="my scenario" time="0.001">\n' +
+          '    <system-out><![CDATA[Given I have <![CDATA[cukes]]]]><![CDATA[> in my belly................................passed]]></system-out>\n' +
+          '  </testcase>\n' +
+          '</testsuite>'
+      )
+    })
+  })
 })
diff --git a/src/runtime/format_error.ts b/src/runtime/format_error.ts
@@ -1,8 +1,12 @@
+import { TestStepResult } from '@cucumber/messages'
 import { format } from 'assertion-error-formatter'
 import errorStackParser from 'error-stack-parser'
 import { filterStackTrace } from '../filter_stack_trace'
 
-export function formatError(error: Error, filterStackTraces: boolean): string {
+export function formatError(
+  error: Error,
+  filterStackTraces: boolean
+): Pick<TestStepResult, 'message' | 'exception'> {
   let filteredStack: string
   if (filterStackTraces) {
     try {
@@ -13,10 +17,17 @@ export function formatError(error: Error, filterStackTraces: boolean): string {
       // if we weren't able to parse and filter, we'll settle for the original
     }
   }
-  return format(error, {
+  const message = format(error, {
     colorFns: {
       errorStack: (stack: string) =>
         filteredStack ? `\n${filteredStack}` : stack,
     },
   })
+  return {
+    message,
+    exception: {
+      type: error.name || 'Error',
+      message: typeof error === 'string' ? error : error.message,
+    },
+  }
 }
diff --git a/src/runtime/format_error_spec.ts b/src/runtime/format_error_spec.ts
@@ -0,0 +1,58 @@
+import { expect } from 'chai'
+import assert from 'assert'
+import { formatError } from './format_error'
+
+describe('formatError', () => {
+  function testFormatError(fn: () => void, filterStackTraces: boolean = false) {
+    try {
+      fn()
+      return undefined
+    } catch (error) {
+      return formatError(error, filterStackTraces)
+    }
+  }
+
+  it('should handle a custom error', () => {
+    expect(
+      testFormatError(() => {
+        assert.ok(false, 'Thing that should have been truthy was falsy!')
+      }).exception
+    ).to.eql({
+      type: 'AssertionError',
+      message: 'Thing that should have been truthy was falsy!',
+    })
+  })
+
+  it('should handle a generic error', () => {
+    expect(
+      testFormatError(() => {
+        throw new Error('A generally bad thing happened!')
+      }).exception
+    ).to.eql({
+      type: 'Error',
+      message: 'A generally bad thing happened!',
+    })
+  })
+
+  it('should handle an omitted message', () => {
+    expect(
+      testFormatError(() => {
+        throw new Error()
+      }).exception
+    ).to.eql({
+      type: 'Error',
+      message: '',
+    })
+  })
+
+  it('should handle a thrown string', () => {
+    expect(
+      testFormatError(() => {
+        throw 'Yikes!'
+      }).exception
+    ).to.eql({
+      type: 'Error',
+      message: 'Yikes!',
+    })
+  })
+})
diff --git a/src/runtime/step_runner.ts b/src/runtime/step_runner.ts
@@ -62,13 +62,13 @@ export async function run({
 
   const duration = stopwatch.stop().duration()
   let status: messages.TestStepResultStatus
-  let message: string
+  let details = {}
   if (result === 'skipped') {
     status = messages.TestStepResultStatus.SKIPPED
   } else if (result === 'pending') {
     status = messages.TestStepResultStatus.PENDING
   } else if (doesHaveValue(error)) {
-    message = formatError(error, filterStackTraces)
+    details = formatError(error, filterStackTraces)
     status = messages.TestStepResultStatus.FAILED
   } else {
     status = messages.TestStepResultStatus.PASSED
@@ -77,7 +77,7 @@ export async function run({
   return {
     duration,
     status,
-    message,
+    ...details,
   }
 }