feat(config): allow env var RETRIES_HIDDEN to hide retried tests… (#22)

* fix: fix before all failure not stopping test execution

* Add demonstration test

* fix before-error-spec

* chore: add e2e test support, tests

* feat(config): add RETRIES_HIDDEN config

- hides command logs of retried tests, so you only
  see the last failure

* chore: fix e2e tests in CI

* chore: fix e2e tests in CI 2

* fix: prevent erroneous passing last test in suite with after hook

circle.yml

@@ -19,6 +19,8 @@ jobs:
         command: yarn lint
     - run:
         command: yarn test
+    - run:
+        command: yarn test-e2e
     - run:
         command: yarn run semantic-release
 workflows:

cypress.json

@@ -1,3 +1,4 @@
 {
-  "projectId": "3e57fb"
+  "projectId": "3e57fb",
+  "integrationFolder": "cypress/tests"
 }

cypress/.eslintrc.json

@@ -4,5 +4,8 @@
   ],
   "extends": [
     "plugin:cypress/recommended"
-  ]
+  ],
+  "rules": {
+    "cypress/no-unnecessary-waiting": "off"
+  }
 }

cypress/tests/e2e/after-hook-fail.spec.1.js

@@ -0,0 +1,30 @@
+/// <reference types="cypress"/>
+
+/* EXPECT: {
+  totalFailed: 1,
+  totalPassed: 1
+} */
+
+describe('should not pass when really failed', () => {
+
+  it('expect pass', () => {
+    Cypress.currentTest.retries(2)
+
+    cy.wait(10).should(() => {
+      // throw new Error('foo')
+    })
+  })
+
+  it('expect fail', () => {
+    Cypress.currentTest.retries(2)
+
+    cy.wait(10).should(() => {
+      throw new Error('foo')
+    })
+  })
+
+  after(() => {
+    Cypress.log({ message: 'foo' })
+  })
+
+})

cypress/tests/e2e/after-hook-fail.spec.js

@@ -0,0 +1,20 @@
+/// <reference types="cypress"/>
+
+/* EXPECT: {
+  totalFailed: 1,
+  totalPassed: 0
+} */
+
+describe('should not pass when really failed', () => {
+  it('expect fail', () => {
+    Cypress.currentTest.retries(2)
+
+    cy.wait(10).should(() => {
+      throw new Error('foo')
+
+    })
+  })
+
+  after(() => {})
+
+})

cypress/tests/unit/after-hook-pass.spec.js

@@ -0,0 +1,44 @@
+/// <reference types="cypress"/>
+
+const { _ } = Cypress
+
+const getFailTwice = () => {
+  return _.before(3, () => {
+    throw new Error('foo')
+  })
+}
+
+let failTwice = getFailTwice()
+
+describe('suite', () => {
+  it('expect pass 1', () => {
+    Cypress.currentTest.retries(2)
+
+    cy.wait(10).should(() => {
+      failTwice()
+      failTwice = getFailTwice()
+    })
+  })
+
+  it('expect pass 2', () => {
+    Cypress.currentTest.retries(2)
+
+    cy.wait(10).should(() => {
+      failTwice()
+      failTwice = getFailTwice()
+    })
+  })
+
+  it('expect pass 3', () => {
+    Cypress.currentTest.retries(2)
+
+    cy.wait(10).should(() => {
+      // throw new Error('foo')
+      failTwice()
+      failTwice = getFailTwice
+    })
+  })
+
+  after(() => {
+  })
+})

cypress/tests/unit/before-error.spec.js

@@ -0,0 +1,71 @@
+/// <reference types="cypress"/>
+
+const runner = Cypress.mocha.getRunner()
+
+const getTest = (r) => r && r.ctx.currentTest || r
+
+const throwMetaError = (mes) => {
+  const err = new Error(mes)
+
+  err.name = 'MetaError'
+  throw err
+}
+
+const throwFakeError = (mes) => {
+  const err = new Error(mes)
+
+  err.name = 'FakeError'
+  throw err
+}
+
+Cypress.on('test:before:run', () => {
+
+  runner.once('fail', (r) => {
+    const runnable = cy.state('runnable')
+    const test = getTest(runnable)
+
+    if (test.err.name === 'Uncaught MetaError') {
+      test.err.message = test.err.message.split('\nThis error originated from')[0]
+
+      return
+    }
+
+    if (test.err.name === 'FakeError') {
+
+      test.error = null
+      test.state = 'passed'
+
+    }
+
+  })
+
+})
+
+describe('Exception in before block', () => {
+  describe('Broken case', () => {
+    before(() => {
+      Cypress.currentTest.retries(1)
+
+      throwFakeError('This is the real error!')
+    })
+
+    it('shouldn\'t run this test', () => {
+      throwMetaError('should not have ran')
+    })
+
+    it(`also shouldn't run this`, () => {
+      throwMetaError('should not have ran 2')
+    })
+  })
+
+  describe('Working case', () => {
+    before(() => {
+      Cypress.currentTest.retries(0)
+      throwFakeError('This is the real error!')
+    })
+
+    it('shouldn\'t run this test', () => {
+      throwMetaError('should not have ran')
+    })
+  })
+})

cypress/tests/unit/retries-hidden.spec.js

@@ -0,0 +1,27 @@
+/// <reference types="cypress"/>
+
+const { _ } = Cypress
+
+Cypress.env('RETRIES_HIDDEN', true)
+
+if (!top.alreadyRan) {
+  top.alreadyRan = true
+  cy.$$('.restart', top.document).click()
+}
+
+const failTwice = _.before(3, () => {
+  throw new Error('foo')
+})
+
+describe('suite', () => {
+
+  it('retries are hidden when RETRIES_HIDDEN', () => {
+    Cypress.currentTest.retries(2)
+
+    cy.wait(10).should(() => {
+      failTwice()
+      cy.wrap(cy.$$('.retry-0', top.document)).should('not.be.visible')
+    })
+
+  })
+})

package.json

@@ -5,25 +5,40 @@
   "main": "src",
   "scripts": {
     "lint": "eslint --ext json,js,ts .",
-    "test": "cypress run"
+    "test": "cypress run --spec '**/unit/**'",
+    "test-e2e": "mocha test/e2e"
   },
   "husky": {
     "hooks": {
       "pre-commit": "lint-staged"
     }
   },
   "devDependencies": {
+    "@babel/code-frame": "^7.5.5",
     "@cypress/eslint-plugin-dev": "^4.0.0",
+    "@types/fs-extra": "^8.0.0",
+    "@types/lodash": "^4.14.138",
     "@typescript-eslint/eslint-plugin": "^2.2.0",
     "@typescript-eslint/parser": "^2.2.0",
-    "cypress": "^3.1.5",
+    "bluebird": "^3.5.5",
+    "chai": "^4.2.0",
+    "chalk": "^2.4.2",
+    "cypress": "^3.4.1",
+    "debug": "^4.1.1",
     "eslint": "^5.16.0",
     "eslint-plugin-cypress": "^2.6.0",
     "eslint-plugin-json-format": "^2.0.1",
     "eslint-plugin-mocha": "^6.0.0",
+    "fast-glob": "^3.0.4",
+    "fs-extra": "^8.1.0",
+    "glob": "^7.1.4",
     "husky": "^3.0.5",
+    "json5": "^2.1.0",
     "lint-staged": "^9.2.5",
+    "lodash": "^4.17.15",
+    "mocha": "^6.2.0",
     "semantic-release": "^15.13.24",
+    "strip-ansi": "^5.2.0",
     "typescript": "^3.6.2"
   },
   "license": "MIT",

src/index.js

@@ -4,6 +4,14 @@ const getDefaultRetries = () => {
   return Cypress.env('RETRIES')
 }
 
+const getRetriesHidden = () => {
+  return Cypress.env('RETRIES_HIDDEN')
+}
+
+const debug = function () {
+  // console.log.apply(this, arguments)
+}
+
 const _clone = Cypress.mocha._mocha.Mocha.Test.prototype.clone
 
 Cypress.mocha._mocha.Mocha.Test.prototype.clone = function () {
@@ -39,7 +47,7 @@ Cypress.runner.onRunnableRun = function (runnableRun, runnable, args) {
   const r = runnable
   const isHook = r.type === 'hook'
   const isAfterAllHook = isHook && r.hookName.match(/after all/)
-  const isBeforeHook = isHook && r.hookName.match(/before/)
+  const isBeforeHook = isHook && r.hookName.match(/before each/)
   const test = r.ctx.currentTest || r
 
   if (test._currentRetry === 0 && logs.testId !== test.id) {
@@ -49,13 +57,6 @@ Cypress.runner.onRunnableRun = function (runnableRun, runnable, args) {
 
   const next = args[0]
 
-  if (isAfterAllHook) {
-    if (test.state !== 'failed') {
-      test.err = null
-      test.state = 'passed'
-    }
-  }
-
   debug('on:', r.title)
 
   if (
@@ -129,7 +130,8 @@ addGlobalStyle(/*css*/ `
 }
 
 .command.ignored {
-  opacity: 0.3
+  opacity: 0.3;
+  ${getRetriesHidden() ? 'display: none' : ''}
 }
 .command.ignored:hover  {
   opacity: 1
@@ -184,10 +186,6 @@ function addGlobalStyle (css) {
   head.appendChild(style)
 }
 
-const debug = function () {
-  // console.log.apply(this, arguments)
-}
-
 Object.defineProperty(Cypress, 'currentTest', {
   configurable: true,
   get () {

test/.eslintrc.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "plugin:@cypress/dev/tests"
+  ]
+}

test/e2e/main.spec.js

@@ -0,0 +1,18 @@
+const { runTest } = require('../helpers')
+const path = require('path')
+const glob = require('fast-glob')
+
+describe('can test', async () => {
+
+  glob.sync(path.join(__dirname, '../../cypress/tests/e2e/**.*'))
+  .map((v) => {
+  // .mapSeries((v) => {
+    const filename = path.relative(process.cwd(), v)
+
+    it(`test: ${filename}`, async () => {
+      await runTest({
+        spec: v,
+      })
+    })
+  })
+})