test: add REPORT_ONLY mode for test collection

test/jest/playwrightEnvironment.js

@@ -20,10 +20,11 @@ const os = require('os');
 const path = require('path');
 const fs = require('fs');
 const debug = require('debug');
-const platform = os.platform();
+const platform = process.env.REPORT_ONLY_PLATFORM || os.platform();
 const GoldenUtils = require('../../utils/testrunner/GoldenUtils');
 const {installCoverageHooks} = require('./coverage');
 const browserName = process.env.BROWSER || 'chromium';
+const reportOnly = !!process.env.REPORT_ONLY_PLATFORM;
 
 class PlaywrightEnvironment extends NodeEnvironment {
   constructor(config, context) {
@@ -109,21 +110,32 @@ class PlaywrightEnvironment extends NodeEnvironment {
           return args[0] ? describeSkip : this.global.describe;
         return describeSkip(...args);
       };
-      this.global.describe.fail = this.global.describe.skip;
+
+      function addSlow(f) {
+        f.slow = () => {
+          return (...args) => f(...args, 90000);
+        };
+        return f;
+      }
 
       const itSkip = this.global.it.skip;
-      itSkip.slow = () => itSkip;
+      addSlow(itSkip);
+      addSlow(this.global.it);
       this.global.it.skip = (...args) => {
         if (args.length === 1)
           return args[0] ? itSkip : this.global.it;
         return itSkip(...args);
       };
-      this.global.it.fail = this.global.it.skip;
-      this.global.it.slow = () => {
-        return (name, fn) => {
-          return this.global.it(name, fn, 90000);
+      if (reportOnly) {
+        this.global.it.fail = condition => {
+          return addSlow((...inner) => {
+            inner[1].__fail = !!condition;
+            return this.global.it(...inner);
+          });
         };
-      };
+      } else {
+        this.global.it.fail = this.global.it.skip;
+      }
 
       const testOptions = this.global.testOptions;
       function toBeGolden(received, goldenName) {
@@ -159,6 +171,11 @@ class PlaywrightEnvironment extends NodeEnvironment {
     if (event.name === 'test_start') {
       const fn = event.test.fn;
       event.test.fn = async () => {
+        if (reportOnly) {
+          if (fn.__fail)
+            throw new Error('fail');
+          return;
+        }
         debug('pw:test')(`start "${testOrSuiteName(event.test)}"`);
         try {
           await this.fixturePool.resolveParametersAndRun(fn);

test/jest/reporter.js

@@ -14,10 +14,13 @@
  * limitations under the License.
  */
 
-const fs = require("fs")
+const fs = require('fs');
+const os = require('os');
 
 module.exports = function Reporter() {
   this.onRunComplete = (test, runResults) => {
+    runResults.platform = process.env.REPORT_ONLY_PLATFORM || os.platform();
+    runResults.browserName = process.env.BROWSER || 'chromium';
     fs.writeFileSync('jest-report.json', JSON.stringify(runResults, undefined, 2));
-  }
+  };
 }

test/page.jest.js

@@ -113,7 +113,8 @@ describe('Async stacks', () => {
   });
 });
 
-describe.fail(FFOX && WIN || USES_HOOKS)('Page.Events.Crash', function() {
+const CRASH_FAIL = (FFOX && WIN) || USES_HOOKS;
+describe('Page.Events.Crash', function() {
   // Firefox Win: it just doesn't crash sometimes.
 
   function crash(pageImpl) {
@@ -125,27 +126,27 @@ describe.fail(FFOX && WIN || USES_HOOKS)('Page.Events.Crash', function() {
       pageImpl._delegate._session.send('Page.crash', {}).catch(e => {});
   }
 
-  it('should emit crash event when page crashes', async({page, toImpl}) => {
+  it.fail(CRASH_FAIL)('should emit crash event when page crashes', async({page, toImpl}) => {
     await page.setContent(`<div>This page should crash</div>`);
     crash(toImpl(page));
     await new Promise(f => page.on('crash', f));
   });
-  it('should throw on any action after page crashes', async({page, toImpl}) => {
+  it.fail(CRASH_FAIL)('should throw on any action after page crashes', async({page, toImpl}) => {
     await page.setContent(`<div>This page should crash</div>`);
     crash(toImpl(page));
     await page.waitForEvent('crash');
     const err = await page.evaluate(() => {}).then(() => null, e => e);
     expect(err).toBeTruthy();
     expect(err.message).toContain('crash');
   });
-  it('should cancel waitForEvent when page crashes', async({page, toImpl}) => {
+  it.fail(CRASH_FAIL)('should cancel waitForEvent when page crashes', async({page, toImpl}) => {
     await page.setContent(`<div>This page should crash</div>`);
     const promise = page.waitForEvent('response').catch(e => e);
     crash(toImpl(page));
     const error = await promise;
     expect(error.message).toContain('Page crashed');
   });
-  it('should cancel navigation when page crashes', async({page, toImpl, server}) => {
+  it.fail(CRASH_FAIL)('should cancel navigation when page crashes', async({page, toImpl, server}) => {
     await page.setContent(`<div>This page should crash</div>`);
     server.setRoute('/one-style.css', () => {});
     const promise = page.goto(server.PREFIX + '/one-style.html').catch(e => e);
@@ -154,7 +155,7 @@ describe.fail(FFOX && WIN || USES_HOOKS)('Page.Events.Crash', function() {
     const error = await promise;
     expect(error.message).toContain('Navigation failed because page crashed');
   });
-  it('should be able to close context when page crashes', async({page, toImpl}) => {
+  it.fail(CRASH_FAIL)('should be able to close context when page crashes', async({page, toImpl}) => {
     await page.setContent(`<div>This page should crash</div>`);
     crash(toImpl(page));
     await page.waitForEvent('crash');