chore(test-runner): move into its own folder and typescript project (#…

…3548)

.eslintignore

@@ -15,3 +15,4 @@ src/webkit/protocol.ts
 utils/generate_types/overrides.d.ts
 utils/generate_types/test/test.ts
 test/
+test-runner/

.github/workflows/auto_roll.yml

@@ -27,7 +27,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"

.github/workflows/tests.yml

@@ -37,7 +37,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000 && npm run coverage"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 && npm run coverage"
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
@@ -68,7 +68,7 @@ jobs:
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
     - run: npm run build
-    - run: node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000
+    - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
@@ -102,7 +102,7 @@ jobs:
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
     - run: npm run build
-    - run: node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000
+    - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000
       shell: bash
       env:
         BROWSER: ${{ matrix.browser }}
@@ -159,7 +159,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
       if: ${{ always() }}
       env:
         BROWSER: ${{ matrix.browser }}
@@ -197,7 +197,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"

package.json

@@ -9,19 +9,19 @@
     "node": ">=10.15.0"
   },
   "scripts": {
-    "ctest": "cross-env BROWSER=chromium node test/runner/cli test/",
-    "ftest": "cross-env BROWSER=firefox node test/runner/cli test/",
-    "wtest": "cross-env BROWSER=webkit node test/runner/cli test/",
-    "test": "node test/runner/cli test/",
+    "ctest": "cross-env BROWSER=chromium node test-runner/cli test/",
+    "ftest": "cross-env BROWSER=firefox node test-runner/cli test/",
+    "wtest": "cross-env BROWSER=webkit node test-runner/cli test/",
+    "test": "node test-runner/cli test/",
     "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts ./src || eslint --ext js,ts ./src",
     "tsc": "tsc -p .",
     "tsc-installer": "tsc -p ./src/install/tsconfig.json",
     "doc": "node utils/doclint/cli.js",
-    "test-infra": "node test/runner/cli utils/doclint/check_public_api/test/test.js && node test/runner/cli utils/doclint/preprocessor/test.js",
+    "test-infra": "node test-runner/cli utils/doclint/check_public_api/test/test.js && node test-runner/cli utils/doclint/preprocessor/test.js",
     "lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && npm run generate-channels && npm run test-types && npm run test-infra",
     "clean": "rimraf lib && rimraf types",
     "prepare": "node install-from-github.js",
-    "build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run generate-types",
+    "build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run build-testrunner && npm run generate-types",
     "watch": "node utils/watch.js",
     "test-types": "npm run generate-types && npx -p [email protected] tsc -p utils/generate_types/test/tsconfig.json && npm run typecheck-tests",
     "generate-types": "node utils/generate_types/",
@@ -30,7 +30,8 @@
     "roll-browser": "node utils/roll_browser.js",
     "coverage": "node test/checkCoverage.js",
     "check-deps": "node utils/check_deps.js",
-    "build-driver": "pkg --public --targets node12-linux-x64,node12-macos-x64,node12-win-x64 --out-path=drivers packages/playwright-driver/main.js"
+    "build-driver": "pkg --public --targets node12-linux-x64,node12-macos-x64,node12-win-x64 --out-path=drivers packages/playwright-driver/main.js",
+    "build-testrunner": "tsc -p test-runner"
   },
   "author": {
     "name": "Microsoft Corporation"

test-runner/cli.js

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+module.exports = require('./lib/cli')

test-runner/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "test-runner",
+  "version": "0.0.7",
+  "bin": {
+    "test-runner": "./cli.js"
+  },
+  "main": "./lib/index.js"
+}

test/runner/GoldenUtils.js → test-runner/src/GoldenUtils.js

@@ -42,7 +42,7 @@ const GoldenComparators = {
  * @param {?Object} actualBuffer
  * @param {!Buffer} expectedBuffer
  * @param {!string} mimeType
- * @return {?{diff: (!Object:undefined), errorMessage: (string|undefined)}}
+ * @return {?{diff?: Object, errorMessage?: string}}
  */
 function compareImages(actualBuffer, expectedBuffer, mimeType, config = {}) {
   if (!actualBuffer || !(actualBuffer instanceof Buffer))
@@ -63,7 +63,7 @@ function compareImages(actualBuffer, expectedBuffer, mimeType, config = {}) {
 /**
  * @param {?Object} actual
  * @param {!Buffer} expectedBuffer
- * @return {?{diff: (!Object:undefined), errorMessage: (string|undefined)}}
+ * @return {?{diff?: Object, errorMessage?: string, diffExtension?: string}}
  */
 function compareText(actual, expectedBuffer) {
   if (typeof actual !== 'string')
@@ -85,8 +85,8 @@ function compareText(actual, expectedBuffer) {
 
 /**
  * @param {?Object} actual
- * @param {string} path
- * @return {!{pass: boolean, message: (undefined|string)}}
+ * @param {string} name
+ * @return {!{pass: boolean, message?: string}}
  */
 function compare(actual, name, options) {
   const { relativeTestFile, snapshotDir, outputDir, updateSnapshots } = options;

test-runner/src/builtin.fixtures.ts

@@ -0,0 +1,72 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import os from 'os';
+import path from 'path';
+import {promisify} from 'util';
+import fs from 'fs';
+import rimraf from 'rimraf';
+import {registerFixture} from './fixtures';
+
+
+declare global {
+  type DescribeFunction = ((name: string, inner: () => void) => void) & {
+    fail(condition: boolean): DescribeFunction;
+    skip(condition: boolean): DescribeFunction;
+    slow(): DescribeFunction;
+    repeat(n: number): DescribeFunction;
+  };
+
+  type ItFunction<STATE> = ((name: string, inner: (state: STATE) => Promise<void>) => void) & {
+    fail(condition: boolean): ItFunction<STATE>;
+    skip(condition: boolean): ItFunction<STATE>;
+    slow(): ItFunction<STATE>;
+    repeat(n: number): ItFunction<STATE>;
+  };
+
+  const describe: DescribeFunction;
+  const fdescribe: DescribeFunction;
+  const xdescribe: DescribeFunction;
+  const it: ItFunction<TestState & WorkerState & FixtureParameters>;
+  const fit: ItFunction<TestState & WorkerState & FixtureParameters>;
+  const dit: ItFunction<TestState & WorkerState & FixtureParameters>;
+  const xit: ItFunction<TestState & WorkerState & FixtureParameters>;
+
+  const beforeEach: (inner: (state: TestState & WorkerState & FixtureParameters) => Promise<void>) => void;
+  const afterEach: (inner: (state: TestState & WorkerState & FixtureParameters) => Promise<void>) => void;
+  const beforeAll: (inner: (state: WorkerState & FixtureParameters) => Promise<void>) => void;
+  const afterAll: (inner: (state: WorkerState & FixtureParameters) => Promise<void>) => void;
+}
+
+const mkdtempAsync = promisify(fs.mkdtemp);
+const removeFolderAsync = promisify(rimraf);
+
+declare global {
+  interface FixtureParameters {
+    parallelIndex: number;
+  }
+  interface TestState {
+    tmpDir: string;
+  }
+}
+
+export {parameters, registerFixture, registerWorkerFixture, registerParameter} from './fixtures';
+
+registerFixture('tmpDir', async ({}, test) => {
+  const tmpDir = await mkdtempAsync(path.join(os.tmpdir(), 'playwright-test-'));
+  await test(tmpDir);
+  await removeFolderAsync(tmpDir).catch(e => {});
+});

test/runner/cli.js → test-runner/src/cli.js

@@ -25,21 +25,21 @@ let beforeFunction;
 let afterFunction;
 let matrix = {};
 
-global.before = (fn => beforeFunction = fn);
-global.after = (fn => afterFunction = fn);
-global.matrix = (m => matrix = m);
+global['before'] = (fn => beforeFunction = fn);
+global['after'] = (fn => afterFunction = fn);
+global['matrix'] = (m => matrix = m);
 
 program
-  .version('Version ' + require('../../package.json').version)
+  .version('Version ' + /** @type {any} */ (require)('../package.json').version)
   .option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
   .option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
-  .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2))
+  .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2).toString())
   .option('--reporter <reporter>', 'Specify reporter to use', '')
   .option('--trial-run', 'Only collect the matching tests and report them as passing')
   .option('--quiet', 'Suppress stdio', false)
   .option('--debug', 'Run tests in-process for debugging', false)
   .option('--output <outputDir>', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results'))
-  .option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', 10000)
+  .option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000')
   .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run')
   .action(async (command) => {
     // Collect files]
@@ -101,7 +101,7 @@ program
     try {
       if (beforeFunction)
         await beforeFunction();
-      await runner.run(files);
+      await runner.run();
       await runner.stop();
     } finally {
       if (afterFunction)

test/runner/dotReporter.js → test-runner/src/dotReporter.js

@@ -69,7 +69,7 @@ class DotReporter extends Base {
       this.failures.forEach((failure, index) => {
         const relativePath = path.relative(process.cwd(), failure.file);
         const header = `  ${index +1}. ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`;
-        console.log(colors.bold.red(header));
+        console.log(colors.bold(colors.red(header)));
         const stack = failure.err.stack;
         if (stack) {
           console.log('');