test runner: remove dependencies on playwright (#3539)

.github/workflows/auto_roll.yml

@@ -27,7 +27,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner test/ --jobs=1 --forbid-only --timeout=30000"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"

.github/workflows/tests.yml

@@ -37,7 +37,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner test/ --jobs=1 --forbid-only --timeout=30000 && npm run coverage"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000 && npm run coverage"
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
@@ -68,7 +68,7 @@ jobs:
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
     - run: npm run build
-    - run: node test/runner test/ --jobs=1 --forbid-only --timeout=30000
+    - run: node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
@@ -102,7 +102,7 @@ jobs:
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
     - run: npm run build
-    - run: node test/runner test/ --jobs=1 --forbid-only --timeout=30000
+    - run: node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000
       shell: bash
       env:
         BROWSER: ${{ matrix.browser }}
@@ -159,7 +159,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner test/ --jobs=1 --forbid-only --timeout=30000"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
       if: ${{ always() }}
       env:
         BROWSER: ${{ matrix.browser }}
@@ -197,7 +197,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner test/ --jobs=1 --forbid-only --timeout=30000"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test/runner/cli test/ --jobs=1 --forbid-only --timeout=30000"
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"

package.json

@@ -9,10 +9,10 @@
     "node": ">=10.15.0"
   },
   "scripts": {
-    "ctest": "cross-env BROWSER=chromium node test/runner test/",
-    "ftest": "cross-env BROWSER=firefox node test/runner test/",
-    "wtest": "cross-env BROWSER=webkit node test/runner test/",
-    "test": "npm run ctest && npm run ftest && npm run wtest",
+    "ctest": "cross-env BROWSER=chromium node test/runner/cli test/",
+    "ftest": "cross-env BROWSER=firefox node test/runner/cli test/",
+    "wtest": "cross-env BROWSER=webkit node test/runner/cli test/",
+    "test": "node test/runner/cli test/",
     "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts ./src || eslint --ext js,ts ./src",
     "tsc": "tsc -p .",
     "tsc-installer": "tsc -p ./src/install/tsconfig.json",
@@ -28,7 +28,7 @@
     "generate-channels": "node utils/generate_channels.js",
     "typecheck-tests": "tsc -p ./test/",
     "roll-browser": "node utils/roll_browser.js",
-    "coverage": "node test/runner/checkCoverage.js",
+    "coverage": "node test/checkCoverage.js",
     "check-deps": "node utils/check_deps.js",
     "build-driver": "pkg --public --targets node12-linux-x64,node12-macos-x64,node12-win-x64 --out-path=drivers packages/playwright-driver/main.js"
   },

test/base.fixture.ts

@@ -23,9 +23,8 @@ import { TestServer } from '../utils/testserver/';
 import { Connection } from '../lib/rpc/client/connection';
 import { Transport } from '../lib/rpc/transport';
 import { setUnderTest } from '../lib/helper';
-import { installCoverageHooks } from './runner/coverage';
-import { valueFromEnv } from './runner/utils';
-import { registerFixture, registerWorkerFixture, registerOption, registerOptionGenerator } from './runner/fixtures';
+import { installCoverageHooks } from './coverage';
+import { registerFixture, registerWorkerFixture, registerOption, registerOptionGenerator } from './runner';
 import './runner/builtin.fixtures';
 
 import {mkdtempAsync, removeFolderAsync} from './utils';
@@ -66,11 +65,11 @@ declare global {
 (global as any).LINUX = platform === 'linux';
 (global as any).WIN = platform === 'win32';
 
-registerWorkerFixture('httpService', async ({parallelIndex}, test) => {
+registerWorkerFixture('httpService', async ({}, test) => {
   const assetsPath = path.join(__dirname, 'assets');
   const cachedPath = path.join(__dirname, 'assets', 'cached');
 
-  const port = 8907 + parallelIndex * 2;
+  const port = 8907 + options.parallelIndex * 2;
   const server = await TestServer.create(assetsPath, port);
   server.enableHTTPCache(cachedPath);
 
@@ -108,7 +107,7 @@ registerWorkerFixture('defaultBrowserOptions', async({browserName}, test) => {
   });
 });
 
-registerWorkerFixture('playwright', async({parallelIndex, browserName}, test) => {
+registerWorkerFixture('playwright', async({browserName}, test) => {
   const {coverage, uninstall} = installCoverageHooks(browserName);
   if (options.WIRE) {
     const connection = new Connection();
@@ -138,7 +137,7 @@ registerWorkerFixture('playwright', async({parallelIndex, browserName}, test) =>
 
   async function teardownCoverage() {
     uninstall();
-    const coveragePath = path.join(path.join(__dirname, 'coverage-report'), 'coverage', parallelIndex + '.json');
+    const coveragePath = path.join(__dirname, 'coverage-report', options.parallelIndex + '.json');
     const coverageJSON = [...coverage.keys()].filter(key => coverage.get(key));
     await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true });
     await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8');
@@ -211,3 +210,9 @@ registerOption('FFOX', ({browserName}) => browserName === 'firefox');
 registerOption('WEBKIT', ({browserName}) => browserName === 'webkit');
 registerOption('HEADLESS', ({}) => !!valueFromEnv('HEADLESS', true));
 registerOption('WIRE', ({}) => process.env.PWWIRE);
+
+function valueFromEnv(name, defaultValue) {
+  if (!(name in process.env))
+    return defaultValue;
+  return JSON.parse(process.env[name]);
+}

test/runner/checkCoverage.js → test/checkCoverage.js

@@ -42,7 +42,7 @@ if (browserName === 'webkit')
 api.delete('browserContext.emit("screencaststarted")');
 api.delete('browserContext.emit("screencaststopped")');
 
-const coverageDir = path.join(__dirname, '..', 'coverage-report', 'coverage');
+const coverageDir = path.join(__dirname, 'coverage-report');
 
 const coveredMethods = new Set();
 for (const file of getCoverageFiles(coverageDir)) {

test/chromium/oopif.spec.ts

@@ -15,7 +15,7 @@
  */
 
 import '../base.fixture';
-import { registerWorkerFixture } from '../runner/fixtures';
+import { registerWorkerFixture } from '../runner';
 
 registerWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, test) => {
   const browser = await browserType.launch({

test/chromium/tracing.spec.ts

@@ -16,7 +16,7 @@
 
 import '../runner/builtin.fixtures';
 import '../base.fixture';
-import { registerFixture } from '../runner/fixtures';
+import { registerFixture } from '../runner';
 
 import fs from 'fs';
 import path from 'path';

test/runner/coverage.js → test/coverage.js

@@ -64,25 +64,25 @@ function apiForBrowser(browserName) {
   const BROWSER_CONFIGS = [
     {
       name: 'Firefox',
-      events: require('../../lib/events').Events,
+      events: require('../lib/events').Events,
     },
     {
       name: 'WebKit',
-      events: require('../../lib/events').Events,
+      events: require('../lib/events').Events,
     },
     {
       name: 'Chromium',
       events: {
-        ...require('../../lib/events').Events,
-        ...require('../../lib/chromium/events').Events,
+        ...require('../lib/events').Events,
+        ...require('../lib/chromium/events').Events,
       }
     },
   ];
   const browserConfig = BROWSER_CONFIGS.find(config => config.name.toLowerCase() === browserName);
   const events = browserConfig.events;
   // TODO: we should rethink our api.ts approach to ensure coverage and async stacks.
   const api = {
-    ...require('../../lib/rpc/client/api'),
+    ...require('../lib/rpc/client/api'),
   };
 
   const filteredKeys = Object.keys(api).filter(apiName => {

test/defaultbrowsercontext.spec.ts

@@ -17,7 +17,7 @@
 
 import './runner/builtin.fixtures';
 import './base.fixture';
-import { registerFixture } from './runner/fixtures';
+import { registerFixture } from './runner';
 import fs from 'fs';
 import utils from './utils';
 import { BrowserType, Browser, BrowserContext, Page } from '..';

test/downloads-path.spec.ts

@@ -18,7 +18,7 @@
 import './runner/builtin.fixtures';
 import './base.fixture';
 
-import { registerFixture } from './runner/fixtures';
+import { registerFixture } from './runner';
 
 import path from 'path';
 import fs from 'fs';

test/electron/electron.fixture.ts

@@ -15,7 +15,7 @@
  */
 
 import '../base.fixture';
-import { registerFixture } from '../runner/fixtures';
+import { registerFixture } from '../runner';
 import {ElectronApplication, ElectronLauncher, ElectronPage} from '../../electron-types';
 import path from 'path';
 

test/fixtures.spec.ts

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 import './base.fixture';
-import { registerFixture } from './runner/fixtures';
+import { registerFixture } from './runner';
 
 import path from 'path';
 import {spawn, execSync} from 'child_process';

test/proxy.spec.ts

@@ -97,7 +97,7 @@ it.fail(options.CHROMIUM && !options.HEADLESS)('should exclude patterns', async
   await browser.close();
 });
 
-it('should use socks proxy', async ({ browserType, defaultBrowserOptions, parallelIndex }) => {
+it('should use socks proxy', async ({ browserType, defaultBrowserOptions }) => {
   const server = socks.createServer((info, accept, deny) => {
     let socket;
     if (socket = accept(true)) {
@@ -114,7 +114,7 @@ it('should use socks proxy', async ({ browserType, defaultBrowserOptions, parall
       ].join('\r\n'));
     }
   });
-  const socksPort = 9107 + parallelIndex * 2;
+  const socksPort = 9107 + options.parallelIndex * 2;
   server.listen(socksPort, 'localhost');
   server.useAuth(socks.auth.None());
 

test/runner/utils.js → test/runner/builtin.fixtures.ts

@@ -14,10 +14,23 @@
  * limitations under the License.
  */
 
-function valueFromEnv(name, defaultValue) {
-  if (!(name in process.env))
-    return defaultValue;
-  return JSON.parse(process.env[name]);
+import os from 'os';
+import path from 'path';
+import { mkdtempAsync, removeFolderAsync } from '../utils';
+import { registerFixture } from '.';
+
+declare global {
+  interface Options {
+    parallelIndex: number;
+  }
+
+  interface WorkerState {
+    tmpDir: string;
+  }
 }
 
-module.exports = { valueFromEnv };
+registerFixture('tmpDir', async ({}, test) => {
+  const tmpDir = await mkdtempAsync(path.join(os.tmpdir(), 'playwright-test-'));
+  await test(tmpDir);
+  await removeFolderAsync(tmpDir).catch(e => {});
+});