test: remove output and golden directory notions (#3456)

test/base.fixture.ts

@@ -16,6 +16,7 @@
 
 import fs from 'fs';
 import path from 'path';
+import os from 'os';
 import childProcess from 'child_process';
 import { LaunchOptions, BrowserType, Browser, BrowserContext, Page, BrowserServer } from '../index';
 import { TestServer } from '../utils/testserver/';
@@ -24,20 +25,24 @@ import { Transport } from '../lib/rpc/transport';
 import { setUnderTest } from '../lib/helper';
 import { installCoverageHooks } from './runner/coverage';
 import { valueFromEnv } from './runner/utils';
+import { registerFixture, registerWorkerFixture} from './runner/fixtures';
+
+import {mkdtempAsync, removeFolderAsync} from './utils';
 
 setUnderTest(); // Note: we must call setUnderTest before requiring Playwright
 
 const browserName = process.env.BROWSER || 'chromium';
 
 declare global {
   interface WorkerState {
+    asset: (path: string) => string;
     parallelIndex: number;
-    http_server: {server: TestServer, httpsServer: TestServer};
     defaultBrowserOptions: LaunchOptions;
+    golden: (path: string) => string;
     playwright: typeof import('../index');
     browserType: BrowserType<Browser>;
     browser: Browser;
-    outputDir: string;
+    tmpDir: string;
   }
   interface FixtureState {
     toImpl: (rpcObject: any) => any;
@@ -53,7 +58,7 @@ registerWorkerFixture('parallelIndex', async ({}, test) => {
   await test(parseInt(process.env.JEST_WORKER_ID, 10) - 1);
 });
 
-registerWorkerFixture('http_server', async ({parallelIndex}, test) => {
+registerWorkerFixture('httpService', async ({parallelIndex}, test) => {
   const assetsPath = path.join(__dirname, 'assets');
   const cachedPath = path.join(__dirname, 'assets', 'cached');
 
@@ -158,21 +163,30 @@ registerFixture('page', async ({context}, test) => {
   await test(page);
 });
 
-registerFixture('server', async ({http_server}, test) => {
-  http_server.server.reset();
-  await test(http_server.server);
+registerFixture('server', async ({httpService}, test) => {
+  httpService.server.reset();
+  await test(httpService.server);
 });
 
-registerFixture('httpsServer', async ({http_server}, test) => {
-  http_server.httpsServer.reset();
-  await test(http_server.httpsServer);
+registerFixture('browserName', async ({}, test) => {
+  await test(browserName);
 });
 
-registerWorkerFixture('outputDir', async ({}, test) => {
-  const outputDir = path.join(__dirname, 'output-' + browserName);
-  try {
-    await fs.promises.mkdir(outputDir, { recursive: true });
-  } catch (e) {
-  }
-  await test(outputDir);
+registerFixture('httpsServer', async ({httpService}, test) => {
+  httpService.httpsServer.reset();
+  await test(httpService.httpsServer);
+});
+
+registerFixture('tmpDir', async ({}, test) => {
+  const tmpDir = await mkdtempAsync(path.join(os.tmpdir(), 'playwright-test-'));
+  await test(tmpDir);
+  await removeFolderAsync(tmpDir);
+});
+
+registerWorkerFixture('asset', async ({}, test) => {
+  await test(p => path.join(__dirname, `assets`, p));
+});
+
+registerWorkerFixture('golden', async ({browserName}, test) => {
+  await test(p => path.join(__dirname, `golden-${browserName}`, p));
 });

test/capabilities.spec.ts

@@ -18,7 +18,7 @@ import './base.fixture';
 import path from 'path';
 import url from 'url';
 
-const {FFOX, CHROMIUM, WEBKIT, WIN, LINUX, ASSETS_DIR} = testOptions;
+const {FFOX, CHROMIUM, WEBKIT, WIN, LINUX} = testOptions;
 
 it.fail(WEBKIT && WIN)('Web Assembly should work', async function({page, server}) {
   await page.goto(server.PREFIX + '/wasm/table2.html');
@@ -51,14 +51,14 @@ it('should respect CSP', async({page, server}) => {
   expect(await page.evaluate(() => window['testStatus'])).toBe('SUCCESS');
 });
 
-it.fail(WEBKIT && (WIN || LINUX))('should play video', async({page}) => {
+it.fail(WEBKIT && (WIN || LINUX))('should play video', async({page, asset}) => {
   // TODO: the test passes on Windows locally but fails on GitHub Action bot,
   // apparently due to a Media Pack issue in the Windows Server.
   // Also the test is very flaky on Linux WebKit.
   //
   // Safari only plays mp4 so we test WebKit with an .mp4 clip.
   const fileName = WEBKIT ? 'video_mp4.html' : 'video.html';
-  const absolutePath = path.join(ASSETS_DIR, fileName);
+  const absolutePath = asset(fileName);
   // Our test server doesn't support range requests required to play on Mac,
   // so we load the page using a file url.
   await page.goto(url.pathToFileURL(absolutePath).href);

test/chromium/oopif.spec.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 import '../base.fixture';
+import { registerFixture } from '../runner/fixtures';
 import { Page, Browser, BrowserContext } from '../..';
 
 const {FFOX, CHROMIUM, WEBKIT} = testOptions;
@@ -188,14 +189,14 @@ it.skip(!CHROMIUM)('should respect route', async({sppBrowser, sppPage, server})
   expect(intercepted).toBe(true);
 });
 
-it.skip(!CHROMIUM)('should take screenshot', async({sppBrowser, sppPage, server}) => {
+it.skip(!CHROMIUM)('should take screenshot', async({sppBrowser, sppPage, server, golden}) => {
   const browser = sppBrowser;
   const page = sppPage;
   await page.setViewportSize({width: 500, height: 500});
   await page.goto(server.PREFIX + '/dynamic-oopif.html');
   expect(page.frames().length).toBe(2);
   expect(await countOOPIFs(browser)).toBe(1);
-  expect(await page.screenshot()).toBeGolden('screenshot-oopif.png');
+  expect(await page.screenshot()).toMatchImage(golden('screenshot-oopif.png'));
 });
 
 it.skip(!CHROMIUM)('should load oopif iframes with subresources and request interception', async function({sppBrowser, sppPage, server, context}) {

test/chromium/tracing.spec.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 import '../base.fixture';
+import { registerFixture } from '../runner/fixtures';
 
 import fs from 'fs';
 import path from 'path';
@@ -24,8 +25,8 @@ declare global {
     outputFile: string;
   }
 }
-registerFixture('outputFile', async ({outputDir, parallelIndex}, test) => {
-  const outputFile = path.join(outputDir, `trace-${parallelIndex}.json`);
+registerFixture('outputFile', async ({tmpDir}, test) => {
+  const outputFile = path.join(tmpDir, `trace.json`);
   await test(outputFile);
   if (fs.existsSync(outputFile))
     fs.unlinkSync(outputFile);

test/defaultbrowsercontext.spec.ts

@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 import './base.fixture';
+import { registerFixture } from './runner/fixtures';
 
 import fs from 'fs';
 import path from 'path';

test/download.spec.ts

@@ -18,22 +18,9 @@ import './base.fixture';
 import fs from 'fs';
 import path from 'path';
 import util from 'util';
-import os from 'os';
-import {mkdtempAsync, removeFolderAsync} from './utils';
 
 const {FFOX, CHROMIUM, WEBKIT, HEADLESS} = testOptions;
 
-declare global {
-  interface FixtureState {
-    persistentDirectory: string;
-  }
-}
-registerFixture('persistentDirectory', async ({}, test) => {
-  const persistentDirectory = await mkdtempAsync(path.join(os.tmpdir(), 'playwright-test-'));
-  await test(persistentDirectory);
-  await removeFolderAsync(persistentDirectory);
-});
-
 beforeEach(async ({server}) => {
   server.setRoute('/download', (req, res) => {
     res.setHeader('Content-Type', 'application/octet-stream');
@@ -74,28 +61,28 @@ it('should report downloads with acceptDownloads: true', async({browser, server}
   await page.close();
 });
 
-it('should save to user-specified path', async({persistentDirectory, browser, server}) => {
+it('should save to user-specified path', async({tmpDir, browser, server}) => {
   const page = await browser.newPage({ acceptDownloads: true });
   await page.setContent(`<a href="${server.PREFIX}/download">download</a>`);
   const [ download ] = await Promise.all([
     page.waitForEvent('download'),
     page.click('a')
   ]);
-  const userPath = path.join(persistentDirectory, "download.txt");
+  const userPath = path.join(tmpDir, "download.txt");
   await download.saveAs(userPath);
   expect(fs.existsSync(userPath)).toBeTruthy();
   expect(fs.readFileSync(userPath).toString()).toBe('Hello world');
   await page.close();
 });
 
-it('should save to user-specified path without updating original path', async({persistentDirectory, browser, server}) => {
+it('should save to user-specified path without updating original path', async({tmpDir, browser, server}) => {
   const page = await browser.newPage({ acceptDownloads: true });
   await page.setContent(`<a href="${server.PREFIX}/download">download</a>`);
   const [ download ] = await Promise.all([
     page.waitForEvent('download'),
     page.click('a')
   ]);
-  const userPath = path.join(persistentDirectory, "download.txt");
+  const userPath = path.join(tmpDir, "download.txt");
   await download.saveAs(userPath);
   expect(fs.existsSync(userPath)).toBeTruthy();
   expect(fs.readFileSync(userPath).toString()).toBe('Hello world');
@@ -106,77 +93,77 @@ it('should save to user-specified path without updating original path', async({p
   await page.close();
 });
 
-it('should save to two different paths with multiple saveAs calls', async({persistentDirectory, browser, server}) => {
+it('should save to two different paths with multiple saveAs calls', async({tmpDir, browser, server}) => {
   const page = await browser.newPage({ acceptDownloads: true });
   await page.setContent(`<a href="${server.PREFIX}/download">download</a>`);
   const [ download ] = await Promise.all([
     page.waitForEvent('download'),
     page.click('a')
   ]);
-  const userPath = path.join(persistentDirectory, "download.txt");
+  const userPath = path.join(tmpDir, "download.txt");
   await download.saveAs(userPath);
   expect(fs.existsSync(userPath)).toBeTruthy();
   expect(fs.readFileSync(userPath).toString()).toBe('Hello world');
 
-  const anotherUserPath = path.join(persistentDirectory, "download (2).txt");
+  const anotherUserPath = path.join(tmpDir, "download (2).txt");
   await download.saveAs(anotherUserPath);
   expect(fs.existsSync(anotherUserPath)).toBeTruthy();
   expect(fs.readFileSync(anotherUserPath).toString()).toBe('Hello world');
   await page.close();
 });
 
-it('should save to overwritten filepath', async({persistentDirectory, browser, server}) => {
+it('should save to overwritten filepath', async({tmpDir, browser, server}) => {
   const page = await browser.newPage({ acceptDownloads: true });
   await page.setContent(`<a href="${server.PREFIX}/download">download</a>`);
   const [ download ] = await Promise.all([
     page.waitForEvent('download'),
     page.click('a')
   ]);
-  const userPath = path.join(persistentDirectory, "download.txt");
+  const userPath = path.join(tmpDir, "download.txt");
   await download.saveAs(userPath);
-  expect((await util.promisify(fs.readdir)(persistentDirectory)).length).toBe(1);
+  expect((await util.promisify(fs.readdir)(tmpDir)).length).toBe(1);
   await download.saveAs(userPath);
-  expect((await util.promisify(fs.readdir)(persistentDirectory)).length).toBe(1);
+  expect((await util.promisify(fs.readdir)(tmpDir)).length).toBe(1);
   expect(fs.existsSync(userPath)).toBeTruthy();
   expect(fs.readFileSync(userPath).toString()).toBe('Hello world');
   await page.close();
 });
 
-it('should create subdirectories when saving to non-existent user-specified path', async({persistentDirectory, browser, server}) => {
+it('should create subdirectories when saving to non-existent user-specified path', async({tmpDir, browser, server}) => {
   const page = await browser.newPage({ acceptDownloads: true });
   await page.setContent(`<a href="${server.PREFIX}/download">download</a>`);
   const [ download ] = await Promise.all([
     page.waitForEvent('download'),
     page.click('a')
   ]);
-  const nestedPath = path.join(persistentDirectory, "these", "are", "directories", "download.txt");
+  const nestedPath = path.join(tmpDir, "these", "are", "directories", "download.txt");
   await download.saveAs(nestedPath)
   expect(fs.existsSync(nestedPath)).toBeTruthy();
   expect(fs.readFileSync(nestedPath).toString()).toBe('Hello world');
   await page.close();
 });
 
-it('should error when saving with downloads disabled', async({persistentDirectory, browser, server}) => {
+it('should error when saving with downloads disabled', async({tmpDir, browser, server}) => {
   const page = await browser.newPage({ acceptDownloads: false });
   await page.setContent(`<a href="${server.PREFIX}/download">download</a>`);
   const [ download ] = await Promise.all([
     page.waitForEvent('download'),
     page.click('a')
   ]);
-  const userPath = path.join(persistentDirectory, "download.txt");
+  const userPath = path.join(tmpDir, "download.txt");
   const { message } = await download.saveAs(userPath).catch(e => e);
   expect(message).toContain('Pass { acceptDownloads: true } when you are creating your browser context');
   await page.close();
 });
 
-it('should error when saving after deletion', async({persistentDirectory, browser, server}) => {
+it('should error when saving after deletion', async({tmpDir, browser, server}) => {
   const page = await browser.newPage({ acceptDownloads: true });
   await page.setContent(`<a href="${server.PREFIX}/download">download</a>`);
   const [ download ] = await Promise.all([
     page.waitForEvent('download'),
     page.click('a')
   ]);
-  const userPath = path.join(persistentDirectory, "download.txt");
+  const userPath = path.join(tmpDir, "download.txt");
   await download.delete();
   const { message } = await download.saveAs(userPath).catch(e => e);
   expect(message).toContain('Download already deleted. Save before deleting.');

test/downloads-path.spec.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 import './base.fixture';
+import { registerFixture } from './runner/fixtures';
 
 import path from 'path';
 import fs from 'fs';

test/electron/electron.fixture.ts

@@ -1,4 +1,21 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 import '../base.fixture';
+import { registerFixture } from '../runner/fixtures';
 import {ElectronApplication, ElectronLauncher, ElectronPage} from '../../electron-types';
 import path from 'path';