[Experiment] Capture screencasts for failing tests

.github/workflows/end2end-test.yml

@@ -47,11 +47,12 @@ jobs:
             - name: Running the tests
               run: |
                   $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
-                  $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == ${{ matrix.part }} - 1' < ~/.jest-e2e-tests )
+                  $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --puppeteer-screencasts --runTestsByPath $( awk 'NR % 4 == ${{ matrix.part }} - 1' < ~/.jest-e2e-tests )
 
-            - name: Archive debug artifacts (screenshots, HTML snapshots)
+            - name: Archive debug artifacts (screenshots, HTML snapshots, screencasts)
               uses: actions/upload-artifact@e448a9b857ee2131e752b06002bf0e093c65e571 # v2.2.2
-              if: always()
+              if: failure()
               with:
                   name: failures-artifacts
                   path: artifacts
+                  if-no-files-found: ignore

docs/contributors/code/testing-overview.md

@@ -464,6 +464,22 @@ OFFLINE=true npm run test-e2e
 
 See [Chrome docs: emulateNetworkConditions](https://chromedevtools.github.io/devtools-protocol/tot/Network#method-emulateNetworkConditions)
 
+### Capturing debugging screencasts.
+
+Sometimes it's hard to debug a flaky test, especially when it can only be reproduced in a certain environment. We can append the `--puppeteer-screencasts` flag to record the failing tests into `.webm` videos to help us debugging. It's automatically enabled in GitHub CI.
+
+```bash
+npm run test-e2e -- --puppeteer-screencasts
+```
+
+By default, it will only record failing tests. If you want to record all tests, then specify `always` as the value.
+
+```bash
+npm run test-e2e -- --puppeteer-screencasts=always
+```
+
+Note that recording screencasts will inevitably slow down the tests and consume disk size. Even though it's not a huge overhead, you probably won't want to record it for every test during local development.
+
 ### Core Block Testing
 
 Every core block is required to have at least one set of fixture files for its main save function and one for each deprecation. These fixtures test the parsing and serialization of the block. See [the e2e tests fixtures readme](https://github.com/wordpress/gutenberg/blob/HEAD/packages/e2e-tests/fixtures/blocks/README.md) for more information and instructions.

packages/e2e-tests/specs/widgets/customizing-widgets.test.js

@@ -36,6 +36,20 @@ describe( 'Widgets Customizer', () => {
 					.__unstableToggleFeature( 'welcomeGuide' )
 			);
 		}
+
+		const widgetsPanel = await find( {
+			role: 'heading',
+			name: /Widgets/,
+			level: 3,
+		} );
+		await widgetsPanel.click();
+
+		const footer1Section = await find( {
+			role: 'heading',
+			name: /Footer #1/,
+			level: 3,
+		} );
+		await footer1Section.click();
 	} );
 
 	beforeAll( async () => {
@@ -56,20 +70,6 @@ describe( 'Widgets Customizer', () => {
 	} );
 
 	it( 'should add blocks', async () => {
-		const widgetsPanel = await find( {
-			role: 'heading',
-			name: /Widgets/,
-			level: 3,
-		} );
-		await widgetsPanel.click();
-
-		const footer1Section = await find( {
-			role: 'heading',
-			name: /Footer #1/,
-			level: 3,
-		} );
-		await footer1Section.click();
-
 		await addBlock( 'Paragraph' );
 		await page.keyboard.type( 'First Paragraph' );
 
@@ -147,20 +147,6 @@ describe( 'Widgets Customizer', () => {
 	} );
 
 	it( 'should open the inspector panel', async () => {
-		const widgetsPanel = await find( {
-			role: 'heading',
-			name: /Widgets/,
-			level: 3,
-		} );
-		await widgetsPanel.click();
-
-		const footer1Section = await find( {
-			role: 'heading',
-			name: /Footer #1/,
-			level: 3,
-		} );
-		await footer1Section.click();
-
 		await addBlock( 'Paragraph' );
 		await page.keyboard.type( 'First Paragraph' );
 
@@ -241,20 +227,6 @@ describe( 'Widgets Customizer', () => {
 	} );
 
 	it( 'should handle the inserter outer section', async () => {
-		const widgetsPanel = await find( {
-			role: 'heading',
-			name: /Widgets/,
-			level: 3,
-		} );
-		await widgetsPanel.click();
-
-		const footer1Section = await find( {
-			role: 'heading',
-			name: /^Footer #1/,
-			level: 3,
-		} );
-		await footer1Section.click();
-
 		// We need to make some changes for the publish settings to appear.
 		await addBlock( 'Paragraph' );
 		await page.keyboard.type( 'First Paragraph' );
@@ -348,20 +320,6 @@ describe( 'Widgets Customizer', () => {
 	} );
 
 	it( 'should move focus to the block', async () => {
-		const widgetsPanel = await find( {
-			role: 'heading',
-			name: /Widgets/,
-			level: 3,
-		} );
-		await widgetsPanel.click();
-
-		const footer1Section = await find( {
-			role: 'heading',
-			name: /^Footer #1/,
-			level: 3,
-		} );
-		await footer1Section.click();
-
 		await addBlock( 'Paragraph' );
 		await page.keyboard.type( 'First Paragraph' );
 
@@ -443,20 +401,6 @@ describe( 'Widgets Customizer', () => {
 	} );
 
 	it( 'should clear block selection', async () => {
-		const widgetsPanel = await find( {
-			role: 'heading',
-			name: /Widgets/,
-			level: 3,
-		} );
-		await widgetsPanel.click();
-
-		const footer1Section = await find( {
-			role: 'heading',
-			name: /^Footer #1/,
-			level: 3,
-		} );
-		await footer1Section.click();
-
 		const paragraphBlock = await addBlock( 'Paragraph' );
 		await page.keyboard.type( 'First Paragraph' );
 		await showBlockToolbar();
@@ -505,20 +449,6 @@ describe( 'Widgets Customizer', () => {
 	} );
 
 	it( 'should handle legacy widgets', async () => {
-		const widgetsPanel = await find( {
-			role: 'heading',
-			name: /Widgets/,
-			level: 3,
-		} );
-		await widgetsPanel.click();
-
-		const footer1Section = await find( {
-			role: 'heading',
-			name: /^Footer #1/,
-			level: 3,
-		} );
-		await footer1Section.click();
-
 		const legacyWidgetBlock = await addBlock( 'Legacy Widget' );
 		const selectLegacyWidgets = await find( {
 			role: 'combobox',
@@ -636,20 +566,6 @@ describe( 'Widgets Customizer', () => {
 	} );
 
 	it( 'should handle esc key events', async () => {
-		const widgetsPanel = await find( {
-			role: 'heading',
-			name: /Widgets/,
-			level: 3,
-		} );
-		await widgetsPanel.click();
-
-		const footer1Section = await find( {
-			role: 'heading',
-			name: /^Footer #1/,
-			level: 3,
-		} );
-		await footer1Section.click();
-
 		const paragraphBlock = await addBlock( 'Paragraph' );
 		await page.keyboard.type( 'First Paragraph' );
 		await showBlockToolbar();
@@ -683,20 +599,6 @@ describe( 'Widgets Customizer', () => {
 	} );
 
 	it( 'should move (inner) blocks to another sidebar', async () => {
-		const widgetsPanel = await find( {
-			role: 'heading',
-			name: /Widgets/,
-			level: 3,
-		} );
-		await widgetsPanel.click();
-
-		const footer1Section = await find( {
-			role: 'heading',
-			name: /Footer #1/,
-			level: 3,
-		} );
-		await footer1Section.click();
-
 		await addBlock( 'Paragraph' );
 		await page.keyboard.type( 'First Paragraph' );
 

packages/scripts/config/jest-environment-puppeteer/index.js

@@ -26,6 +26,7 @@ const chalk = require( 'chalk' );
  * Internal dependencies
  */
 const { readConfig, getPuppeteer } = require( './config' );
+const setupScreencast = require( './screencast' );
 
 const handleError = ( error ) => {
 	// To match the same behavior in jest-jasmine2:
@@ -44,6 +45,7 @@ const KEYS = {
 
 const root = process.env.GITHUB_WORKSPACE || process.cwd();
 const ARTIFACTS_PATH = path.join( root, 'artifacts' );
+const PUPPETEER_SCREENCASTS = process.env.PUPPETEER_SCREENCASTS;
 
 class PuppeteerEnvironment extends NodeEnvironment {
 	// Jest is not available here, so we have to reverse engineer
@@ -177,6 +179,13 @@ class PuppeteerEnvironment extends NodeEnvironment {
 				throw err;
 			}
 		}
+
+		if ( PUPPETEER_SCREENCASTS ) {
+			this.screencast = await setupScreencast(
+				this.global.page,
+				this.global.browser
+			);
+		}
 	}
 
 	async teardown() {
@@ -194,16 +203,16 @@ class PuppeteerEnvironment extends NodeEnvironment {
 			await page.close();
 		}
 
+		if ( this.screencast ) {
+			await this.screencast.teardown();
+		}
+
 		if ( browser ) {
 			await browser.disconnect();
 		}
 	}
 
-	async storeArtifacts( testName ) {
-		const datetime = new Date().toISOString().split( '.' )[ 0 ];
-		const fileName = filenamify( `${ testName } ${ datetime }`, {
-			replacement: '-',
-		} );
+	async storeArtifacts( fileName ) {
 		await writeFile(
 			path.join( ARTIFACTS_PATH, `${ fileName }-snapshot.html` ),
 			await this.global.page.content()
@@ -214,11 +223,51 @@ class PuppeteerEnvironment extends NodeEnvironment {
 	}
 
 	async handleTestEvent( event, state ) {
-		if ( event.name === 'test_fn_failure' ) {
+		if ( state.currentlyRunningTest ) {
 			const testName = state.currentlyRunningTest.name;
-			await this.storeArtifacts( testName );
+			const fileName = getFileName( testName );
+
+			if ( this.screencast && event.name === 'test_fn_start' ) {
+				await this.screencast.start().catch( ( err ) => {
+					// Ignore error to prevent it from failing the test,
+					// instead just log it for debugging.
+					// eslint-disable-next-line no-console
+					console.error( err );
+				} );
+			}
+
+			if (
+				this.screencast &&
+				( event.name === 'test_fn_success' ||
+					event.name === 'test_fn_failure' )
+			) {
+				await this.screencast
+					.stop(
+						( event.name === 'test_fn_failure' ||
+							PUPPETEER_SCREENCASTS === 'always' ) &&
+							path.join( ARTIFACTS_PATH, fileName + '.webm' )
+					)
+					.catch( ( err ) => {
+						// Ignore error to prevent it from failing the test,
+						// instead just log it for debugging.
+						// eslint-disable-next-line no-console
+						console.error( err );
+					} );
+			}
+
+			if ( event.name === 'test_fn_failure' ) {
+				await this.storeArtifacts( fileName );
+			}
 		}
 	}
 }
 
+function getFileName( testName ) {
+	const datetime = new Date().toISOString().split( '.' )[ 0 ];
+	const fileName = filenamify( `${ testName } ${ datetime }`, {
+		replacement: '-',
+	} );
+	return fileName;
+}
+
 module.exports = PuppeteerEnvironment;