diff --git a/.wp-env.json b/.wp-env.json
index aa8dfaf3c0c4ab..79810b194b667c 100644
--- a/.wp-env.json
+++ b/.wp-env.json
@@ -8,7 +8,9 @@
 				"wp-content/plugins/gutenberg": ".",
 				"wp-content/mu-plugins": "./packages/e2e-tests/mu-plugins",
 				"wp-content/plugins/gutenberg-test-plugins": "./packages/e2e-tests/plugins",
-				"wp-content/themes/gutenberg-test-themes": "./test/gutenberg-test-themes"
+				"wp-content/themes/gutenberg-test-themes": "./test/gutenberg-test-themes",
+				"wp-content/themes/gutenberg-test-themes/twentytwentyone": "https://downloads.wordpress.org/theme/twentytwentyone.1.7.zip",
+				"wp-content/themes/gutenberg-test-themes/twentytwentythree": "https://downloads.wordpress.org/theme/twentytwentythree.1.0.zip"
 			}
 		}
 	}
diff --git a/bin/plugin/commands/performance.js b/bin/plugin/commands/performance.js
index 57c6674500a221..95e74c851c1d6b 100644
--- a/bin/plugin/commands/performance.js
+++ b/bin/plugin/commands/performance.js
@@ -337,8 +337,6 @@ async function runPerformanceTests( branches, options ) {
 							performanceTestDirectory,
 							'test/emptytheme'
 						),
-						'https://downloads.wordpress.org/theme/twentytwentyone.1.7.zip',
-						'https://downloads.wordpress.org/theme/twentytwentythree.1.0.zip',
 					],
 					env: {
 						tests: {
@@ -352,6 +350,15 @@ async function runPerformanceTests( branches, options ) {
 										performanceTestDirectory,
 										'packages/e2e-tests/plugins'
 									),
+								'wp-content/themes/gutenberg-test-themes':
+									path.join(
+										performanceTestDirectory,
+										'test/gutenberg-test-themes'
+									),
+								'wp-content/themes/gutenberg-test-themes/twentytwentyone':
+									'https://downloads.wordpress.org/theme/twentytwentyone.1.7.zip',
+								'wp-content/themes/gutenberg-test-themes/twentytwentythree':
+									'https://downloads.wordpress.org/theme/twentytwentythree.1.0.zip',
 							},
 						},
 					},
diff --git a/packages/e2e-tests/specs/performance/front-end-block-theme.test.js b/packages/e2e-tests/specs/performance/front-end-block-theme.test.js
index 260a20ce64c4df..d1036b1fdc2021 100644
--- a/packages/e2e-tests/specs/performance/front-end-block-theme.test.js
+++ b/packages/e2e-tests/specs/performance/front-end-block-theme.test.js
@@ -16,13 +16,12 @@ describe( 'Front End Performance', () => {
 	};
 
 	beforeAll( async () => {
-		await activateTheme( 'twentytwentythree' );
+		await activateTheme( 'gutenberg-test-themes/twentytwentythree' );
 		await logout();
 	} );
 
 	afterAll( async () => {
 		saveResultsFile( __filename, results );
-		await activateTheme( 'twentytwentyone' );
 	} );
 
 	it( 'Report TTFB, LCP, and LCP-TTFB', async () => {
diff --git a/packages/e2e-tests/specs/performance/front-end-classic-theme.test.js b/packages/e2e-tests/specs/performance/front-end-classic-theme.test.js
index ffe6906a529d8c..d687f7efdf80ff 100644
--- a/packages/e2e-tests/specs/performance/front-end-classic-theme.test.js
+++ b/packages/e2e-tests/specs/performance/front-end-classic-theme.test.js
@@ -16,7 +16,7 @@ describe( 'Front End Performance', () => {
 	};
 
 	beforeAll( async () => {
-		await activateTheme( 'twentytwentyone' );
+		await activateTheme( 'gutenberg-test-themes/twentytwentyone' );
 		await logout();
 	} );
 
diff --git a/packages/e2e-tests/specs/performance/post-editor.test.js b/packages/e2e-tests/specs/performance/post-editor.test.js
index ee77f122a42bd4..9394d44619890b 100644
--- a/packages/e2e-tests/specs/performance/post-editor.test.js
+++ b/packages/e2e-tests/specs/performance/post-editor.test.js
@@ -7,6 +7,7 @@ import path from 'path';
  * WordPress dependencies
  */
 import {
+	activateTheme,
 	createNewPost,
 	saveDraft,
 	insertBlock,
@@ -83,6 +84,11 @@ describe( 'Post Editor Performance', () => {
 
 	let traceResults;
 
+	beforeAll( async () => {
+		// See https://github.com/WordPress/gutenberg/pull/50905/files#r1209014677;
+		await activateTheme( 'gutenberg-test-themes/twentytwentyone' );
+	} );
+
 	afterAll( async () => {
 		saveResultsFile( __filename, results );
 		deleteFile( traceFilePath );
diff --git a/packages/e2e-tests/specs/performance/site-editor.test.js b/packages/e2e-tests/specs/performance/site-editor.test.js
index c141c2dbf58672..a1a2fa38e73cb7 100644
--- a/packages/e2e-tests/specs/performance/site-editor.test.js
+++ b/packages/e2e-tests/specs/performance/site-editor.test.js
@@ -87,7 +87,6 @@ describe( 'Site Editor Performance', () => {
 		saveResultsFile( __filename, results );
 		await deleteAllTemplates( 'wp_template' );
 		await deleteAllTemplates( 'wp_template_part' );
-		await activateTheme( 'twentytwentyone' );
 	} );
 
 	// Number of loading measurements to take.