diff --git a/packages/block-editor/src/components/block-selection-clearer/index.js b/packages/block-editor/src/components/block-selection-clearer/index.js
index 8501d3179eacd..3806dae48d29d 100644
--- a/packages/block-editor/src/components/block-selection-clearer/index.js
+++ b/packages/block-editor/src/components/block-selection-clearer/index.js
@@ -19,14 +19,19 @@ export function useBlockSelectionClearer( ref ) {
 			return;
 		}
 
-		function onFocus() {
+		function onMouseDown( event ) {
+			// Only handle clicks on the canvas, not the content.
+			if ( event.target.closest( '.wp-block' ) ) {
+				return;
+			}
+
 			clearSelectedBlock();
 		}
 
-		ref.current.addEventListener( 'focus', onFocus );
+		ref.current.addEventListener( 'mousedown', onMouseDown );
 
 		return () => {
-			ref.current.removeEventListener( 'focus', onFocus );
+			ref.current.removeEventListener( 'mousedown', onMouseDown );
 		};
 	}, [ hasSelection, clearSelectedBlock ] );
 }
@@ -34,5 +39,5 @@ export function useBlockSelectionClearer( ref ) {
 export default function BlockSelectionClearer( props ) {
 	const ref = useRef();
 	useBlockSelectionClearer( ref );
-	return <div tabIndex={ -1 } ref={ ref } { ...props } />;
+	return <div ref={ ref } { ...props } />;
 }
diff --git a/packages/e2e-tests/specs/editor/various/keyboard-navigable-blocks.test.js b/packages/e2e-tests/specs/editor/various/keyboard-navigable-blocks.test.js
index 730af28bdb10a..2b092c9b46690 100644
--- a/packages/e2e-tests/specs/editor/various/keyboard-navigable-blocks.test.js
+++ b/packages/e2e-tests/specs/editor/various/keyboard-navigable-blocks.test.js
@@ -108,10 +108,10 @@ describe( 'Order of block keyboard navigation', () => {
 			await page.keyboard.type( paragraphBlock );
 		}
 
-		// Clear the selected block and put focus in front of the block list.
-		await page.evaluate( () => {
-			document.querySelector( '.editor-styles-wrapper' ).focus();
-		} );
+		// Clear the selected block.
+		const paragraph = await page.$( '[data-type="core/paragraph"]' );
+		const box = await paragraph.boundingBox();
+		await page.mouse.click( box.x - 1, box.y );
 
 		await page.keyboard.press( 'Tab' );
 		await expect(
@@ -143,9 +143,13 @@ describe( 'Order of block keyboard navigation', () => {
 			await page.keyboard.type( paragraphBlock );
 		}
 
-		// Clear the selected block and put focus behind the block list.
+		// Clear the selected block.
+		const paragraph = await page.$( '[data-type="core/paragraph"]' );
+		const box = await paragraph.boundingBox();
+		await page.mouse.click( box.x - 1, box.y );
+
+		// Put focus behind the block list.
 		await page.evaluate( () => {
-			document.querySelector( '.editor-styles-wrapper' ).focus();
 			document
 				.querySelector( '.interface-interface-skeleton__sidebar' )
 				.focus();
diff --git a/packages/edit-post/src/components/visual-editor/index.js b/packages/edit-post/src/components/visual-editor/index.js
index cf87daa479eda..0f6d4b73d94f3 100644
--- a/packages/edit-post/src/components/visual-editor/index.js
+++ b/packages/edit-post/src/components/visual-editor/index.js
@@ -64,7 +64,6 @@ export default function VisualEditor() {
 			<div
 				ref={ ref }
 				className="editor-styles-wrapper"
-				tabIndex="-1"
 				style={ resizedCanvasStyles || desktopCanvasStyles }
 			>
 				<WritingFlow>
diff --git a/packages/edit-site/src/components/block-editor/index.js b/packages/edit-site/src/components/block-editor/index.js
index 4e88b651e09c5..61313977e3336 100644
--- a/packages/edit-site/src/components/block-editor/index.js
+++ b/packages/edit-site/src/components/block-editor/index.js
@@ -2,7 +2,7 @@
  * WordPress dependencies
  */
 import { useSelect, useDispatch } from '@wordpress/data';
-import { useCallback } from '@wordpress/element';
+import { useCallback, useRef } from '@wordpress/element';
 import { useEntityBlockEditor } from '@wordpress/core-data';
 import {
 	BlockEditorProvider,
@@ -12,6 +12,7 @@ import {
 	WritingFlow,
 	ObserveTyping,
 	BlockList,
+	__unstableUseBlockSelectionClearer as useBlockSelectionClearer,
 } from '@wordpress/block-editor';
 
 /**
@@ -39,8 +40,11 @@ export default function BlockEditor( { setIsInserterOpen } ) {
 		'postType',
 		templateType
 	);
-
 	const { setPage } = useDispatch( 'core/edit-site' );
+	const ref = useRef();
+
+	useBlockSelectionClearer( ref );
+
 	return (
 		<BlockEditorProvider
 			settings={ settings }
@@ -66,7 +70,10 @@ export default function BlockEditor( { setIsInserterOpen } ) {
 			<SidebarInspectorFill>
 				<BlockInspector />
 			</SidebarInspectorFill>
-			<div className="editor-styles-wrapper edit-site-block-editor__editor-styles-wrapper">
+			<div
+				ref={ ref }
+				className="editor-styles-wrapper edit-site-block-editor__editor-styles-wrapper"
+			>
 				<WritingFlow>
 					<ObserveTyping>
 						<BlockList className="edit-site-block-editor__block-list" />
diff --git a/packages/edit-site/src/components/editor/index.js b/packages/edit-site/src/components/editor/index.js
index 33781bd3dffb8..94094fa335565 100644
--- a/packages/edit-site/src/components/editor/index.js
+++ b/packages/edit-site/src/components/editor/index.js
@@ -19,7 +19,6 @@ import {
 import { EntityProvider } from '@wordpress/core-data';
 import {
 	BlockContextProvider,
-	BlockSelectionClearer,
 	BlockBreadcrumb,
 	__unstableUseEditorStyles as useEditorStyles,
 	__experimentalUseResizeCanvas as useResizeCanvas,
@@ -288,7 +287,7 @@ function Editor() {
 														/>
 													}
 													content={
-														<BlockSelectionClearer
+														<div
 															className="edit-site-visual-editor"
 															style={
 																inlineStyles
@@ -304,7 +303,7 @@ function Editor() {
 																/>
 															) }
 															<KeyboardShortcuts />
-														</BlockSelectionClearer>
+														</div>
 													}
 													actions={
 														<>