diff --git a/packages/block-editor/src/hooks/layout.js b/packages/block-editor/src/hooks/layout.js
index 836a9cd5a57c6..252f7de86714f 100644
--- a/packages/block-editor/src/hooks/layout.js
+++ b/packages/block-editor/src/hooks/layout.js
@@ -326,18 +326,14 @@ export function addAttribute( settings ) {
  * @return {Function} Wrapped component.
  */
 export const withInspectorControls = createHigherOrderComponent(
-	( BlockEdit ) => ( props ) => {
-		const { name: blockName } = props;
-		const supportLayout = hasBlockSupport(
-			blockName,
-			layoutBlockSupportKey
-		);
-
-		return [
-			supportLayout && <LayoutPanel key="layout" { ...props } />,
+	( BlockEdit ) => ( props ) =>
+		[
+			props.isSelected &&
+				hasBlockSupport( props.name, layoutBlockSupportKey ) && (
+					<LayoutPanel key="layout" { ...props } />
+				),
 			<BlockEdit key="edit" { ...props } />,
-		];
-	},
+		],
 	'withInspectorControls'
 );