diff --git a/packages/block-editor/src/components/block-controls/hook.js b/packages/block-editor/src/components/block-controls/hook.js
index e96d1107e7e65d..35d62bc7fc862c 100644
--- a/packages/block-editor/src/components/block-controls/hook.js
+++ b/packages/block-editor/src/components/block-controls/hook.js
@@ -3,17 +3,15 @@
  */
 import groups from './groups';
 import { useBlockEditContext } from '../block-edit/context';
-import useDisplayBlockControls from '../use-display-block-controls';
 
 export default function useBlockControlsFill( group, shareWithChildBlocks ) {
-	const isDisplayed = useDisplayBlockControls();
-	const { shouldDisplayControls } = useBlockEditContext();
-	const isParentDisplayed = shareWithChildBlocks && shouldDisplayControls;
+	const { shouldDisplayControls, shouldDisplayControlsWithinChildren } =
+		useBlockEditContext();
 
-	if ( isDisplayed ) {
+	if ( shouldDisplayControls ) {
 		return groups[ group ]?.Fill;
 	}
-	if ( isParentDisplayed ) {
+	if ( shareWithChildBlocks && shouldDisplayControlsWithinChildren ) {
 		return groups.parent.Fill;
 	}
 	return null;
diff --git a/packages/block-editor/src/components/block-edit/index.js b/packages/block-editor/src/components/block-edit/index.js
index f96706e0f8d396..2b76d039980736 100644
--- a/packages/block-editor/src/components/block-edit/index.js
+++ b/packages/block-editor/src/components/block-edit/index.js
@@ -47,7 +47,6 @@ export default function BlockEdit( props ) {
 				getBlockName,
 				isFirstMultiSelectedBlock,
 				getMultiSelectedBlockClientIds,
-				hasSelectedInnerBlock,
 			} = select( blockEditorStore );
 
 			if ( isFirstMultiSelectedBlock( clientId ) ) {
@@ -56,6 +55,14 @@ export default function BlockEdit( props ) {
 				);
 			}
 
+			return false;
+		},
+		[ clientId, isSelected, name ]
+	);
+	const shouldDisplayControlsWithinChildren = useSelect(
+		( select ) => {
+			const { getBlockName, hasSelectedInnerBlock } =
+				select( blockEditorStore );
 			return (
 				hasBlockSupport(
 					getBlockName( clientId ),
@@ -73,6 +80,7 @@ export default function BlockEdit( props ) {
 		layout: layoutSupport ? layout : null,
 		__unstableLayoutClassNames,
 		shouldDisplayControls,
+		shouldDisplayControlsWithinChildren,
 	};
 	return (
 		<BlockEditContextProvider
diff --git a/packages/block-editor/src/hooks/align.js b/packages/block-editor/src/hooks/align.js
index ac46a227b3428d..be981706bf54ca 100644
--- a/packages/block-editor/src/hooks/align.js
+++ b/packages/block-editor/src/hooks/align.js
@@ -169,10 +169,13 @@ function AlignControls( props ) {
  */
 export const withToolbarControls = createHigherOrderComponent(
 	( BlockEdit ) => ( props ) => {
+		const { shouldDisplayControls, shouldDisplayControlsWithinChildren } =
+			useBlockEditContext();
 		return (
 			<>
-				{ useBlockEditContext().shouldDisplayControls &&
-					!! getBlockSupport( props.name, 'align' ) && (
+				{ ( shouldDisplayControls ||
+					shouldDisplayControlsWithinChildren ) &&
+					hasBlockSupport( props.name, 'align' ) && (
 						<AlignControls { ...props } />
 					) }
 				<BlockEdit { ...props } />