diff --git a/packages/block-library/src/image/image-size.js b/packages/block-library/src/image/image-size.js
deleted file mode 100644
index d0739fcdd3f2f..0000000000000
--- a/packages/block-library/src/image/image-size.js
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * WordPress dependencies
- */
-import { useState, useEffect } from '@wordpress/element';
-
-/**
- * Internal dependencies
- */
-import { calculatePreferedImageSize } from './utils';
-
-export default function useImageSize( ref, src, dependencies ) {
-	const [ state, setState ] = useState( {
-		imageWidth: null,
-		imageHeight: null,
-		imageWidthWithinContainer: null,
-		imageHeightWithinContainer: null,
-	} );
-
-	useEffect( () => {
-		if ( ! src ) {
-			return;
-		}
-
-		const { defaultView } = ref.current.ownerDocument;
-		const image = new defaultView.Image();
-
-		function calculateSize() {
-			const { width, height } = calculatePreferedImageSize(
-				image,
-				ref.current
-			);
-
-			setState( {
-				imageWidth: image.width,
-				imageHeight: image.height,
-				imageWidthWithinContainer: width,
-				imageHeightWithinContainer: height,
-			} );
-		}
-
-		defaultView.addEventListener( 'resize', calculateSize );
-		image.addEventListener( 'load', calculateSize );
-		image.src = src;
-
-		return () => {
-			defaultView.removeEventListener( 'resize', calculateSize );
-			image.removeEventListener( 'load', calculateSize );
-		};
-	}, [ src, ...dependencies ] );
-
-	return state;
-}
diff --git a/packages/block-library/src/image/image.js b/packages/block-library/src/image/image.js
index 1681762027483..7445e09d3199f 100644
--- a/packages/block-library/src/image/image.js
+++ b/packages/block-library/src/image/image.js
@@ -35,7 +35,7 @@ import { createBlock } from '@wordpress/blocks';
  * Internal dependencies
  */
 import { createUpgradedEmbedBlock } from '../embed/util';
-import useImageSize from './image-size';
+import useClientWidth from './use-client-width';
 
 /**
  * Module constants
@@ -87,12 +87,8 @@ export default function Image( {
 	const isLargeViewport = useViewportMatch( 'medium' );
 	const [ captionFocused, setCaptionFocused ] = useState( false );
 	const isWideAligned = includes( [ 'wide', 'full' ], align );
-	const {
-		imageWidthWithinContainer,
-		imageHeightWithinContainer,
-		imageWidth,
-		imageHeight,
-	} = useImageSize( containerRef, url, [ align ] );
+	const [ { naturalWidth, naturalHeight }, setNaturalSize ] = useState( {} );
+	const clientWidth = useClientWidth( containerRef, [ align ] );
 	const isResizable = ! isWideAligned && isLargeViewport;
 	const imageSizeOptions = map(
 		filter( imageSizes, ( { slug } ) =>
@@ -213,8 +209,8 @@ export default function Image( {
 						height={ height }
 						imageSizeOptions={ imageSizeOptions }
 						isResizable={ isResizable }
-						imageWidth={ imageWidth }
-						imageHeight={ imageHeight }
+						imageWidth={ naturalWidth }
+						imageHeight={ naturalHeight }
 					/>
 				</PanelBody>
 			</InspectorControls>
@@ -266,22 +262,43 @@ export default function Image( {
 				alt={ defaultedAlt }
 				onClick={ onImageClick }
 				onError={ () => onImageError() }
+				onLoad={ ( event ) => {
+					setNaturalSize(
+						pick( event.target, [
+							'naturalWidth',
+							'naturalHeight',
+						] )
+					);
+				} }
 			/>
 			{ isBlobURL( url ) && <Spinner /> }
 		</>
 		/* eslint-enable jsx-a11y/no-noninteractive-element-interactions, jsx-a11y/click-events-have-key-events */
 	);
 
+	let imageWidthWithinContainer;
+	let imageHeightWithinContainer;
+
+	if ( clientWidth && naturalWidth && naturalHeight ) {
+		const exceedMaxWidth = naturalWidth > clientWidth;
+		const ratio = naturalHeight / naturalWidth;
+		imageWidthWithinContainer = exceedMaxWidth ? clientWidth : naturalWidth;
+		imageHeightWithinContainer = exceedMaxWidth
+			? clientWidth * ratio
+			: naturalHeight;
+	}
+
 	if ( ! isResizable || ! imageWidthWithinContainer ) {
 		img = <div style={ { width, height } }>{ img }</div>;
 	} else {
 		const currentWidth = width || imageWidthWithinContainer;
 		const currentHeight = height || imageHeightWithinContainer;
 
-		const ratio = imageWidth / imageHeight;
-		const minWidth = imageWidth < imageHeight ? MIN_SIZE : MIN_SIZE * ratio;
+		const ratio = naturalWidth / naturalHeight;
+		const minWidth =
+			naturalWidth < naturalHeight ? MIN_SIZE : MIN_SIZE * ratio;
 		const minHeight =
-			imageHeight < imageWidth ? MIN_SIZE : MIN_SIZE / ratio;
+			naturalHeight < naturalWidth ? MIN_SIZE : MIN_SIZE / ratio;
 
 		// With the current implementation of ResizableBox, an image needs an
 		// explicit pixel value for the max-width. In absence of being able to
diff --git a/packages/block-library/src/image/use-client-width.js b/packages/block-library/src/image/use-client-width.js
new file mode 100644
index 0000000000000..738bfb0373f88
--- /dev/null
+++ b/packages/block-library/src/image/use-client-width.js
@@ -0,0 +1,25 @@
+/**
+ * WordPress dependencies
+ */
+import { useState, useEffect } from '@wordpress/element';
+
+export default function useClientWidth( ref, dependencies ) {
+	const [ clientWidth, setClientWidth ] = useState();
+
+	function calculateClientWidth() {
+		setClientWidth( ref.current.clientWidth );
+	}
+
+	useEffect( calculateClientWidth, dependencies );
+	useEffect( () => {
+		const { defaultView } = ref.current.ownerDocument;
+
+		defaultView.addEventListener( 'resize', calculateClientWidth );
+
+		return () => {
+			defaultView.removeEventListener( 'resize', calculateClientWidth );
+		};
+	}, [] );
+
+	return clientWidth;
+}
diff --git a/packages/block-library/src/image/utils.js b/packages/block-library/src/image/utils.js
index b9df723549d6e..8891d862cb55c 100644
--- a/packages/block-library/src/image/utils.js
+++ b/packages/block-library/src/image/utils.js
@@ -8,15 +8,6 @@ import { isEmpty, each } from 'lodash';
  */
 import { NEW_TAB_REL } from './constants';
 
-export function calculatePreferedImageSize( image, container ) {
-	const maxWidth = container.clientWidth;
-	const exceedMaxWidth = image.width > maxWidth;
-	const ratio = image.height / image.width;
-	const width = exceedMaxWidth ? maxWidth : image.width;
-	const height = exceedMaxWidth ? maxWidth * ratio : image.height;
-	return { width, height };
-}
-
 export function removeNewTabRel( currentRel ) {
 	let newRel = currentRel;