Skip to content

Commit

Permalink
fix(screenshot): show image diff inline in errors list (#32997)
Browse files Browse the repository at this point in the history 
The diff is now shown inline in the errors list.

There are 2 possible failures of toHaveScreenshot
* Previous and actual snapshot mismatch. In this case html report will
show diff between Actual/Previous and have Expected as a separate
screenshot.
* Actual/Previous are equal but they differ from the expected. In this
case html report only contains Actual/Expected images and the diff.

Reference: #32341

<img width="1039" alt="image"
src="https://github.com/user-attachments/assets/b458f986-cc25-4721-862c-0cc2c1b01a42">
  • Loading branch information
@yury-s
yury-s authored Oct 10, 2024
1 parent 10a9e1c commit b9cce59
Show file tree
Hide file tree
Showing 8 changed files with 142 additions and 73 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,4 @@ test-results
.cache/
.eslintcache
playwright.env
firefox
7 changes: 5 additions & 2 deletions packages/html-reporter/src/testErrorView.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,8 @@
limitations under the License.
*/

.test-error-message {
.test-error-view {
white-space: pre;
font-family: monospace;
overflow: auto;
flex: none;
padding: 0;
Expand All @@ -26,3 +25,7 @@
line-height: initial;
margin-bottom: 6px;
}

.test-error-text {
font-family: monospace;
}
36 changes: 27 additions & 9 deletions packages/html-reporter/src/testErrorView.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,39 @@
import ansi2html from 'ansi-to-html';
import * as React from 'react';
import './testErrorView.css';
import type { ImageDiff } from '@web/shared/imageDiffView';
import { ImageDiffView } from '@web/shared/imageDiffView';

export const TestErrorView: React.FC<{
error: string;
}> = ({ error }) => {
const html = React.useMemo(() => {
const config: any = {
bg: 'var(--color-canvas-subtle)',
fg: 'var(--color-fg-default)',
};
config.colors = ansiColors;
return new ansi2html(config).toHtml(escapeHTML(error));
}, [error]);
return <div className='test-error-message' dangerouslySetInnerHTML={{ __html: html || '' }}></div>;
const html = React.useMemo(() => ansiErrorToHtml(error), [error]);
return <div className='test-error-view test-error-text' dangerouslySetInnerHTML={{ __html: html || '' }}></div>;
};

export const TestScreenshotErrorView: React.FC<{
errorPrefix?: string,
diff: ImageDiff,
errorSuffix?: string,
}> = ({ errorPrefix, diff, errorSuffix }) => {
const prefixHtml = React.useMemo(() => ansiErrorToHtml(errorPrefix), [errorPrefix]);
const suffixHtml = React.useMemo(() => ansiErrorToHtml(errorSuffix), [errorSuffix]);
return <div data-testid='test-screenshot-error-view' className='test-error-view'>
<div dangerouslySetInnerHTML={{ __html: prefixHtml || '' }} className='test-error-text' style={{ marginBottom: 20 }}></div>
<ImageDiffView key='image-diff' diff={diff} hideDetails={true}></ImageDiffView>
<div data-testid='error-suffix' dangerouslySetInnerHTML={{ __html: suffixHtml || '' }} className='test-error-text'></div>
</div>;
};

function ansiErrorToHtml(text?: string): string {
const config: any = {
bg: 'var(--color-canvas-subtle)',
fg: 'var(--color-fg-default)',
};
config.colors = ansiColors;
return new ansi2html(config).toHtml(escapeHTML(text || ''));
}

const ansiColors = {
0: '#000',
1: '#C00',
Expand Down
40 changes: 34 additions & 6 deletions packages/html-reporter/src/testResultView.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import { AttachmentLink, generateTraceUrl } from './links';
import { statusIcon } from './statusIcon';
import type { ImageDiff } from '@web/shared/imageDiffView';
import { ImageDiffView } from '@web/shared/imageDiffView';
import { TestErrorView } from './testErrorView';
import { TestErrorView, TestScreenshotErrorView } from './testErrorView';
import './testResultView.css';

function groupImageDiffs(screenshots: Set<TestAttachment>): ImageDiff[] {
Expand Down Expand Up @@ -67,7 +67,7 @@ export const TestResultView: React.FC<{
anchor: 'video' | 'diff' | '',
}> = ({ result, anchor }) => {

const { screenshots, videos, traces, otherAttachments, diffs, htmls } = React.useMemo(() => {
const { screenshots, videos, traces, otherAttachments, diffs, errors, htmls } = React.useMemo(() => {
const attachments = result?.attachments || [];
const screenshots = new Set(attachments.filter(a => a.contentType.startsWith('image/')));
const videos = attachments.filter(a => a.name === 'video');
Expand All @@ -76,7 +76,8 @@ export const TestResultView: React.FC<{
const otherAttachments = new Set<TestAttachment>(attachments);
[...screenshots, ...videos, ...traces, ...htmls].forEach(a => otherAttachments.delete(a));
const diffs = groupImageDiffs(screenshots);
return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, htmls };
const errors = classifyErrors(result.errors, diffs);
return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, errors, htmls };
}, [result]);

const videoRef = React.useRef<HTMLDivElement>(null);
Expand All @@ -94,15 +95,19 @@ export const TestResultView: React.FC<{
}, [scrolled, anchor, setScrolled, videoRef]);

return <div className='test-result'>
{!!result.errors.length && <AutoChip header='Errors'>
{result.errors.map((error, index) => <TestErrorView key={'test-result-error-message-' + index} error={error}></TestErrorView>)}
{!!errors.length && <AutoChip header='Errors'>
{errors.map((error, index) => {
if (error.type === 'screenshot')
return <TestScreenshotErrorView key={'test-result-error-message-' + index} errorPrefix={error.errorPrefix} diff={error.diff!} errorSuffix={error.errorSuffix}></TestScreenshotErrorView>;
return <TestErrorView key={'test-result-error-message-' + index} error={error.error!}></TestErrorView>;
})}
</AutoChip>}
{!!result.steps.length && <AutoChip header='Test Steps'>
{result.steps.map((step, i) => <StepTreeItem key={`step-${i}`} step={step} depth={0}></StepTreeItem>)}
</AutoChip>}

{diffs.map((diff, index) =>
<AutoChip key={`diff-${index}`} header={`Image mismatch: ${diff.name}`} targetRef={imageDiffRef}>
<AutoChip key={`diff-${index}`} dataTestId='test-results-image-diff' header={`Image mismatch: ${diff.name}`} targetRef={imageDiffRef}>
<ImageDiffView key='image-diff' diff={diff}></ImageDiffView>
</AutoChip>
)}
Expand Down Expand Up @@ -145,6 +150,29 @@ export const TestResultView: React.FC<{
</div>;
};

function classifyErrors(testErrors: string[], diffs: ImageDiff[]) {
return testErrors.map(error => {
if (error.includes('Screenshot comparison failed:')) {
const matchingDiff = diffs.find(diff => {
const attachmentName = diff.actual?.attachment.name;
return attachmentName && error.includes(attachmentName);
});

if (matchingDiff) {
const lines = error.split('\n');
const index = lines.findIndex(line => /Expected:|Previous:|Received:/.test(line));
const errorPrefix = index !== -1 ? lines.slice(0, index).join('\n') : lines[0];

const diffIndex = lines.findIndex(line => / +Diff:/.test(line));
const errorSuffix = diffIndex !== -1 ? lines.slice(diffIndex + 2).join('\n') : lines.slice(1).join('\n');

return { type: 'screenshot', diff: matchingDiff, errorPrefix, errorSuffix };
}
}
return { type: 'regular', error };
});
}

const StepTreeItem: React.FC<{
step: TestStep;
depth: number,
Expand Down
9 changes: 6 additions & 3 deletions packages/playwright-core/src/server/page.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ import type { TimeoutOptions } from '../common/types';
import { isInvalidSelectorError } from '../utils/isomorphic/selectorParser';
import { parseEvaluationResultValue, source } from './isomorphic/utilityScriptSerializers';
import type { SerializedValue } from './isomorphic/utilityScriptSerializers';
import { TargetClosedError } from './errors';
import { TargetClosedError, TimeoutError } from './errors';
import { asLocator } from '../utils';
import { helper } from './helper';

Expand Down Expand Up @@ -662,7 +662,7 @@ export class Page extends SdkObject {
return {};
}

if (areEqualScreenshots(actual, options.expected, previous)) {
if (areEqualScreenshots(actual, options.expected, undefined)) {
progress.log(`screenshot matched expectation`);
return {};
}
Expand All @@ -672,10 +672,13 @@ export class Page extends SdkObject {
// A: We want user to receive a friendly diff between actual and expected/previous.
if (js.isJavaScriptErrorInEvaluate(e) || isInvalidSelectorError(e))
throw e;
let errorMessage = e.message;
if (e instanceof TimeoutError && intermediateResult?.previous)
errorMessage = `Failed to take two consecutive stable screenshots. ${e.message}`;
return {
log: e.message ? [...metadata.log, e.message] : metadata.log,
...intermediateResult,
errorMessage: e.message,
errorMessage,
};
});
}
Expand Down
4 changes: 2 additions & 2 deletions packages/playwright/src/matchers/toMatchSnapshot.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,7 @@ export async function toHaveScreenshot(
// - regular matcher (i.e. not a `.not`)
// - perhaps an 'all' flag to update non-matching screenshots
expectScreenshotOptions.expected = await fs.promises.readFile(helper.expectedPath);
const { actual, diff, errorMessage, log } = await page._expectScreenshot(expectScreenshotOptions);
const { actual, previous, diff, errorMessage, log } = await page._expectScreenshot(expectScreenshotOptions);

if (!errorMessage)
return helper.handleMatching();
Expand All @@ -436,7 +436,7 @@ export async function toHaveScreenshot(
return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true);
}

return helper.handleDifferent(actual, expectScreenshotOptions.expected, undefined, diff, errorMessage, log);
return helper.handleDifferent(actual, expectScreenshotOptions.expected, previous, diff, errorMessage, log);
}

function writeFileSync(aPath: string, content: Buffer | string) {
Expand Down
46 changes: 26 additions & 20 deletions packages/web/src/shared/imageDiffView.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,18 +61,21 @@ const checkerboardStyle: React.CSSProperties = {
export const ImageDiffView: React.FC<{
diff: ImageDiff,
noTargetBlank?: boolean,
}> = ({ diff, noTargetBlank }) => {
hideDetails?: boolean,
}> = ({ diff, noTargetBlank, hideDetails }) => {
const [mode, setMode] = React.useState<'diff' | 'actual' | 'expected' | 'slider' | 'sxs'>(diff.diff ? 'diff' : 'actual');
const [showSxsDiff, setShowSxsDiff] = React.useState<boolean>(false);

const [expectedImage, setExpectedImage] = React.useState<HTMLImageElement | null>(null);
const [expectedImageTitle, setExpectedImageTitle] = React.useState<string>('Expected');
const [actualImage, setActualImage] = React.useState<HTMLImageElement | null>(null);
const [diffImage, setDiffImage] = React.useState<HTMLImageElement | null>(null);
const [measure, ref] = useMeasure<HTMLDivElement>();

React.useEffect(() => {
(async () => {
setExpectedImage(await loadImage(diff.expected?.attachment.path));
setExpectedImageTitle(diff.expected?.title || 'Expected');
setActualImage(await loadImage(diff.actual?.attachment.path));
setDiffImage(await loadImage(diff.diff?.attachment.path));
})();
Expand All @@ -98,31 +101,31 @@ export const ImageDiffView: React.FC<{
<div data-testid='test-result-image-mismatch-tabs' style={{ display: 'flex', margin: '10px 0 20px' }}>
{diff.diff && <div style={{ ...modeStyle, fontWeight: mode === 'diff' ? 600 : 'initial' }} onClick={() => setMode('diff')}>Diff</div>}
<div style={{ ...modeStyle, fontWeight: mode === 'actual' ? 600 : 'initial' }} onClick={() => setMode('actual')}>Actual</div>
<div style={{ ...modeStyle, fontWeight: mode === 'expected' ? 600 : 'initial' }} onClick={() => setMode('expected')}>Expected</div>
<div style={{ ...modeStyle, fontWeight: mode === 'expected' ? 600 : 'initial' }} onClick={() => setMode('expected')}>{expectedImageTitle}</div>
<div style={{ ...modeStyle, fontWeight: mode === 'sxs' ? 600 : 'initial' }} onClick={() => setMode('sxs')}>Side by side</div>
<div style={{ ...modeStyle, fontWeight: mode === 'slider' ? 600 : 'initial' }} onClick={() => setMode('slider')}>Slider</div>
</div>
<div style={{ display: 'flex', justifyContent: 'center', flex: 'auto', minHeight: fitHeight + 60 }}>
{diff.diff && mode === 'diff' && <ImageWithSize image={diffImage} alt='Diff' canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
{diff.diff && mode === 'actual' && <ImageWithSize image={actualImage} alt='Actual' canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
{diff.diff && mode === 'expected' && <ImageWithSize image={expectedImage} alt='Expected' canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
{diff.diff && mode === 'slider' && <ImageDiffSlider expectedImage={expectedImage} actualImage={actualImage} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale} />}
{diff.diff && mode === 'diff' && <ImageWithSize image={diffImage} alt='Diff' hideSize={hideDetails} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
{diff.diff && mode === 'actual' && <ImageWithSize image={actualImage} alt='Actual' hideSize={hideDetails} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
{diff.diff && mode === 'expected' && <ImageWithSize image={expectedImage} alt={expectedImageTitle} hideSize={hideDetails} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
{diff.diff && mode === 'slider' && <ImageDiffSlider expectedImage={expectedImage} actualImage={actualImage} hideSize={hideDetails} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale} expectedTitle={expectedImageTitle} />}
{diff.diff && mode === 'sxs' && <div style={{ display: 'flex' }}>
<ImageWithSize image={expectedImage} title='Expected' canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
<ImageWithSize image={showSxsDiff ? diffImage : actualImage} title={showSxsDiff ? 'Diff' : 'Actual'} onClick={() => setShowSxsDiff(!showSxsDiff)} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
<ImageWithSize image={expectedImage} title={expectedImageTitle} hideSize={hideDetails} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
<ImageWithSize image={showSxsDiff ? diffImage : actualImage} title={showSxsDiff ? 'Diff' : 'Actual'} onClick={() => setShowSxsDiff(!showSxsDiff)} hideSize={hideDetails} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
</div>}
{!diff.diff && mode === 'actual' && <ImageWithSize image={actualImage} title='Actual' canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
{!diff.diff && mode === 'expected' && <ImageWithSize image={expectedImage} title='Expected' canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
{!diff.diff && mode === 'actual' && <ImageWithSize image={actualImage} title='Actual' hideSize={hideDetails} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
{!diff.diff && mode === 'expected' && <ImageWithSize image={expectedImage} title={expectedImageTitle} hideSize={hideDetails} canvasWidth={fitWidth} canvasHeight={fitHeight} scale={scale}/>}
{!diff.diff && mode === 'sxs' && <div style={{ display: 'flex' }}>
<ImageWithSize image={expectedImage} title='Expected' canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
<ImageWithSize image={expectedImage} title={expectedImageTitle} canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
<ImageWithSize image={actualImage} title='Actual' canvasWidth={sxsScale * imageWidth} canvasHeight={sxsScale * imageHeight} scale={sxsScale} />
</div>}
</div>
<div style={{ alignSelf: 'start', lineHeight: '18px', marginLeft: '15px' }}>
{!hideDetails && <div style={{ alignSelf: 'start', lineHeight: '18px', marginLeft: '15px' }}>
<div>{diff.diff && <a target='_blank' href={diff.diff.attachment.path} rel='noreferrer'>{diff.diff.attachment.name}</a>}</div>
<div><a target={noTargetBlank ? '' : '_blank'} href={diff.actual!.attachment.path} rel='noreferrer'>{diff.actual!.attachment.name}</a></div>
<div><a target={noTargetBlank ? '' : '_blank'} href={diff.expected!.attachment.path} rel='noreferrer'>{diff.expected!.attachment.name}</a></div>
</div>
</div>}
</>}
</div>;
};
Expand All @@ -133,7 +136,9 @@ export const ImageDiffSlider: React.FC<{
canvasWidth: number,
canvasHeight: number,
scale: number,
}> = ({ expectedImage, actualImage, canvasWidth, canvasHeight, scale }) => {
expectedTitle: string,
hideSize?: boolean,
}> = ({ expectedImage, actualImage, canvasWidth, canvasHeight, scale, expectedTitle, hideSize }) => {
const absoluteStyle: React.CSSProperties = {
position: 'absolute',
top: 0,
Expand All @@ -144,7 +149,7 @@ export const ImageDiffSlider: React.FC<{
const sameSize = expectedImage.naturalWidth === actualImage.naturalWidth && expectedImage.naturalHeight === actualImage.naturalHeight;

return <div style={{ flex: 'none', display: 'flex', alignItems: 'center', flexDirection: 'column', userSelect: 'none' }}>
<div style={{ margin: 5 }}>
{!hideSize && <div style={{ margin: 5 }}>
{!sameSize && <span style={{ flex: 'none', margin: '0 5px' }}>Expected </span>}
<span>{expectedImage.naturalWidth}</span>
<span style={{ flex: 'none', margin: '0 5px' }}>x</span>
Expand All @@ -153,15 +158,15 @@ export const ImageDiffSlider: React.FC<{
{!sameSize && <span>{actualImage.naturalWidth}</span>}
{!sameSize && <span style={{ flex: 'none', margin: '0 5px' }}>x</span>}
{!sameSize && <span>{actualImage.naturalHeight}</span>}
</div>
</div>}
<div style={{ position: 'relative', width: canvasWidth, height: canvasHeight, margin: 15, ...checkerboardStyle }}>
<ResizeView
orientation={'horizontal'}
offsets={[slider]}
setOffsets={offsets => setSlider(offsets[0])}
resizerColor={'#57606a80'}
resizerWidth={6}></ResizeView>
<img alt='Expected' style={{
<img alt={expectedTitle} style={{
width: expectedImage.naturalWidth * scale,
height: expectedImage.naturalHeight * scale,
}} draggable='false' src={expectedImage.src} />
Expand All @@ -179,18 +184,19 @@ const ImageWithSize: React.FunctionComponent<{
image: HTMLImageElement,
title?: string,
alt?: string,
hideSize?: boolean,
canvasWidth: number,
canvasHeight: number,
scale: number,
onClick?: () => void;
}> = ({ image, title, alt, canvasWidth, canvasHeight, scale, onClick }) => {
}> = ({ image, title, alt, hideSize, canvasWidth, canvasHeight, scale, onClick }) => {
return <div style={{ flex: 'none', display: 'flex', alignItems: 'center', flexDirection: 'column' }}>
<div style={{ margin: 5 }}>
{!hideSize && <div style={{ margin: 5 }}>
{title && <span style={{ flex: 'none', margin: '0 5px' }}>{title}</span>}
<span>{image.naturalWidth}</span>
<span style={{ flex: 'none', margin: '0 5px' }}>x</span>
<span>{image.naturalHeight}</span>
</div>
</div>}
<div style={{ display: 'flex', flex: 'none', width: canvasWidth, height: canvasHeight, margin: 15, ...checkerboardStyle }}>
<img
width={image.naturalWidth * scale}
Expand Down
Loading

0 comments on commit b9cce59

Please sign in to comment.