Skip to content

Commit

Permalink
Only use a single comment for the flakiness report on PRs
Browse files Browse the repository at this point in the history
  • Loading branch information
@kevin940726
kevin940726 committed Dec 26, 2022
1 parent 6051295 commit 537c692
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 32 deletions.
5 changes: 4 additions & 1 deletion packages/report-flaky-tests/src/__tests__/markdown.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,15 +246,17 @@ describe( 'renderCommitComment', () => {
issueUrl: 'url2',
},
];
const commitSHA = 'commitSHA';

const commentBody = renderCommitComment( {
reportedIssues,
runURL,
commitSHA,
} );

expect( commentBody ).toMatchInlineSnapshot( `
"<!-- flaky-tests-report-comment -->
**Flaky tests detected.**
**Flaky tests detected in commitSHA.**
Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information.
🔍 Workflow run URL: runURL
Expand All @@ -270,6 +272,7 @@ describe( 'isReportComment', () => {
const commentBody = renderCommitComment( {
reportedIssues: [],
runURL: '',
commitSHA: 'commitSHA',
} );

expect( isReportComment( commentBody ) ).toBe( true );
Expand Down
27 changes: 15 additions & 12 deletions packages/report-flaky-tests/src/__tests__/run.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ const mockPullRequestEventContext = {
sha: 'mergeSHA',
eventName: 'pull_request',
payload: {
number: 10,
pull_request: {
head: {
ref: 'headBranch',
Expand Down Expand Up @@ -60,7 +61,8 @@ const mockAPI = {
findMergeBaseCommit: jest.fn(),
updateIssue: jest.fn(),
createIssue: jest.fn(),
createComment: jest.fn(),
createCommentOnPR: jest.fn(),
createCommentOnCommit: jest.fn(),
};
jest.mock( '../github-api', () => ( {
GitHubAPI: jest.fn( () => mockAPI ),
Expand Down Expand Up @@ -130,7 +132,7 @@ describe( 'Report flaky tests', () => {
html_url: 'html_url',
} ) );

mockAPI.createComment.mockImplementationOnce( () => ( {
mockAPI.createCommentOnPR.mockImplementationOnce( () => ( {
html_url: 'comment_html_url',
} ) );

Expand Down Expand Up @@ -159,12 +161,12 @@ describe( 'Report flaky tests', () => {
'Created new flaky issue'
);

expect( mockAPI.createComment ).toHaveBeenCalledTimes( 1 );
expect( mockAPI.createComment.mock.calls[ 0 ][ 0 ] ).toBe( 'headSHA' );
expect( mockAPI.createComment.mock.calls[ 0 ][ 1 ] )
expect( mockAPI.createCommentOnPR ).toHaveBeenCalledTimes( 1 );
expect( mockAPI.createCommentOnPR.mock.calls[ 0 ][ 0 ] ).toBe( 10 );
expect( mockAPI.createCommentOnPR.mock.calls[ 0 ][ 1 ] )
.toMatchInlineSnapshot( `
"<!-- flaky-tests-report-comment -->
**Flaky tests detected.**
**Flaky tests detected in headSHA.**
Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information.
🔍 Workflow run URL: https://github.com/WordPress/gutenberg/actions/runs/100
Expand Down Expand Up @@ -230,7 +232,7 @@ describe( 'Report flaky tests', () => {
html_url: 'html_url',
} ) );

mockAPI.createComment.mockImplementationOnce( () => ( {
mockAPI.createCommentOnCommit.mockImplementationOnce( () => ( {
html_url: 'comment_html_url',
} ) );

Expand Down Expand Up @@ -259,14 +261,14 @@ describe( 'Report flaky tests', () => {
'Created new flaky issue'
);

expect( mockAPI.createComment ).toHaveBeenCalledTimes( 1 );
expect( mockAPI.createComment.mock.calls[ 0 ][ 0 ] ).toBe(
expect( mockAPI.createCommentOnCommit ).toHaveBeenCalledTimes( 1 );
expect( mockAPI.createCommentOnCommit.mock.calls[ 0 ][ 0 ] ).toBe(
'commitSHA'
);
expect( mockAPI.createComment.mock.calls[ 0 ][ 1 ] )
expect( mockAPI.createCommentOnCommit.mock.calls[ 0 ][ 1 ] )
.toMatchInlineSnapshot( `
"<!-- flaky-tests-report-comment -->
**Flaky tests detected.**
**Flaky tests detected in commitSHA.**
Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information.
🔍 Workflow run URL: https://github.com/WordPress/gutenberg/actions/runs/100
Expand Down Expand Up @@ -331,6 +333,7 @@ describe( 'Report flaky tests', () => {

expect( mockAPI.updateIssue ).not.toHaveBeenCalled();

expect( mockAPI.createComment ).not.toHaveBeenCalled();
expect( mockAPI.createCommentOnPR ).not.toHaveBeenCalled();
expect( mockAPI.createCommentOnCommit ).not.toHaveBeenCalled();
} );
} );
55 changes: 49 additions & 6 deletions packages/report-flaky-tests/src/github-api.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,6 @@ import { getOctokit } from '@actions/github';
import type { GitHub } from '@actions/github/lib/utils';
import type { Endpoints } from '@octokit/types';

/**
* Internal dependencies
*/
import { isReportComment } from './markdown';

type Octokit = InstanceType< typeof GitHub >;

type Repo = {
Expand Down Expand Up @@ -78,7 +73,11 @@ class GitHubAPI {
return data;
}

async createComment( sha: string, body: string ) {
async createCommentOnCommit(
sha: string,
body: string,
isReportComment: ( body: string ) => boolean
) {
const { data: comments } =
await this.#octokit.rest.repos.listCommentsForCommit( {
...this.#repo,
Expand Down Expand Up @@ -108,6 +107,50 @@ class GitHubAPI {

return data;
}

async createCommentOnPR(
prNumber: number,
body: string,
isReportComment: ( body: string ) => boolean
) {
let reportComment;
let page = 1;

while ( ! reportComment ) {
const { data: comments } =
await this.#octokit.rest.issues.listComments( {
...this.#repo,
issue_number: prNumber,
page,
} );
reportComment = comments.find(
( comment ) => comment.body && isReportComment( comment.body )
);
if ( comments.length > 0 ) {
page += 1;
} else {
break;
}
}

if ( reportComment ) {
const { data } = await this.#octokit.rest.issues.updateComment( {
...this.#repo,
comment_id: reportComment.id,
body,
} );

return data;
}

const { data } = await this.#octokit.rest.issues.createComment( {
...this.#repo,
issue_number: prNumber,
body,
} );

return data;
}
}

export { GitHubAPI };
4 changes: 3 additions & 1 deletion packages/report-flaky-tests/src/markdown.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,12 +208,14 @@ const FLAKY_TESTS_REPORT_COMMENT_TOKEN = `flaky-tests-report-comment`;
function renderCommitComment( {
reportedIssues,
runURL,
commitSHA,
}: {
reportedIssues: ReportedIssue[];
runURL: string;
commitSHA: string;
} ) {
return `<!-- ${ FLAKY_TESTS_REPORT_COMMENT_TOKEN } -->
**Flaky tests detected.**
**Flaky tests detected in ${ commitSHA }.**
Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information.
🔍 Workflow run URL: ${ runURL }
Expand Down
36 changes: 24 additions & 12 deletions packages/report-flaky-tests/src/run.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
formatTestResults,
parseIssueBody,
renderCommitComment,
isReportComment,
} from './markdown';
import type { ReportedIssue } from './types';

Expand Down Expand Up @@ -163,19 +164,30 @@ async function run() {
return;
}

const commitSHA =
const { html_url: commentUrl } =
github.context.eventName === 'pull_request'
? // Cast the payload type: https://github.com/actions/toolkit/tree/main/packages/github#webhook-payload-typescript-definitions
( github.context.payload as PullRequestEvent ).pull_request.head
.sha
: github.context.sha;
const { html_url: commentUrl } = await api.createComment(
commitSHA,
renderCommitComment( {
runURL,
reportedIssues,
} )
);
? await api.createCommentOnPR(
// Cast the payload type: https://github.com/actions/toolkit/tree/main/packages/github#webhook-payload-typescript-definitions
( github.context.payload as PullRequestEvent ).number,
renderCommitComment( {
runURL,
reportedIssues,
commitSHA: (
github.context.payload as PullRequestEvent
).pull_request.head.sha,
} ),
isReportComment
)
: await api.createCommentOnCommit(
github.context.sha,
renderCommitComment( {
runURL,
reportedIssues,
commitSHA: github.context.sha,
} ),
isReportComment
);

core.info( `Reported the summary of the flaky tests to ${ commentUrl }` );
}

Expand Down

1 comment on commit 537c692

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Flaky tests detected.
Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See the documentation for more information.

🔍 Workflow run URL: https://github.com/WordPress/gutenberg/actions/runs/3780237719
📝 Reported issues:

Please sign in to comment.