-
Notifications
You must be signed in to change notification settings - Fork 4.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'trunk' into replace/confirm-et-al
* trunk: (74 commits) Update docs for ClipboardButton component (#34711) Post Title Block: add typography formatting options (#31623) Bump plugin version to 11.5.0 Navigation Screen: Adjust header toolbar icon styles (#34833) Fix the parent menu item field in REST API responses (#34835) Rewrite FocusableIframe as hook API (#26753) Create Block: Remove wp-cli callout since not recommended and outdated (#34821) Global Styles: Fix dimensions panel default controls display (#34828) [RNMobile][Embed block] Enable embed preview for Instagram and Vimeo providers (#34563) Increase Link UI search results to 10 on Nav Editor screen (#34808) Prevent welcome guide overflow x scroll (#34713) Enable open on click for Page List inside Navigation. (#34675) [RNMobile] [Embed block] - Unavailable preview fallback bottom sheet title update (#34674) Add missing field _invalid in menu item REST API (#34670) Fix Dropdown/DropdownMenu toggle closing in all UAs (#31170) Navigation submenu block: replace global shortcut event handlers with local ones (#34812) Navigation Screen: Consolidate menu name and switcher (#34786) Remove parent and position validation from menu item REST API endpoint (#34672) Clean theme data when switching themes in the customizer (#34540) Components: add reset timeout to ColorPicker's copy functionality. (#34601) ...
- Loading branch information
Showing
252 changed files
with
6,234 additions
and
2,223 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
name: 'Report flaky tests' | ||
description: 'Report flaky tests to GitHub issues' | ||
inputs: | ||
repo-token: | ||
description: 'GitHub token' | ||
required: true | ||
label: | ||
description: 'The flaky-test label name' | ||
required: true | ||
default: 'flaky-test' | ||
artifact-name: | ||
description: 'The name of the uploaded artifact' | ||
required: true | ||
default: 'flaky-tests-report' | ||
runs: | ||
using: 'node12' | ||
main: 'index.js' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,269 @@ | ||
/** | ||
* External dependencies | ||
*/ | ||
const path = require( 'path' ); | ||
const github = require( '@actions/github' ); | ||
const core = require( '@actions/core' ); | ||
const unzipper = require( 'unzipper' ); | ||
const { formatResultsErrors } = require( 'jest-message-util' ); | ||
|
||
const TEST_RESULTS_LIST = { | ||
open: `<!-- __TEST_RESULTS_LIST__ -->`, | ||
close: `<!-- /__TEST_RESULTS_LIST__ -->`, | ||
}; | ||
const TEST_RESULT = { | ||
open: '<!-- __TEST_RESULT__ -->', | ||
close: '<!-- /__TEST_RESULT__ -->', | ||
}; | ||
|
||
const metaData = { | ||
render: ( json ) => `<!-- __META_DATA__:${ JSON.stringify( json ) } -->`, | ||
get: ( str ) => { | ||
const matched = str.match( /<!-- __META_DATA__:(.*) -->/ ); | ||
if ( matched ) { | ||
return JSON.parse( matched[ 1 ] ); | ||
} | ||
}, | ||
}; | ||
|
||
( async function run() { | ||
const token = core.getInput( 'repo-token', { required: true } ); | ||
const label = core.getInput( 'label', { required: true } ); | ||
const artifactName = core.getInput( 'artifact-name', { required: true } ); | ||
|
||
const octokit = github.getOctokit( token ); | ||
|
||
const flakyTests = await downloadReportFromArtifact( | ||
octokit, | ||
artifactName | ||
); | ||
|
||
if ( ! flakyTests ) { | ||
return; | ||
} | ||
|
||
const issues = await fetchAllIssuesLabeledFlaky( octokit, label ); | ||
|
||
for ( const flakyTest of flakyTests ) { | ||
const { | ||
title: testTitle, | ||
path: testPath, | ||
results: testResults, | ||
} = flakyTest; | ||
const issueTitle = getIssueTitle( testTitle ); | ||
const reportedIssue = issues.find( | ||
( issue ) => issue.title === issueTitle | ||
); | ||
const isTrunk = getHeadBranch() === 'trunk'; | ||
let issue; | ||
|
||
if ( reportedIssue ) { | ||
let body = reportedIssue.body; | ||
const meta = metaData.get( body ); | ||
|
||
if ( isTrunk ) { | ||
const headCommit = github.context.sha; | ||
const baseCommit = meta.baseCommit || github.context.sha; | ||
|
||
try { | ||
const { data } = await octokit.rest.repos.compareCommits( { | ||
...github.context.repo, | ||
base: baseCommit, | ||
head: headCommit, | ||
per_page: 1, | ||
} ); | ||
|
||
meta.failedTimes += testResults.length; | ||
meta.totalCommits = data.total_commits + 1; | ||
} catch ( err ) { | ||
// It might be a deleted commit, | ||
// treat the current commit as the base commit. | ||
meta.baseCommit = headCommit; | ||
meta.failedTimes = testResults.length; | ||
meta.totalCommits = 1; | ||
} | ||
} | ||
|
||
// Reconstruct the body with the description + previous errors + new error. | ||
body = | ||
renderIssueDescription( { meta, testTitle, testPath } ) + | ||
body.slice( | ||
body.indexOf( TEST_RESULTS_LIST.open ), | ||
body.indexOf( TEST_RESULTS_LIST.close ) | ||
) + | ||
[ | ||
renderTestErrorMessage( { testPath, testResults } ), | ||
TEST_RESULTS_LIST.close, | ||
].join( '\n' ); | ||
|
||
const response = await octokit.rest.issues.update( { | ||
...github.context.repo, | ||
issue_number: reportedIssue.number, | ||
state: 'open', | ||
body, | ||
} ); | ||
|
||
issue = response.data; | ||
} else { | ||
const meta = isTrunk | ||
? { | ||
failedTimes: testResults.length, | ||
totalCommits: 1, | ||
baseCommit: github.context.sha, | ||
} | ||
: { | ||
failedTimes: 0, | ||
totalCommits: 0, | ||
}; | ||
|
||
const response = await octokit.rest.issues.create( { | ||
...github.context.repo, | ||
title: issueTitle, | ||
body: renderIssueBody( { | ||
meta, | ||
testTitle, | ||
testPath, | ||
testResults, | ||
} ), | ||
labels: [ label ], | ||
} ); | ||
|
||
issue = response.data; | ||
} | ||
|
||
core.info( `Reported flaky test to ${ issue.html_url }` ); | ||
} | ||
} )().catch( ( err ) => { | ||
core.error( err ); | ||
} ); | ||
|
||
async function fetchAllIssuesLabeledFlaky( octokit, label ) { | ||
const issues = await octokit.paginate( 'GET /repos/{owner}/{repo}/issues', { | ||
...github.context.repo, | ||
state: 'all', | ||
labels: label, | ||
} ); | ||
|
||
return issues; | ||
} | ||
|
||
async function downloadReportFromArtifact( octokit, artifactName ) { | ||
const { | ||
data: { artifacts }, | ||
} = await octokit.rest.actions.listWorkflowRunArtifacts( { | ||
...github.context.repo, | ||
run_id: github.context.payload.workflow_run.id, | ||
} ); | ||
|
||
const matchArtifact = artifacts.find( | ||
( artifact ) => artifact.name === artifactName | ||
); | ||
|
||
if ( ! matchArtifact ) { | ||
// No flaky tests reported in this run. | ||
return; | ||
} | ||
|
||
const download = await octokit.rest.actions.downloadArtifact( { | ||
...github.context.repo, | ||
artifact_id: matchArtifact.id, | ||
archive_format: 'zip', | ||
} ); | ||
|
||
const { files } = await unzipper.Open.buffer( | ||
Buffer.from( download.data ) | ||
); | ||
const fileBuffers = await Promise.all( | ||
files.map( ( file ) => file.buffer() ) | ||
); | ||
const parsedFiles = fileBuffers.map( ( buffer ) => | ||
JSON.parse( buffer.toString() ) | ||
); | ||
|
||
return parsedFiles; | ||
} | ||
|
||
function getIssueTitle( testTitle ) { | ||
return `[Flaky Test] ${ testTitle }`; | ||
} | ||
|
||
function renderIssueBody( { meta, testTitle, testPath, testResults } ) { | ||
return ( | ||
renderIssueDescription( { meta, testTitle, testPath } ) + | ||
renderTestResults( { testPath, testResults } ) | ||
); | ||
} | ||
|
||
function renderIssueDescription( { meta, testTitle, testPath } ) { | ||
return `${ metaData.render( meta ) } | ||
**Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.** | ||
## Test title | ||
${ testTitle } | ||
## Test path | ||
\`${ testPath }\` | ||
## Flaky rate (_estimated_) | ||
\`${ meta.failedTimes } / ${ meta.totalCommits + meta.failedTimes }\` runs | ||
## Errors | ||
`; | ||
} | ||
|
||
function renderTestResults( { testPath, testResults } ) { | ||
return `${ TEST_RESULTS_LIST.open } | ||
${ renderTestErrorMessage( { testPath, testResults } ) } | ||
${ TEST_RESULTS_LIST.close } | ||
`; | ||
} | ||
|
||
function renderTestErrorMessage( { testPath, testResults } ) { | ||
const date = new Date().toISOString(); | ||
// It will look something like this without formatting: | ||
// ▶ [2021-08-31T16:15:19.875Z] Test passed after 2 failed attempts on trunk | ||
return `${ TEST_RESULT.open }<details> | ||
<summary> | ||
<time datetime="${ date }"><code>[${ date }]</code></time> | ||
Test passed after ${ testResults.length } failed ${ | ||
testResults.length === 0 ? 'attempt' : 'attempts' | ||
} on <a href="${ getRunURL() }"><code>${ getHeadBranch() }</code></a>. | ||
</summary> | ||
\`\`\` | ||
${ stripAnsi( | ||
formatResultsErrors( | ||
testResults, | ||
{ rootDir: path.join( process.cwd(), 'packages/e2e-tests' ) }, | ||
{}, | ||
testPath | ||
) | ||
) } | ||
\`\`\` | ||
</details>${ TEST_RESULT.close }`; | ||
} | ||
|
||
function getHeadBranch() { | ||
return github.context.payload.workflow_run.head_branch; | ||
} | ||
|
||
function getRunURL() { | ||
return github.context.payload.workflow_run.html_url; | ||
} | ||
|
||
/** | ||
* Copied pasted from `strip-ansi` to use without ESM. | ||
* @see https://github.com/chalk/strip-ansi | ||
* Licensed under MIT license. | ||
*/ | ||
function stripAnsi( string ) { | ||
return string.replace( | ||
new RegExp( | ||
[ | ||
'[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)', | ||
'(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))', | ||
].join( '|' ), | ||
'g' | ||
) | ||
); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
name: Report Flaky Tests | ||
|
||
on: | ||
workflow_run: | ||
workflows: ['End-to-End Tests'] | ||
types: | ||
- completed | ||
|
||
jobs: | ||
report-to-issues: | ||
name: Report to GitHub issues | ||
runs-on: ubuntu-latest | ||
if: ${{ github.event.workflow_run.conclusion == 'success' }} | ||
steps: | ||
- uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f # v2.3.4 | ||
|
||
- name: Use desired version of NodeJS | ||
uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f # v2.2.2 | ||
with: | ||
node-version: 14 | ||
cache: npm | ||
|
||
- name: Npm install | ||
run: | | ||
npm ci | ||
- name: Report flaky tests | ||
uses: ./.github/report-flaky-tests | ||
with: | ||
repo-token: '${{ secrets.GITHUB_TOKEN }}' | ||
label: '[Type] Flaky Test' | ||
artifact-name: flaky-tests-report |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.