E2E test #22790
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E test | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| cases: | |
| description: 'specific cases to be excuted. Sample: ["./aad/a.tests.ts", "./bot/b.tests.ts"]. Set empty to run all cases' | |
| required: false | |
| target-testplan-id: | |
| description: "target testplan id: 24569079. If not set, will not archive ado testplan" | |
| required: false | |
| type: string | |
| target-version: | |
| description: "target TTK version. Sample: 5.0.2-alpha.2fdf2b99b.0, 5.0.2-rc.0" | |
| required: false | |
| type: string | |
| schedule: | |
| - cron: "0 22 * * *" | |
| pull_request_target: | |
| jobs: | |
| setup: | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx') }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| cases: ${{ steps.schedule-cases.outputs.cases || steps.dispatch-cases.outputs.cases || steps.pr-cases.outputs.cases }} | |
| env: | |
| AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }} | |
| AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }} | |
| AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }} | |
| M365_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }} | |
| M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} | |
| M365_TENANT_ID: ${{ secrets.TEST_TENANT_ID_2 }} | |
| CI_ENABLED: "true" | |
| M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME }} | |
| AZURE_DEVOPS_EXT_PAT: ${{ secrets.ADO_PAT }} | |
| AUTO_TEST_PLAN_ID: ${{ github.event.inputs.target-testplan-id }} | |
| ADO_TOKEN: ${{ secrets.ADO_PAT }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: Setup legacy-peer-deps | |
| run: | | |
| npm config set legacy-peer-deps true | |
| - name: manage pkgs to setup | |
| run: bash .github/scripts/lerna.sh tests | |
| - name: Setup project | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 5 | |
| command: | | |
| npm run setup | |
| - name: List cases for schedule | |
| id: schedule-cases | |
| if: ${{ github.event_name == 'schedule' }} | |
| working-directory: packages/tests/src/e2e | |
| run: | | |
| cases=`find . -wholename "*.tests.ts" | jq -Rsc '[split("\n") | .[]| select(.!="")]'` | |
| echo "cases=$cases" >> $GITHUB_OUTPUT | |
| - name: List cases for dispatch | |
| id: dispatch-cases | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| working-directory: packages/tests/src/e2e | |
| run: | | |
| inputCases='${{ github.event.inputs.cases }}' | |
| if [ -z "$inputCases" ]; then | |
| allCases=`find . -wholename "*.tests.ts" | jq -Rsc '[split("\n") | .[]| select(.!="")]'` | |
| echo "cases=$allCases" >> $GITHUB_OUTPUT | |
| else | |
| echo "cases=$inputCases" >> $GITHUB_OUTPUT | |
| fi | |
| - name: List cases for pull request | |
| id: pr-cases | |
| if: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx' }} | |
| working-directory: packages/tests/src/e2e | |
| run: | | |
| cases=`find ./bot ./frontend -path "*.tests.ts" -and \ | |
| -not '(' -path '*/*.dotnet.tests.ts' -or -path '*/Blazor*' ')' \ | |
| | jq -Rsc '[split("\n") | .[]| select(.!="")]'` | |
| echo "cases=$cases" >> $GITHUB_OUTPUT | |
| - name: E2E Test clean | |
| working-directory: packages/tests | |
| run: | | |
| npm run test:e2e:clean | |
| - name: Archive Test Plan | |
| if: ${{ github.event.inputs.target-testplan-id != '' }} | |
| working-directory: ./packages/tests | |
| run: | | |
| npm install | |
| testplanid=`npx ts-node src/scripts/testPlan.ts obtain vscode ${{ github.event.inputs.target-version }}` | |
| npx ts-node src/scripts/testPlan.ts archive $testplanid | |
| - name: Upload testplan to artifact | |
| if: ${{ github.event.inputs.target-testplan-id != '' }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: testplan | |
| path: | | |
| ./packages/tests/testplan.json | |
| execute-case: | |
| if: ${{ needs.setup.outputs.cases }} | |
| env: | |
| AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }} | |
| AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }} | |
| AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }} | |
| M365_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }} | |
| M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} | |
| M365_TENANT_ID: ${{ secrets.TEST_TENANT_ID_2 }} | |
| CI_ENABLED: "true" | |
| M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME }} | |
| AZURE_DEVOPS_EXT_PAT: ${{ secrets.ADO_PAT }} | |
| TEAMSFX_DEBUG_TEMPLATE: "true" | |
| NODE_ENV: "development" | |
| TEAMSFX_AAD_DEPLOY_ONLY: "true" | |
| SIDELOADING_SERVICE_ENDPOINT: ${{ secrets.SIDELOADING_SERVICE_ENDPOINT }} | |
| SIDELOADING_SERVICE_SCOPE: ${{ secrets.SIDELOADING_SERVICE_SCOPE }} | |
| ADO_TOKEN: ${{ secrets.ADO_PAT }} | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cases: ${{ fromJson(needs.setup.outputs.cases) }} | |
| name: ${{ matrix.cases }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: Setup Azure Functions Core Tools For Linux | |
| run: | | |
| curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg | |
| sudo mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg | |
| sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-$(lsb_release -cs)-prod $(lsb_release -cs) main" > /etc/apt/sources.list.d/dotnetdev.list' | |
| sudo apt-get update | |
| sudo apt-get install azure-functions-core-tools-4 | |
| which func | |
| func --version | |
| - name: manage pkgs to setup | |
| run: bash .github/scripts/lerna.sh tests | |
| - name: Setup legacy-peer-deps | |
| run: | | |
| npm config set legacy-peer-deps true | |
| - name: Setup .net | |
| uses: actions/setup-dotnet@v2 | |
| with: | |
| dotnet-version: 6.0.x | |
| - name: Setup project | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 5 | |
| command: | | |
| npm run setup | |
| - name: Link CLI | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| run: | | |
| npm link --force | |
| - name: Update CLI and legacy-peer-deps for PR cases | |
| if: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx' }} | |
| run: | | |
| npm install -g @microsoft/teamsfx-cli@latest | |
| npm config set legacy-peer-deps false | |
| - name: print system info | |
| run: | | |
| lscpu | |
| - name: Download samples(daily) | |
| if: github.event_name == 'schedule' && startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage') == false | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: OfficeDev/TeamsFx-Samples | |
| ref: dev | |
| path: packages/tests/src/e2e/resource | |
| - name: Download samples(rc) | |
| if: github.event_name == 'workflow_dispatch' && startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage') == false && contains(matrix.cases, 'SignatureOutlook') == false | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: OfficeDev/TeamsFx-Samples | |
| ref: v3 | |
| path: packages/tests/src/e2e/resource | |
| - name: Download samples from another repo | |
| if: startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage') | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: OfficeDev/Microsoft-Teams-Samples | |
| ref: main | |
| path: packages/tests/src/e2e/resource | |
| - name: Download samples from office repo | |
| if: startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'SignatureOutlook') | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: OfficeDev/Office-Add-in-samples | |
| ref: main | |
| path: packages/tests/src/e2e/resource | |
| - name: run test | |
| working-directory: packages/tests/src/e2e | |
| run: | | |
| file=`find . -wholename "${{ matrix.cases }}"` | |
| if [ -z "$file" ]; then | |
| echo "can't find target case in $file" | |
| exit 1 | |
| else | |
| npx mocha --reporter mochawesome --timeout 1200000 $file | |
| fi | |
| - name: get report name | |
| id: get-report-name | |
| if: ${{ always() }} | |
| run: | | |
| name="${{ matrix.cases }}" | |
| name="${name//'.tests.ts'/}" | |
| name="${name//.\//}" | |
| name="${name//\//_}" | |
| echo "name=$name" >> $GITHUB_OUTPUT | |
| - name: Upload test report | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ github.event_name != 'schedule' || success() || (failure() && github.run_attempt >= 5) }} | |
| with: | |
| name: test-result-${{ steps.get-report-name.outputs.name }} | |
| path: | | |
| ./packages/tests/src/e2e/mochawesome-report/mochawesome.json | |
| - name: Download TestPlan | |
| if: ${{ always() && github.event.inputs.target-testplan-id != '' }} | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: testplan | |
| path: ./packages/tests | |
| - name: Sync to Azure DevOps Test Plan | |
| if: ${{ always() && github.event.inputs.target-testplan-id != '' }} | |
| working-directory: packages/tests | |
| run: | | |
| npx ts-node src/scripts/testPlan.ts report ./testplan.json src/e2e/mochawesome-report/mochawesome.json | |
| tear-down: | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx') }} | |
| needs: execute-case | |
| runs-on: ubuntu-latest | |
| env: | |
| AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }} | |
| AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }} | |
| AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }} | |
| M365_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }} | |
| M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} | |
| M365_TENANT_ID: ${{ secrets.TEST_TENANT_ID_2 }} | |
| CI_ENABLED: "true" | |
| M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME }} | |
| AZURE_DEVOPS_EXT_PAT: ${{ secrets.ADO_PAT }} | |
| steps: | |
| - name: Checkout (dev) | |
| uses: actions/checkout@v3 | |
| - name: Setup node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: manage pkgs to setup | |
| run: bash .github/scripts/lerna.sh tests | |
| - name: Setup legacy-peer-deps | |
| run: | | |
| npm config set legacy-peer-deps true | |
| - name: Setup project | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 5 | |
| command: | | |
| npm run setup | |
| - name: E2E Test clean | |
| working-directory: packages/tests | |
| run: | | |
| npm run test:e2e:clean | |
| rerun: | |
| needs: tear-down | |
| if: ${{ (github.event_name == 'schedule' || (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == 'teamsfx-bot/TeamsFx')) && failure() && github.run_attempt < 5 }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: rerun | |
| run: | | |
| curl \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/${{ github.repository }}/actions/workflows/rerun.yml/dispatches \ | |
| -d '{"ref":"${{ github.ref_name }}","inputs":{"run_id":"${{ github.run_id }}", "max_attempts":"5"}}' | |
| report: | |
| needs: execute-case | |
| if: ${{ github.event_name == 'schedule' && (success() || (failure() && github.run_attempt >= 5)) }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Install Dateutils | |
| run: | | |
| sudo apt install dateutils | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| path: ~/artifacts | |
| - name: List jobs | |
| id: list-jobs | |
| working-directory: packages/tests | |
| run: | | |
| page=1 | |
| jobs="[]" | |
| while : | |
| do | |
| url=https://api.github.com/repos/OfficeDev/TeamsFx/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs\?per_page\=100\&page\=$page | |
| resp=`curl -H "Accept: application/vnd.github.v3+json" -u:${{ secrets.GITHUB_TOKEN }} $url` | |
| new_jobs=`echo $resp | jq -cr '.jobs'` | |
| jobs=`jq -cr --slurp 'add' <(echo "$jobs") <(echo "$new_jobs")` | |
| has_next=`curl -I -H "Accept: application/vnd.github.v3+json" -u:${{ secrets.GITHUB_TOKEN }} $url | grep -Fi "link:" | grep "rel=\"last\"" || true` | |
| if [ -z "$has_next" ]; then | |
| break | |
| fi | |
| page=$((page+1)) | |
| done | |
| cases=`echo $jobs| jq -r '.[] | select(.name | contains("tests.ts")) | .name'` | |
| passed=0 | |
| failed=0 | |
| skipped=0 | |
| passed_lists="" | |
| failed_lists="" | |
| skipped_lists="" | |
| emails="[email protected];" | |
| while IFS= read -r case; | |
| do | |
| if [ -z "$case" ]; then | |
| continue | |
| fi | |
| file="src/e2e/$case" | |
| elegant_path="${file//.\//}" | |
| started_at=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .steps[] | select(.name == "run test") | .started_at'` | |
| completed_at=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .steps[] | select(.name == "run test") | .completed_at'` | |
| duration=`dateutils.ddiff $started_at $completed_at -f "%Mm %Ss"` | |
| email="" | |
| if grep -q "@author" $file; then | |
| email=`grep '@author' $file | grep -i -o '[A-Z0-9._%+-]\+@[A-Z0-9.-]\+\.[A-Z]\{2,4\}'` | |
| fi | |
| author="" | |
| if [ -z "$email" ]; then | |
| author="N/A" | |
| else | |
| author="<a href=\\\"mailto:$email\\\"><span>$email</span></a>" | |
| fi | |
| url=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .html_url'` | |
| url="<a href=\\\"$url\\\">$elegant_path</a>" | |
| target_type="TS/JS" | |
| if [[ $case == *".dotnet."* ]]; then | |
| target_type=".NET" | |
| fi | |
| name="${case//'.tests.ts'/}" | |
| name="${name//.\//}" | |
| name="${name//\//_}" | |
| job_id=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .id'` | |
| report_file=`find ~/artifacts -wholename "*${name}*/mochawesome.json"` | |
| if [ ! -z "$report_file" ]; then | |
| echo "Found the $report_file with $job_id and $name" | |
| tests=`cat $report_file | jq -cr '[ .. | objects | with_entries(select(.key=="tests")) | select(. != {}) | select(.tests | type=="array") ] | map(.tests | .[]) | .[]'` | |
| lable="" | |
| while IFS= read -r test; | |
| do | |
| name=`echo $test | jq -cr .fullTitle` | |
| duration=`echo $test | jq -cr .duration` | |
| if [[ ! -z `echo $test | jq 'select(.pass==true)'` ]]; then | |
| passed=$((passed+1)) | |
| label="<span style=\\\"background-color:#2aa198;color:white;font-weight:bold;\\\">PASSED</span>" | |
| elif [[ ! -z `echo $test | jq 'select(.fail==true)'` ]]; then | |
| failed=$((failed+1)) | |
| label="<span style=\\\"background-color: #dc322f;color:white;font-weight:bold;\\\">FAILED</span>" | |
| if [[ ! -z "$email" && ! "$emails" == *"$email"* ]]; then | |
| emails="$emails;$email" | |
| fi | |
| elif [[ ! -z `echo $test | jq 'select(.skipped==true)'` || ! -z `echo $test | jq 'select(.pending==true)'` ]]; then | |
| skipped=$((skipped+1)) | |
| label="<span style=\\\"background-color: #b58900;color:white;font-weight:bold;\\\">SKIPPED</span>" | |
| fi | |
| row="<tr> <td style=\\\"text-align: left;\\\">$url</td> <td style=\\\"text-align: left;\\\">$name</td> <td style=\\\"text-align: center;\\\">$target_type</td> <td style=\\\"text-align: center;\\\">$label</td> <td style=\\\"text-align: center;\\\">$author</td> <td>$((duration/1000)) sec</td> </tr>" | |
| if [[ ! -z `echo $test | jq 'select(.pass==true)'` ]]; then | |
| passed_lists="$passed_lists $row" | |
| elif [[ ! -z `echo $test | jq 'select(.fail==true)'` ]]; then | |
| failed_lists="$failed_lists $row" | |
| elif [[ ! -z `echo $test | jq 'select(.skipped==true)'` || ! -z `echo $test | jq 'select(.pending==true)'` ]]; then | |
| skipped_lists="$skipped_lists $row" | |
| fi | |
| done <<< $tests | |
| else | |
| echo "Failed to find the $report_file with $job_id and $name" | |
| fi | |
| done <<< $cases | |
| body="<table class=\\\"w3-table w3-striped w3-bordered\\\"> <tr> <th>PATH</th> <th>CASE</th> <th>TARGET TYPE</th> <th>STATUS</th> <th>AUTHOR</th> <th>DURATION</th> </tr> $failed_lists $skipped_lists $passed_lists </table> <br />" | |
| total=$((passed+failed+skipped)) | |
| subject="TeamsFx E2E V3 Test Report ($passed/$total Passed, $failed/$total Failed, $skipped/$total Skipped)" | |
| if [ $failed -gt 0 ]; then | |
| subject="[FAILED] $subject" | |
| else | |
| subject="[PASSED] $subject" | |
| fi | |
| echo "body=$body" >> $GITHUB_OUTPUT | |
| echo "to=$emails" >> $GITHUB_OUTPUT | |
| echo "subject=$subject" >> $GITHUB_OUTPUT |