API Test #58
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: API Test | |
| on: | |
| workflow_call: | |
| inputs: | |
| environment: | |
| type: string | |
| required: true | |
| description: "where to test" | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| type: choice | |
| required: true | |
| description: "where to test" | |
| default: "tools" | |
| options: | |
| - tools | |
| - production | |
| repository_dispatch: | |
| types: | |
| - webhook | |
| jobs: | |
| define-test-matrix: | |
| runs-on: ubuntu-latest | |
| environment: ${{ inputs.environment }} | |
| outputs: | |
| tests: ${{ steps.tests.outputs.tests }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: tests | |
| working-directory: bin/si-api-test/tests | |
| run: | | |
| # Find .ts files, remove ./ prefix, and format as JSON array | |
| files=$(find "." -mindepth 1 -maxdepth 1 -type f -name "*.ts" | sed -r "s/\.\/(.*)\.ts/\1/" | sort) | |
| # Get the number of tests | |
| test_count=$(echo "$files" | wc -l) | |
| # Get the list of workspace IDs from the environment variable | |
| workspace_ids="${{ vars.API_TEST_WORKSPACE_IDS }}" | |
| echo "workspace_ids found to be $workspace_ids" | |
| workspace_count=$(echo "$workspace_ids" | tr ',' '\n' | wc -l) | |
| # Validate that the number of workspace IDs is greater than or equal to the number of tests | |
| if [ "$workspace_count" -lt "$test_count" ]; then | |
| echo "Error: The number of workspace IDs ($workspace_count) is less than the number of tests ($test_count)." | |
| exit 1 | |
| fi | |
| # Format files as JSON array with correct numbering and sorted order | |
| indexed_files=$(echo "$files" | awk '{print "{\"name\": \"" $0 "\", \"index\": " NR-1 "}"}' | jq -s .) | |
| # Ensure indexed_files are formatted correctly | |
| test_output=$(echo "$indexed_files" | jq -c '.') | |
| echo "tests=$test_output" >> "$GITHUB_OUTPUT" | |
| echo "$test_output" | |
| api-test: | |
| name: API Test SDF | |
| environment: ${{ inputs.environment }} | |
| runs-on: ubuntu-latest | |
| needs: define-test-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tests: ${{ fromJSON(needs.define-test-matrix.outputs.tests) }} | |
| env: | |
| SDF_API_URL: ${{ vars.SDF_API_URL }} | |
| AUTH_API_URL: ${{ vars.AUTH_API_URL }} | |
| outputs: | |
| last_exit_code: ${{ steps.capture-exit-code.outputs.last_exit_code }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Deno | |
| uses: denoland/setup-deno@v1 | |
| with: | |
| deno-version: v1.x | |
| - name: Run the deno exec with retry | |
| run: | | |
| cd bin/si-api-test | |
| echo "Running test ${{ matrix.tests.name }} with index: ${{ matrix.tests.index }}" | |
| # Split the workspace IDs into an array | |
| workspace_ids="${{ vars.API_TEST_WORKSPACE_IDS }}" | |
| IFS=',' read -r -a workspace_array <<< "$workspace_ids" | |
| # Pick the correct workspace ID based on the index | |
| workspace_id=${workspace_array[${{ matrix.tests.index }}]} | |
| echo "Using workspace ID: $workspace_id" | |
| # Retry loop with 5 attempts | |
| n=0 | |
| max_retries=5 | |
| exit_code=0 | |
| until [ $n -ge $max_retries ] | |
| do | |
| unset exit_code || echo "exit_code not set" | |
| # Run the deno task and store exit code in a variable | |
| deno task run \ | |
| --workspaceId "$workspace_id" \ | |
| --userId ${{ secrets.API_TEST_EMAIL }} \ | |
| --password ${{ secrets.API_TEST_PASSWORD }} \ | |
| --tests ${{ matrix.tests.name }} || exit_code=$? | |
| # Check the exit code | |
| if [ -z "$exit_code" ]; then | |
| echo "Deno task succeeded [ or this failed for a totally non-valid reason ]!" | |
| # Randomly make it fail to test logic | |
| if [[ "$(( ( RANDOM % 5 ) + 1 ))" == "3" ]]; then | |
| echo "last_exit_code=$exit_code" >> "$GITHUB_ENV" | |
| exit 53 | |
| fi | |
| break | |
| fi | |
| n=$((n+1)) | |
| echo "Attempt $n/$max_retries failed with exit code $exit_code! Retrying..." | |
| sleep 60 | |
| done | |
| if [ $n -ge $max_retries ]; then | |
| echo "All $max_retries attempts failed." | |
| exit_code=1 | |
| fi | |
| echo "last_exit_code=$exit_code" >> "$GITHUB_ENV" | |
| exit $last_exit_code | |
| - name: Upload artifact if exit code 53 | |
| if: ${{ failure() && env.last_exit_code == '53' }} | |
| run: | | |
| echo "Uploading marker for test ${{ matrix.tests.name }}" | |
| mkdir -p artifacts/${{ matrix.tests.name }} | |
| echo "failure-marker" > artifacts/${{ matrix.tests.name }}/failure-marker | |
| - name: Upload artifacts | |
| if: ${{ failure() && env.last_exit_code == '53' }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: debug-${{ matrix.tests.name }} | |
| path: artifacts/${{ matrix.tests.name }} | |
| on-failure: | |
| runs-on: ubuntu-latest | |
| needs: api-test | |
| environment: ${{ inputs.environment }} | |
| if: ${{ failure() }} #&& github.ref == 'refs/heads/main' }} #TODO(johnrwatson): disable comment before merging | |
| steps: | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: artifacts | |
| - run: | | |
| has_artifacts=false | |
| # Check for marker files | |
| for marker in artifacts/*/failure-marker; do | |
| if [ -f "$marker" ]; then | |
| echo "Artifact detected for a failed test: $marker" | |
| has_artifacts=true | |
| break | |
| fi | |
| done | |
| # If valid failure markers are present page | |
| if [ "$has_artifacts" = true ]; then | |
| curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \ | |
| --header "Content-Type: application/json" \ | |
| --data "{ | |
| \"summary\": \"API ${{ inputs.environment }} Tests Fail\", | |
| \"body\": \"API Tests have failed for ${{ inputs.environment }}.\", | |
| \"links\": [ | |
| { | |
| \"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\", | |
| \"text\": \"E2E Test Run ${{ inputs.environment }}\" | |
| } | |
| ], | |
| \"tags\": [ | |
| \"service:github\" | |
| ] | |
| }" | |
| fi | |
| - run: | | |
| # Always send the Internal Slack Notification if failure detected, even if we didn't page | |
| curl --location "${{ secrets.SLACK_WEBHOOK_URL }}" -X POST \ | |
| --header 'Content-type: application/json' \ | |
| --data "{\"text\": \":si: Failed API Tests for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}" |