Asharapov/dynamic steps driver (#3209) #3465
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Libraries | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| libraries: | |
| description: 'Enter short libraries names separated by spaces (e.g. bio tutorials)' | |
| required: true | |
| type: string | |
| push: | |
| paths: | |
| - 'libraries/**' | |
| pull_request: | |
| paths: | |
| - 'libraries/**' | |
| jobs: | |
| common-check: | |
| name: Common checks | |
| uses: ./.github/workflows/common_check.yaml | |
| with: | |
| run_trigger: ${{ github.event_name }} | |
| metadata: | |
| name: Check changes | |
| needs: common-check | |
| runs-on: ubuntu-22.04 | |
| if: needs.common-check.outputs.continue == 'true' | |
| outputs: | |
| publish_matrix: ${{ steps.generate-matrix.outputs.publish_matrix }} | |
| continue: ${{ steps.generate-matrix.outputs.continue }} | |
| steps: | |
| - name: Set git fetch depth | |
| run: | | |
| if [[ ${{ github.event_name }} != 'pull_request' ]]; then | |
| echo "FETCH_DEPTH=2" >> "$GITHUB_ENV" | |
| else | |
| echo "FETCH_DEPTH=0" >> "$GITHUB_ENV" | |
| fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ env.FETCH_DEPTH }} | |
| sparse-checkout: | | |
| ./libraries | |
| - name: Get changed files | |
| uses: tj-actions/changed-files@v42 | |
| id: changed-files | |
| with: | |
| files: 'libraries/**' | |
| separator: "\n" | |
| dir_names: true | |
| dir_names_max_depth: 2 | |
| safe_output: false | |
| - name: Generate matrix | |
| id: generate-matrix | |
| run: | | |
| if [[ "${{ github.event.inputs.libraries }}" == "" ]]; then | |
| CHANGED_LIBS="$(echo -e "${{ steps.changed-files.outputs.all_changed_files }}" | awk -F'/' '{print $2}' | sort -u)" | |
| else | |
| CHANGED_LIBS="${{ github.event.inputs.libraries }}" | |
| fi | |
| MATRIX_PUBLISH_JSON="[" | |
| for LIB in $(echo ${CHANGED_LIBS} | sort -u); do | |
| DIR="libraries/$LIB" | |
| if [ -f "${DIR}/package.json" ]; then | |
| MATRIX_PUBLISH_JSON+="{\"project\": \"${LIB}\"" | |
| current_version="$(jq -r '.version' "${DIR}/package.json")" | |
| MATRIX_PUBLISH_JSON+=", \"version\": \"${current_version}\"" | |
| scripts="$(jq '. | select( has("scripts") == true ).scripts' "${DIR}/package.json")" | |
| dependencies="$(jq '(. | select( has("dependencies") == true ).dependencies) * (. | select( has("devDependencies") == true ).devDependencies)' "${DIR}/package.json")" | |
| job_name='Check' | |
| if [ ! -z "$(jq '. | select( has("build") == true )' <<< "$scripts")" ]; then | |
| MATRIX_PUBLISH_JSON+=", \"build\": \"build\"" | |
| job_name='Build' | |
| fi | |
| if [ ! -z "$(jq '. | select( has("test") == true )' <<< "$scripts")" ]; then | |
| MATRIX_PUBLISH_JSON+=", \"test\": \"test\"" | |
| job_name='Build, test' | |
| fi | |
| name="$(jq -r .name "${DIR}/package.json")" | |
| npm_version="$(curl --retry 3 -s "https://registry.npmjs.org/${name}/${current_version}" | jq -r '.? | select( has("version") == true ).version')" | |
| unpublished_deps="$(jq -r '. | to_entries | map(select(.value | match("\\.\\./.*")))[] | "\(.key)=\(.value)"' <<<$dependencies | tr '\n' ' ')" | |
| MATRIX_PUBLISH_JSON+=", \"unpublished_deps\": \"$unpublished_deps\"" | |
| if [[ "${{ github.event_name }}" != "pull_request" ]]; then | |
| if [[ $npm_version != ${current_version} ]]; then | |
| if [[ $unpublished_deps == "" ]] && [[ ${{ github.ref }} == 'refs/heads/master' ]]; then | |
| MATRIX_PUBLISH_JSON+=", \"publish\": \"publish\"" | |
| packages=$(grep -l "$name" packages/*/package.json | awk -F'/' '{printf ("%s ", $2)}' | sort -u) | |
| MATRIX_PUBLISH_JSON+=", \"packages\": \"$packages\"" | |
| job_name+=' and publish to NPM' | |
| else | |
| echo "Library $LIB version ${current_version} has unpublished dependencies: $unpublished_deps" | |
| echo "::notice title=${LIB}::Version ${current_version} has unpublished dependencies and is not going to be published" | |
| fi | |
| else | |
| echo "Library $LIB version ${current_version} is already published" | |
| echo "::notice title=${LIB}::Version ${current_version} is already published" | |
| fi | |
| else | |
| echo "It is an actions for PR. Publish will be skipped." | |
| echo "::notice title=${LIB}::It is an actions for PR. Publish will be skipped." | |
| fi | |
| MATRIX_PUBLISH_JSON+=", \"job_name\": \"${job_name}\"" | |
| MATRIX_PUBLISH_JSON+="}" | |
| fi | |
| done | |
| MATRIX_PUBLISH_JSON="${MATRIX_PUBLISH_JSON//\}\{/\}, \{}" | |
| MATRIX_PUBLISH_JSON+="]" | |
| PUBLISH_JSON="{\"include\": ${MATRIX_PUBLISH_JSON}}" | |
| CONTINUE_JOB="no" | |
| if [[ "${MATRIX_PUBLISH_JSON}" != "[]" ]]; then | |
| CONTINUE_JOB="yes" | |
| fi | |
| echo "continue=${CONTINUE_JOB}" >> $GITHUB_OUTPUT | |
| echo "publish_matrix=${PUBLISH_JSON}" >> $GITHUB_OUTPUT | |
| - name: Output | |
| run: | | |
| echo -e "publish_matrix: ${{ steps.generate-matrix.outputs.publish_matrix }}" | |
| echo -e "continue: ${{ steps.generate-matrix.outputs.continue }}" | |
| publish: | |
| name: "${{ matrix.project }}: ${{ matrix.job_name }}" | |
| needs: [ metadata, common-check ] | |
| if: needs.metadata.outputs.continue == 'yes' && needs.common-check.outputs.continue == 'true' | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.metadata.outputs.publish_matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.WRITE_TOKEN || github.token }} | |
| sparse-checkout: | | |
| ./libraries | |
| ./js-api | |
| ./.github | |
| - name: Check library properties | |
| working-directory: libraries/${{ matrix.project }} | |
| run: | | |
| if [[ "$(jq '.name' package.json | sed -E 's/(^"|"$)//g')" != "@datagrok-libraries/"* ]]; then | |
| echo "::error title=${{ matrix.project }}: failed properties check::Library should be in '@datagrok-libraries' scope. Change library name to '@datagrok-libraries/<name>' in ${{ matrix.project }}/package.json" | |
| exit 1 | |
| fi | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18.x' | |
| registry-url: 'https://registry.npmjs.org' | |
| scope: '@datagrok-libraries' | |
| cache: 'npm' | |
| cache-dependency-path: | | |
| libraries/${{ matrix.project }}/package-lock.json | |
| - name: Upgrade npm | |
| run: npm install -g npm@x | |
| - name: npm version | |
| run: npm version | |
| - name: unpublished dependencies | |
| if: ${{ matrix.unpublished_deps != '' }} | |
| run: | | |
| crnt=$(pwd) | |
| for dep in $(echo -e ${{ matrix.unpublished_deps }}); do | |
| cd $(awk -F'=' '{print $2}' <<<$dep) | |
| unpublished_deps_deep="$(jq -r '. | to_entries | map(select(.value | match("\\.\\./.*")))[] | "\(.key)=\(.value)"' <<<$dependencies | tr '\n' ' ')" | |
| for dep_deep in $(echo -e ${unpublished_deps_deep}); do | |
| crnt_deep=$(pwd) | |
| echo "Install dependencies for $(awk -F'=' '{print $2}' <<<$dep_deep)" | |
| cd $(awk -F'=' '{print $2}' <<<$dep_deep) | |
| npm install | |
| npm run build | |
| cd $crnt_deep | |
| done | |
| echo "Install dependencies for $(awk -F'=' '{print $2}' <<<$dep)" | |
| npm install | |
| npm run build | |
| cd $crnt | |
| done | |
| working-directory: libraries/${{ matrix.project }} | |
| - name: unpublished dependencies in package-lock.json | |
| id: package-lock | |
| if: ${{ matrix.unpublished_deps == '' }} | |
| run: grep -q '\.\./\.\./' package-lock.json && rm -f package-lock.json || true | |
| working-directory: libraries/${{ matrix.project }} | |
| - run: npm install | |
| working-directory: libraries/${{ matrix.project }} | |
| - run: npm run build | |
| working-directory: libraries/${{ matrix.project }} | |
| if: ${{ matrix.build == 'build' }} | |
| - id: datagrok-tools | |
| run: npm install -g datagrok-tools@latest | |
| - run: grok check | |
| id: grok_check | |
| working-directory: libraries/${{ matrix.project }} | |
| - run: npm run test | |
| working-directory: libraries/${{ matrix.project }} | |
| if: ${{ matrix.test == 'test' }} | |
| - name: npm publish | |
| id: publish | |
| run: npm publish --access public | |
| if: matrix.publish == 'publish' | |
| working-directory: libraries/${{ matrix.project }} | |
| env: | |
| NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} | |
| - name: Get actors slack | |
| id: actor | |
| uses: ./.github/actions/slack-user-action | |
| with: | |
| slack-token: ${{ secrets.SLACKBOT_TOKEN }} | |
| email-mapping: ${{ secrets.EMAIL_SLACK_ID }} | |
| - name: Get owner email | |
| run: echo "owner=$(jq -r .author.email package.json)" >> $GITHUB_OUTPUT | |
| id: owner-email | |
| working-directory: libraries/${{ matrix.project }} | |
| - name: Get owners slack | |
| id: owner | |
| uses: ./.github/actions/slack-user-action | |
| with: | |
| slack-token: ${{ secrets.SLACKBOT_TOKEN }} | |
| emails: ${{ steps.owner-email.outputs.owner }} | |
| - name: Prepare Slack Message | |
| id: slack-prepare | |
| if: steps.publish.outcome == 'success' | |
| shell: bash | |
| run: | | |
| mentions=$(echo -e "${{ steps.actor.outputs.user-ids }}\n${{ steps.owner.outputs.user-ids }}" | sort -u) | |
| header="Library *${{ matrix.project }}* version *${{ matrix.version }}* published to <https://www.npmjs.com/package/$(jq -r .name "packages/${{ matrix.project }}/package.json")/v/${{ matrix.version }}|NPM>\n$(for value in $mentions; do echo -n "<@$value> "; done) FYI" | |
| echo "SLACK_MESSAGE_HEADER=$header" >> $GITHUB_ENV | |
| - name: Send to Slack | |
| id: slack | |
| if: steps.slack-prepare.outcome == 'success' | |
| uses: ./.github/actions/slack-post-action | |
| with: | |
| channel: '#build' | |
| slack-token: ${{ secrets.SLACKBOT_TOKEN }} | |
| message: ${{ env.SLACK_MESSAGE_HEADER }} | |
| - name: Commit package-lock.json | |
| continue-on-error: true | |
| if: steps.publish.outcome == 'success' || steps.package-lock.outcome == 'success' | |
| run: | | |
| if [ -n "$(git status -s libraries/${{ matrix.project }}/package-lock.json)" ]; then | |
| git config --global user.name 'github-actions[bot]' | |
| git config --global user.email 'github-actions[bot]@users.noreply.github.com' | |
| git pull | |
| git add libraries/${{ matrix.project }}/package-lock.json | |
| skip_ci="" | |
| if [ ${{ github.ref }} == 'refs/heads/master' ]; then | |
| skip_ci="[skip ci]" | |
| fi | |
| git commit -m "GitHub Actions: Update libraries/${{ matrix.project }}/package-lock.json $skip_ci | |
| Workflow ${{ github.workflow }} ${{ github.run_number }} | |
| https://github.com/datagrok-ai/public/actions/runs/${{ github.run_id }}" | |
| count=0 | |
| retries=10 | |
| until git push; do | |
| exit=$? | |
| wait=$((2 ** count)) | |
| count=$((count + 1)) | |
| if [ $count -lt "$retries" ]; then | |
| echo "Retry $count/$retries exited $exit, retrying 'git push' in $wait seconds..." | |
| sleep $wait | |
| git pull --rebase | |
| else | |
| echo "Retry $count/$retries exited $exit, no more retries left for 'git push'." | |
| exit $exit | |
| fi | |
| done | |
| fi | |
| - name: Check packages | |
| if: ${{ matrix.packages != '' }} | |
| id: commit | |
| run: | | |
| name="$(jq -r '.name' "libraries/${{ matrix.project }}/package.json")" | |
| echo "name=$name" >> $GITHUB_OUTPUT | |
| author="$(jq -r '.author.email' "libraries/${{ matrix.project }}/package.json")" | |
| echo "author=$author" >> $GITHUB_OUTPUT | |
| crnt=$(pwd) | |
| git config --global user.name 'github-actions[bot]' | |
| git config --global user.email 'github-actions[bot]@users.noreply.github.com' | |
| git pull | |
| changed="" | |
| for package in ${{ matrix.packages }}; do | |
| cd packages/$package | |
| sed -i -e 's#"${name}": "../../libraries/${{ matrix.project }}"#"${name}": "^${{ matrix.version }}"#g' package.json | |
| if [ -n "$(git status -s package.json)" ]; then | |
| changed+="$package " | |
| npm install | |
| git add package.json | |
| git add package-lock.json | |
| git commit -m "GitHub Actions: Update package $package ${name} dependency | |
| Workflow ${{ github.workflow }} ${{ github.run_number }} | |
| https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| fi | |
| cd $crnt | |
| done | |
| echo "changed=$changed" >> $GITHUB_OUTPUT | |
| if [ -n "$(git log @{u}..)" ]; then | |
| count=0 | |
| until git push; do | |
| exit=$? | |
| wait=$((2 ** count)) | |
| count=$((count + 1)) | |
| if [ $count -lt "10" ]; then | |
| echo "Retry $count/$retries exited $exit, retrying 'git push' in $wait seconds..." | |
| sleep $wait | |
| git pull --rebase | |
| else | |
| echo "Retry $count/$retries exited $exit, no more retries left for 'git push'." | |
| exit $exit | |
| fi | |
| done | |
| fi | |
| - name: Send mail | |
| if: always() && steps.commit.outcome == 'failure' | |
| uses: dawidd6/action-send-mail@v3 | |
| with: | |
| server_address: smtp.mailgun.org | |
| server_port: 587 | |
| secure: true | |
| username: ${{ secrets.MAIL_USERNAME }} | |
| password: ${{ secrets.MAIL_PASSWORD }} | |
| subject: "Github Actions: Failed update for library ${{ matrix.project }}" | |
| to: ${{ steps.commit.outputs.author }} | |
| from: [email protected] | |
| body: "Failed to update library ${{ matrix.project }} (${{ steps.commit.outputs.name }}) for packages ${{ steps.commit.outputs.changed }}. Update the dependency manually to version '^${{ needs.version.outputs.current_version }}'\nFailed in Workflow ${{ github.workflow }} ${{ github.run_number }}: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| reply_to: [email protected] | |
| # - name: Slack Failed Notification | |
| # if: failure() | |
| # uses: act10ns/[email protected] | |
| # with: | |
| # status: ${{ job.status }} | |
| # steps: ${{ toJson(steps) }} | |
| # config: ".github/config/slack.yml" | |
| # webhook-url: ${{ secrets.SLACK_WEBHOOK_REPORT }} |