Chembl: Moved databases demo from Chem to Chembl, version up #3063
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Packages | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| packages: | |
| description: 'Packages separated by spaces' | |
| required: true | |
| type: string | |
| push: | |
| paths: | |
| - 'packages/**' | |
| pull_request: | |
| paths: | |
| - 'packages/**' | |
| jobs: | |
| common-check: | |
| name: Common checks | |
| uses: ./.github/workflows/common_check.yaml | |
| with: | |
| run_trigger: ${{ github.event_name }} | |
| matrix: | |
| name: Check changes | |
| runs-on: ubuntu-22.04 | |
| needs: common-check | |
| if: needs.common-check.outputs.continue == 'true' | |
| outputs: | |
| matrix_publish: ${{ steps.generate-matrix.outputs.matrix_publish }} | |
| publish: ${{ steps.generate-matrix.outputs.publish }} | |
| permissions: | |
| pull-requests: read | |
| steps: | |
| - name: Set git fetch depth | |
| run: | | |
| if [[ ${{ github.event_name }} != 'pull_request' ]]; then | |
| echo "FETCH_DEPTH=2" >> "$GITHUB_ENV" | |
| else | |
| echo "FETCH_DEPTH=0" >> "$GITHUB_ENV" | |
| fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ env.FETCH_DEPTH }} | |
| sparse-checkout: | | |
| ./packages | |
| - name: Get changed files | |
| uses: tj-actions/changed-files@v42 | |
| id: changed-files | |
| with: | |
| files: 'packages/**' | |
| separator: "\n" | |
| dir_names: true | |
| safe_output: false | |
| - name: Generate matrix | |
| id: generate-matrix | |
| env: | |
| ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} | |
| run: | | |
| if [[ "${{ github.event.inputs.packages }}" == "" ]]; then | |
| CHANGED_PACKAGES="$(echo -e "${ALL_CHANGED_FILES}" | awk -F'/' '{print $2}' | sort -u)" | |
| else | |
| CHANGED_PACKAGES="${{ github.event.inputs.packages }}" | |
| fi | |
| MATRIX_PUBLISH_JSON="[" | |
| for PACKAGE in $(echo ${CHANGED_PACKAGES} | sort -u); do | |
| DIR="packages/${PACKAGE}" | |
| if [ -f "${DIR}/package.json" ]; then | |
| MATRIX_PUBLISH_JSON+="{\"package\": \"${PACKAGE}\"" | |
| current_version="$(jq -r '.version' "${DIR}/package.json")" | |
| MATRIX_PUBLISH_JSON+=", \"version\": \"${current_version}\"" | |
| scripts="$(jq '. | select( has("scripts") == true ).scripts' "${DIR}/package.json")" | |
| dependencies="$(jq '(. | select( has("dependencies") == true ).dependencies) * (. | select( has("devDependencies") == true ).devDependencies)' "${DIR}/package.json")" | |
| job_name='Check' | |
| if [ ! -z "$(jq -r '. | select( has("build") == true )' <<< "$scripts")" ]; then | |
| MATRIX_PUBLISH_JSON+=", \"build\": \"build\"" | |
| job_name='Build' | |
| fi | |
| unpublished_deps="$(jq -r '. | to_entries | map(select(.value | match("\\.\\./.*")))[] | "\(.key)=\(.value)"' <<<$dependencies | tr '\n' ' ')" | |
| job_test_env='' | |
| if [ -n "$(jq '. | select( has("test") == true )' <<< "$scripts")" ] && \ | |
| [[ "$(jq -r '. | select( has("skipCI") == true ).skipCI' "${DIR}/package.json")" != "true" ]] && \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${PACKAGE})" != "docking" ]]; then | |
| MATRIX_PUBLISH_JSON+=", \"test\": \"test\"" | |
| job_name='Build, test' | |
| job_test_env='Datagrok' | |
| grok_deps="$(jq -r '. | select( has("devDependencies") == true ).devDependencies | to_entries[] | .key | select(test("@datagrok/.*")?)' ${DIR}/package.json)" | |
| if [[ "$(tr '[:upper:]' '[:lower:]' <<<${PACKAGE})" == "chem" ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${PACKAGE})" == "simpkpd" ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${PACKAGE})" == "dendrogram" ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${PACKAGE})" == "cvmtests" ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${PACKAGE})" == "samples" ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${PACKAGE})" == "admetica" ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"chem"* ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"simpkpd"* ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"cvmtests"* ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"dendrogram"* ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"admetica"* ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"samples"* ]]; then | |
| job_test_env+=',JKG' | |
| fi | |
| if [ -d "${DIR}/dockerfiles" ]; then | |
| job_test_env+=',Spawner' | |
| fi | |
| if [[ "${unpublished_deps}" == "" ]] && [[ "$(tr '[:upper:]' '[:lower:]' <<<${PACKAGE})" != *"tests"* ]]; then | |
| job_test_env+=' latest version' | |
| else | |
| job_test_env+=' bleeding-edge version' | |
| fi | |
| fi | |
| runner='ubuntu-22.04' | |
| if [[ "${job_test_env}" == *"JKG"* ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${PACKAGE})" == "admetica" ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"admetica"* ]]; then | |
| runner='ubuntu-latest-m' | |
| fi | |
| name="$(jq -r .name "${DIR}/package.json")" | |
| npm_version="$(curl --retry 3 -s "https://registry.npmjs.org/${name}/${current_version}" | jq -r '.? | select( has("version") == true ).version')" | |
| MATRIX_PUBLISH_JSON+=", \"unpublished_deps\": \"$unpublished_deps\"" | |
| if [[ "${{ github.event_name }}" != "pull_request" ]]; then | |
| if [[ $npm_version != ${current_version} ]]; then | |
| if [ "$(awk -F. '{print $1}' <<<$current_version)" -gt "0" ]; then | |
| if [[ $unpublished_deps == "" ]]; then | |
| if [[ '${{ github.ref }}' == 'refs/heads/master' ]]; then | |
| publish="publish" | |
| MATRIX_PUBLISH_JSON+=", \"publish\": \"publish\"" | |
| job_name+=' and publish to NPM' | |
| else | |
| echo "It is an action for branch other than 'master'. Publish will be skipped." | |
| echo "::notice title=${PACKAGE}::It is an action for branch other than 'master'. Publish will be skipped." | |
| fi | |
| else | |
| echo "Package $PACKAGE version ${current_version} has unpublished dependencies: $unpublished_deps. It is not going to be published" | |
| echo "::notice title=${PACKAGE}::Version ${current_version} has unpublished dependencies and is not going to be published" | |
| fi | |
| else | |
| MATRIX_PUBLISH_JSON+=", \"beta\": \"beta\"" | |
| echo "Package $PACKAGE version ${current_version} is under 1.0.0 and is not going to be published" | |
| echo "::notice title=${PACKAGE}::Version ${current_version} is under 1.0.0 and is not going to be published" | |
| fi | |
| else | |
| echo "Package $PACKAGE version ${current_version} already published" | |
| echo "::notice title=${PACKAGE}::Version ${current_version} is already published" | |
| fi | |
| else | |
| echo "It is an actions for PR. Publish will be skipped." | |
| echo "::notice title=${PACKAGE}::It is an actions for PR. Publish will be skipped." | |
| fi | |
| if [[ -d "${DIR}/dockerfiles" ]]; then | |
| build_docker=false | |
| if [[ $publish == "publish" ]] || [[ "${{ github.event.inputs.packages }}" != "" ]]; then | |
| build_docker=true | |
| else | |
| if [[ "${{ steps.changed-files.outputs.all_changed_files }}" == *"${DIR}/dockerfiles"* ]]; then | |
| build_docker=true | |
| fi | |
| fi | |
| MATRIX_PUBLISH_JSON+=", \"build_docker\": \"$build_docker\"" | |
| MATRIX_PUBLISH_JSON+=", \"docker\": \"$(find ${DIR}/dockerfiles -type f -name Dockerfile | tr '\n' ' ')\"" | |
| fi | |
| if [ -n "${job_test_env}" ]; then | |
| job_name+=": ${job_test_env}" | |
| fi | |
| MATRIX_PUBLISH_JSON+=", \"job_name\": \"${job_name}\"" | |
| MATRIX_PUBLISH_JSON+=", \"runner\": \"${runner}\"" | |
| MATRIX_PUBLISH_JSON+="}" | |
| fi | |
| done | |
| MATRIX_PUBLISH_JSON="${MATRIX_PUBLISH_JSON//\}\{/\}, \{}" | |
| MATRIX_PUBLISH_JSON+="]" | |
| PUBLISH_JSON="{\"include\": ${MATRIX_PUBLISH_JSON}}" | |
| CONTINUE_PUBLISH_JOB="no" | |
| if [[ "${MATRIX_PUBLISH_JSON}" != "[]" ]]; then | |
| CONTINUE_PUBLISH_JOB="yes" | |
| fi | |
| echo "publish=${CONTINUE_PUBLISH_JOB}" >> $GITHUB_OUTPUT | |
| echo "matrix_publish=${PUBLISH_JSON}" >> $GITHUB_OUTPUT | |
| - name: Output | |
| run: | | |
| echo -e "matrix_publish: ${{ steps.generate-matrix.outputs.matrix_publish }}" | |
| echo -e "publish: ${{ steps.generate-matrix.outputs.publish }}" | |
| publish: | |
| name: "${{ matrix.package }}: ${{ matrix.job_name }}" | |
| needs: | |
| - matrix | |
| - common-check | |
| if: needs.matrix.outputs.publish == 'yes' && needs.common-check.outputs.continue == 'true' | |
| runs-on: ${{ matrix.runner }} | |
| env: | |
| HOST: GitHubAction | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.matrix.outputs.matrix_publish) }} | |
| permissions: | |
| actions: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.WRITE_TOKEN || github.token }} | |
| sparse-checkout: | | |
| ./libraries | |
| ./packages | |
| ./js-api | |
| ./.github | |
| ./docker | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18.x' | |
| registry-url: 'https://registry.npmjs.org' | |
| scope: '@datagrok' | |
| cache: 'npm' | |
| cache-dependency-path: | | |
| packages/${{ matrix.package }}/package-lock.json | |
| - name: Upgrade npm | |
| run: npm install -g npm@x | |
| - name: npm version | |
| run: npm version | |
| - name: unpublished dependencies in package-lock.json | |
| id: package-lock | |
| if: ${{ matrix.unpublished_deps == '' }} | |
| run: grep -q '\.\./\.\./' package-lock.json && rm -f package-lock.json || true | |
| working-directory: packages/${{ matrix.package }} | |
| - name: npm install | |
| run: npm install | |
| id: install | |
| working-directory: packages/${{ matrix.package }} | |
| - name: Get build info | |
| id: info | |
| run: | | |
| commit_sha=$(echo ${{ github.sha }} | cut -c1-8) | |
| echo "commit_sha=$commit_sha" >> $GITHUB_OUTPUT | |
| - name: Run datagrok stand | |
| id: datagrok-image | |
| if: ${{ matrix.test == 'test' }} | |
| run: | | |
| echo "Find grok dependencies packages" | |
| grok_deps="$(jq -r '. | select( has("devDependencies") == true ).devDependencies | to_entries[] | .key | select(test("@datagrok/.*")?)' packages/${{ matrix.package }}/package.json)" | |
| # if [[ "$(tr '[:upper:]' '[:lower:]' <<<${{ matrix.package }})" == "chem" ]] || | |
| # [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"chem"* ]] || | |
| # [[ "$(tr '[:upper:]' '[:lower:]' <<<${{ matrix.package }})" == "ketchersketcher" ]] || | |
| # [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"ketchersketcher"* ]]; then | |
| # if [[ "${{ matrix.unpublished_deps }}" == "" ]]; then | |
| # echo 'Use test stand for tests' | |
| # apiUrl='https://test.datagrok.ai/api' | |
| # alias=test | |
| # else | |
| # echo 'Use dev stand for tests' | |
| # apiUrl='https://dev.datagrok.ai/api' | |
| # alias=dev | |
| # fi | |
| # else | |
| profiles='--profile datagrok --profile db' | |
| echo "Check if Grok Connect is required for the package" | |
| if [ -d "packages/${{ matrix.package }}/connections" ] || [[ -n $(find packages/${{ matrix.package }}/node_modules/@datagrok -type d -name connections) ]] || \ | |
| [ -d "packages/${{ matrix.package }}/queries" ] || [[ -n $(find packages/${{ matrix.package }}/node_modules/@datagrok -type d -name queries) ]]; then | |
| profiles+=' --profile grok_connect' | |
| fi | |
| echo "Check if CVM is required for the package" | |
| if [[ "$(tr '[:upper:]' '[:lower:]' <<<${{ matrix.package }})" == "chem" ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${{ matrix.package }})" == "simpkpd" ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${{ matrix.package }})" == "dendrogram" ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${{ matrix.package }})" == "cvmtests" ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${{ matrix.package }})" == "samples" ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"chem"* ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"simpkpd"* ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"cvmtests"* ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"dendrogram"* ]] || \ | |
| [[ "$(tr '[:upper:]' '[:lower:]' <<<${grok_deps})" == *"samples"* ]]; then | |
| echo "Add Scripting as dependency for package ${{ matrix.package }}" | |
| profiles+=' --profile scripting' | |
| fi | |
| echo "Check if Grok Spawner is required for the package" | |
| if [ -d "packages/${{ matrix.package }}/dockerfiles" ]; then | |
| profiles+=' --profile grok_spawner' | |
| fi | |
| echo "Run datagrok stand with profiles ${profiles}" | |
| # docker system prune -af --volumes | |
| if [[ "${{ matrix.unpublished_deps }}" == "" ]] && [[ "$(tr '[:upper:]' '[:lower:]' <<<${{ matrix.package }})" != *"tests"* ]]; then | |
| DATAGROK_VERSION='latest' | |
| else | |
| DATAGROK_VERSION='bleeding-edge' | |
| export GROK_SPAWNER_VERSION='bleeding-edge' | |
| export GROK_JUPYTER_KERNEL_GATEWAY_VERSION='bleeding-edge' | |
| fi | |
| export DATAGROK_VERSION | |
| # Skip initial setup in CI | |
| sed -i '/"dbServer": "database"/a\ \ \ \ \ \ \ \ \ \ "initialSetupCompleted": true,' docker/localhost.docker-compose.yaml | |
| docker compose -p datagrok -f "docker/localhost.docker-compose.yaml" ${profiles} up -d | |
| echo "Set docker_sha" | |
| docker_sha=$(docker images --quiet "datagrok/datagrok:$DATAGROK_VERSION") | |
| echo "docker_sha=$docker_sha" >> $GITHUB_OUTPUT | |
| echo "Notify about version used for tests" | |
| echo "::notice title=${{ matrix.package }}::datagrok/datagrok:$DATAGROK_VERSION SHA=$docker_sha docker version was used for tests" | |
| df -h | |
| apiUrl='http://127.0.0.1:8080/api' | |
| alias=${HOST} | |
| # fi | |
| echo "apiUrl=$apiUrl" >> $GITHUB_OUTPUT | |
| echo "alias=$alias" >> $GITHUB_OUTPUT | |
| - id: datagrok-tools | |
| run: npm install -g datagrok-tools@latest | |
| - if: ${{ matrix.unpublished_deps != '' }} | |
| run: grok link | |
| working-directory: packages/${{ matrix.package }} | |
| - name: webpack cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: packages/${{ matrix.package }}/node_modules/.cache/webpack/ | |
| key: ${{ matrix.package }}-${{ github.ref_name }}-webpack-build | |
| restore-keys: | | |
| ${{ matrix.package }}-master-webpack-build | |
| - name: Remove GPU from docker configuration | |
| if: matrix.docker != '' && steps.datagrok-image.outcome == 'success' | |
| run: | | |
| for dockerfile in $(echo -e ${{ matrix.docker }}); do | |
| docker_config_file="$(dirname $dockerfile)/container.json" | |
| if [ -f "${docker_config_file}" ]; then | |
| jq 'del(.gpu)' ${docker_config_file} > temp.json && mv temp.json ${docker_config_file} | |
| fi | |
| done | |
| - name: Build package | |
| if: ${{ matrix.build == 'build' }} | |
| id: build | |
| run: | | |
| if [[ $(jq -r '. | select( has("scripts") == true ).scripts | select( has("build") == true ).build' "package.json") == "webpack" ]]; then | |
| npm run build -- --mode=production | |
| else | |
| npm run build | |
| fi | |
| working-directory: packages/${{ matrix.package }} | |
| - name: Wait for Datagrok to become available | |
| if: ${{ matrix.test == 'test' }} | |
| timeout-minutes: 5 | |
| run: | | |
| until .github/scripts/check-output.sh "curl -s ${{ steps.datagrok-image.outputs.apiUrl }}/info/server" '"Http Server"' | |
| do | |
| sleep 1 | |
| if [[ "${{ steps.datagrok-image.outputs.apiUrl }}" == *"127.0.0.1"* ]]; then | |
| docker compose -p datagrok logs --no-color --timestamps datagrok | |
| fi | |
| echo -e "\nRetrying..." | |
| done | |
| curl -s ${{ steps.datagrok-image.outputs.apiUrl }}/info/server | |
| curl -s ${{ steps.datagrok-image.outputs.apiUrl }}/admin/health | |
| token=$(curl -s -X POST ${{ steps.datagrok-image.outputs.apiUrl }}/users/login/dev/admin | jq -r .token) | |
| curl -s -H "Authorization: $token" -H "Content-Type": "application/json" '${{ steps.datagrok-image.outputs.apiUrl }}/admin/plugins/admin/settings' -X POST -d '{"#type":"AdminPluginSettings", "agreementDate":null}' | |
| - name: Create configuration for grok command-line tool | |
| if: ${{ matrix.test == 'test' }} | |
| id: grok-config | |
| env: | |
| DEV_TEST_DEV_KEY: ${{ secrets.DEV_TEST_DEV_KEY }} | |
| TEST_TEST_DEV_KEY: ${{ secrets.TEST_TEST_DEV_KEY }} | |
| run: | | |
| if [[ "${{ steps.datagrok-image.outputs.alias }}" == "test" ]]; then | |
| key="$TEST_TEST_DEV_KEY" | |
| elif [[ "${{ steps.datagrok-image.outputs.alias }}" == "dev" ]]; then | |
| key="$DEV_TEST_DEV_KEY" | |
| else | |
| key='admin' | |
| fi | |
| grok config add --default --alias ${{ steps.datagrok-image.outputs.alias }} --server '${{ steps.datagrok-image.outputs.apiUrl }}' --key "$key" | |
| - name: Install Grok Dependencies before package publish to Datagrok | |
| if: ${{ matrix.test == 'test' }} | |
| run: | | |
| grok_deps="$(jq -r '. | select( has("devDependencies") == true ).devDependencies | to_entries[] | .key | select(test("@datagrok/.*")?)' package.json)" | |
| if [ -n "$grok_deps" ]; then | |
| for dep in $grok_deps; do | |
| current_dir=$(pwd) | |
| cd node_modules/$dep | |
| count=0 | |
| retries=5 | |
| echo "Publishing $dep to ${{ steps.datagrok-image.outputs.alias }}..." | |
| until grok publish ${{ steps.datagrok-image.outputs.alias }} --release; do | |
| exit=$? | |
| wait=$((2 ** count)) | |
| count=$((count + 1)) | |
| if [ $count -lt "$retries" ]; then | |
| echo "Retry $count/$retries exited $exit, retrying 'grok publish ${{ steps.datagrok-image.outputs.alias }}' for $dep in $wait seconds..." | |
| sleep $wait | |
| else | |
| echo "Retry $count/$retries exited $exit, no more retries left for 'grok publish ${{ steps.datagrok-image.outputs.alias }}' for $dep." | |
| exit $exit | |
| fi | |
| done | |
| cd $current_dir | |
| done | |
| fi | |
| working-directory: packages/${{ matrix.package }} | |
| - name: Publish package to Datagrok | |
| if: ${{ matrix.test == 'test' }} | |
| run: | | |
| count=0 | |
| retries=5 | |
| until grok publish ${{ steps.datagrok-image.outputs.alias }} --release; do | |
| exit=$? | |
| if [[ "${{ steps.datagrok-image.outputs.apiUrl }}" == *"127.0.0.1"* ]]; then | |
| docker compose -p datagrok logs --no-color --timestamps datagrok | |
| fi | |
| wait=$((2 ** count)) | |
| count=$((count + 1)) | |
| if [ "$count" -lt "$retries" ]; then | |
| echo "Retry $count/$retries exited $exit, retrying 'grok publish ${{ steps.datagrok-image.outputs.alias }}' in $wait seconds..." | |
| sleep $wait | |
| else | |
| echo "Retry $count/$retries exited $exit, no more retries left for 'grok publish ${{ steps.datagrok-image.outputs.alias }}'." | |
| exit $exit | |
| fi | |
| done | |
| working-directory: packages/${{ matrix.package }} | |
| - name: Set required credentials | |
| if: ${{ matrix.test == 'test' }} | |
| env: | |
| CHEMSPACE_APIKEY: ${{ secrets.CHEMSPACE_APIKEY }} | |
| run: | | |
| if [[ "${{ steps.datagrok-image.outputs.apiUrl }}" == *"127.0.0.1"* ]]; then | |
| if [[ "${{ matrix.package }}" == "Chemspace" ]]; then | |
| token=$(curl -s -X POST ${{ steps.datagrok-image.outputs.apiUrl }}/users/login/dev/admin | jq -r .token) | |
| curl -s -H "Authorization: $token" -H "Content-Type": "application/json" '${{ steps.datagrok-image.outputs.apiUrl }}/credentials/for/Chemspace.Chemspace' -X POST -d "{\"apiKey\": \"$CHEMSPACE_APIKEY\"}" | |
| fi | |
| fi | |
| - name: Wait for published docker to become available | |
| id: docker-wait | |
| if: matrix.docker != '' && steps.datagrok-image.outcome == 'success' | |
| env: | |
| DEV_TEST_DEV_KEY: ${{ secrets.DEV_TEST_DEV_KEY }} | |
| TEST_TEST_DEV_KEY: ${{ secrets.TEST_TEST_DEV_KEY }} | |
| timeout-minutes: 20 | |
| run: | | |
| check_container=false | |
| for dockerfile in $(echo -e ${{ matrix.docker }}); do | |
| docker_config_file="$(dirname $dockerfile)/container.json" | |
| if [ -f "${docker_config_file}" ]; then | |
| on_demand=$(jq '.on_demand' ${docker_config_file}) | |
| if [ "$on_demand" == "true" ]; then | |
| check_container=false | |
| else | |
| check_container=true | |
| break | |
| fi | |
| else | |
| check_container=true | |
| break | |
| fi | |
| done | |
| if [[ "${check_container}" == "true" ]] ; then | |
| if [[ "${{ steps.datagrok-image.outputs.alias }}" == "test" ]]; then | |
| key="$TEST_TEST_DEV_KEY" | |
| elif [[ "${{ steps.datagrok-image.outputs.alias }}" == "dev" ]]; then | |
| key="$DEV_TEST_DEV_KEY" | |
| else | |
| key='admin' | |
| fi | |
| token=$(curl -s -X POST ${{ steps.datagrok-image.outputs.apiUrl }}/users/login/dev/$key | jq -r .token) | |
| long_term_token=$(curl -s -X POST ${{ steps.datagrok-image.outputs.apiUrl }}/users/sessions/current/refresh -H "Authorization: ${token}" | jq -r .token) | |
| until .github/scripts/check-output.sh "curl -s -H 'Authorization: $long_term_token' '${{ steps.datagrok-image.outputs.apiUrl }}/docker/containers'" '${{ matrix.package }}' | |
| do | |
| docker compose -p datagrok -f "docker/localhost.docker-compose.yaml" --profile all exec db psql -U postgres -d datagrok -c "SELECT * FROM docker_containers;" | |
| .github/scripts/check-output.sh "curl -s -H 'Authorization: $token' '${{ steps.datagrok-image.outputs.apiUrl }}/docker/containers'" '${{ matrix.package }}' | |
| sleep 1 | |
| echo -e "\nContainer for ${{ matrix.package }} did not start yet..." | |
| echo -e "\nRetrying '/api/docker/containers'..." | |
| done | |
| if [[ "${{ steps.datagrok-image.outputs.apiUrl }}" == *"127.0.0.1"* ]]; then | |
| until .github/scripts/check-output.sh "docker ps" '${{ matrix.package }}' | |
| do | |
| sleep 1 | |
| echo -e "\nContainer for ${{ matrix.package }} did not start yet..." | |
| echo -e "\nRetrying 'docker ps'..." | |
| done | |
| fi | |
| fi | |
| - name: Test Package | |
| if: ${{ matrix.test == 'test' }} | |
| continue-on-error: ${{ contains(matrix.package, 'Tests') || matrix.package == 'Chemspace' }} | |
| timeout-minutes: 30 | |
| id: test-package | |
| run: npm run test -- --skip-build --skip-publish --record --csv --host ${{ steps.datagrok-image.outputs.alias }} --verbose | |
| working-directory: packages/${{ matrix.package }} | |
| - name: Upload Artifact | |
| if: always() && steps.test-package.outcome != 'skipped' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.package }}.test-results | |
| path: packages/${{ matrix.package }}/test-* | |
| retention-days: 14 | |
| if-no-files-found: warn | |
| - name: Create error notification | |
| if: failure() && (steps.test-package.outcome == 'failure' || steps.docker-wait.outcome == 'failure') | |
| run: | | |
| if [[ "${{ steps.datagrok-image.outputs.apiUrl }}" == *"127.0.0.1"* ]]; then | |
| docker compose -p datagrok logs --no-color --timestamps | |
| else | |
| echo 'Check dev logs' | |
| fi | |
| echo "::error title=${{ matrix.package }}: failed tests::Check job output for details" | |
| echo "status=error" >> $GITHUB_OUTPUT | |
| - name: Get docker image version | |
| if: matrix.build_docker == 'true' | |
| id: docker_name_ver | |
| working-directory: packages/${{ matrix.package }} | |
| run: | | |
| image_ver=$(jq -r .version package.json) | |
| if [[ "${{ matrix.publish }}" != "publish" ]]; then | |
| image_ver+="-${{ steps.info.outputs.commit_sha }}" | |
| fi | |
| echo "image_ver=$image_ver" >> $GITHUB_OUTPUT | |
| echo "package_name=$(jq -r .name package.json | sed -e 's,@datagrok/,,g')" >> $GITHUB_OUTPUT | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| if: steps.docker_name_ver.outcome == 'success' | |
| - name: Build package Docker image | |
| if: steps.docker_name_ver.outcome == 'success' | |
| id: build-docker | |
| run: | | |
| for dockerfile in $(echo -e ${{ matrix.docker }}); do | |
| image_name=${{ steps.docker_name_ver.outputs.package_name }} | |
| if [[ $(dirname $dockerfile) != *"dockerfiles" ]]; then | |
| image_name+="-$(dirname $dockerfile | awk -F/ '{print $NF}')" | |
| fi | |
| cache_args="--cache-from type=registry,ref=datagrok/${image_name}:cache" | |
| if [ '${{ github.ref }}' == 'refs/heads/master' ]; then | |
| cache_args+=" --cache-to type=registry,ref=datagrok/${image_name}:cache" | |
| fi | |
| docker buildx create --use | |
| docker buildx build $(dirname $dockerfile) --load \ | |
| --tag datagrok/${image_name}:${{ steps.docker_name_ver.outputs.image_ver }} \ | |
| $cache_args | |
| images+=" datagrok/${image_name}:${{ steps.docker_name_ver.outputs.image_ver }}" | |
| done | |
| echo "images=$images" >> $GITHUB_OUTPUT | |
| - name: General checks | |
| run: | | |
| if [[ "$(jq '.name' package.json | sed -E 's/(^"|"$)//g')" != "@datagrok/"* ]]; then | |
| echo "::error title=${{ matrix.package }}: failed properties check::Package should be in '@datagrok' scope. Change package name to '@datagrok/<name>' in ${{ matrix.package }}/package.json" | |
| exit 1 | |
| fi | |
| if [ ! -z "$(jq '. | select( has("beta") == true ).beta' package.json)" ]; then | |
| echo "::error title=${{ matrix.package }}: failed properties check::Remove beta property in ${{ matrix.package }}/package.json. It is deprecated." | |
| exit 1 | |
| fi | |
| id: gen_checks | |
| working-directory: packages/${{ matrix.package }} | |
| - run: grok check | |
| if: ${{ !contains(matrix.package, 'Tests') && !contains(matrix.package, 'Meta') && matrix.beta != 'true' }} | |
| id: check | |
| working-directory: packages/${{ matrix.package }} | |
| - name: Publish to npm | |
| run: npm publish --access public | |
| id: publish | |
| working-directory: packages/${{ matrix.package }} | |
| env: | |
| NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} | |
| if: matrix.publish == 'publish' && (steps.build.outcome == 'success' || steps.build.outcome == 'skipped') && steps.gen_checks.outcome == 'success' && (steps.check.outcome == 'success' || steps.check.outcome == 'skipped') && steps.install.outcome == 'success' && (steps.test-package.outcome == 'success' || steps.test-package.outcome == 'skipped' || (contains(matrix.package, 'Tests') || matrix.package == 'Chemspace')) | |
| - name: Get actors slack | |
| id: actor | |
| uses: ./.github/actions/slack-user-action | |
| with: | |
| slack-token: ${{ secrets.SLACKBOT_TOKEN }} | |
| email-mapping: ${{ secrets.EMAIL_SLACK_ID }} | |
| - name: Get owner email | |
| run: echo "owner=$(jq -r .author.email package.json)" >> $GITHUB_OUTPUT | |
| id: owner-email | |
| working-directory: packages/${{ matrix.package }} | |
| - name: Get owners slack | |
| id: owner | |
| uses: ./.github/actions/slack-user-action | |
| with: | |
| slack-token: ${{ secrets.SLACKBOT_TOKEN }} | |
| emails: ${{ steps.owner-email.outputs.owner }} | |
| - name: Get QA slack | |
| id: qa | |
| uses: ./.github/actions/slack-user-action | |
| if: ${{ (matrix.package == 'Tutorials' || matrix.package == 'Chem' || matrix.package == 'DiffStidio' || matrix.package == 'PowerPack') }} | |
| with: | |
| slack-token: ${{ secrets.SLACKBOT_TOKEN }} | |
| emails: "[email protected]" | |
| - name: Prepare Slack Message | |
| id: slack-prepare | |
| if: ${{ steps.publish.outcome == 'success' && !contains(matrix.package, 'Meta') }} | |
| shell: bash | |
| run: | | |
| mentions=$(echo -e "${{ steps.actor.outputs.user-ids }}\n${{ steps.owner.outputs.user-ids }}\n${{ steps.qa.outputs.user-ids }}" | sort -u) | |
| header="Package *${{ matrix.package }}* version *${{ matrix.version }}* published to <https://www.npmjs.com/package/$(jq -r .name "packages/${{ matrix.package }}/package.json")/v/${{ matrix.version }}|NPM>\n$(for value in $mentions; do echo -n "<@$value> "; done) FYI" | |
| echo "SLACK_MESSAGE_HEADER=$header" >> $GITHUB_ENV | |
| - name: Send to Slack | |
| id: slack | |
| if: steps.slack-prepare.outcome == 'success' | |
| uses: ./.github/actions/slack-post-action | |
| with: | |
| channel: '#build' | |
| slack-token: ${{ secrets.SLACKBOT_TOKEN }} | |
| message: ${{ env.SLACK_MESSAGE_HEADER }} | |
| - name: Save package Docker image to tar archive | |
| if: steps.build-docker.outcome == 'success' && (steps.publish.outcome == 'failure' || steps.publish.outcome == 'skipped') | |
| id: save-docker | |
| run: | | |
| docker save -o /tmp/${{ matrix.package }}.docker.tar ${{ steps.build-docker.outputs.images }} | |
| - uses: actions/upload-artifact@v4 | |
| if: steps.save-docker.outcome == 'success' | |
| with: | |
| path: /tmp/${{ matrix.package }}.docker.tar | |
| name: ${{ matrix.package }}.docker.tar | |
| retention-days: 14 | |
| if-no-files-found: warn | |
| - name: Publish package Docker image | |
| if: steps.publish.outcome == 'success' && steps.build-docker.outcome == 'success' | |
| id: publish-docker | |
| run: | | |
| for image in $(echo -e ${{ steps.build-docker.outputs.images }}); do | |
| docker push $image | |
| done | |
| - name: Commit package-lock.json | |
| id: git | |
| continue-on-error: true | |
| if: steps.publish.outcome == 'success' || steps.package-lock.outcome == 'success' | |
| run: | | |
| git config --global user.name 'github-actions[bot]' | |
| git config --global user.email 'github-actions[bot]@users.noreply.github.com' | |
| git pull | |
| continue=false | |
| if [ -n "$(git status -s packages/${{ matrix.package }}/package-lock.json)" ]; then | |
| git add packages/${{ matrix.package }}/package-lock.json | |
| skip_ci="" | |
| if [ ${{ github.ref }} == 'refs/heads/master' ]; then | |
| skip_ci="[skip ci]" | |
| fi | |
| git commit -m "GitHub Actions: Update packages/${{ matrix.package }}/package-lock.json $skip_ci | |
| Workflow ${{ github.workflow }} ${{ github.run_number }} | |
| https://github.com/datagrok-ai/public/actions/runs/${{ github.run_id }}" | |
| continue=true | |
| fi | |
| if [[ "${continue}" == "true" ]]; then | |
| count=0 | |
| retries=10 | |
| until git push; do | |
| exit=$? | |
| wait=$((2 ** count)) | |
| count=$((count + 1)) | |
| if [ $count -lt "$retries" ]; then | |
| echo "Retry $count/$retries exited $exit, retrying 'git push' in $wait seconds..." | |
| sleep $wait | |
| git pull --rebase | |
| else | |
| echo "Retry $count/$retries exited $exit, no more retries left for 'git push'." | |
| exit $exit | |
| fi | |
| done | |
| fi | |
| - name: Trigger help update | |
| continue-on-error: true | |
| if: steps.publish.outcome == 'success' | |
| run: gh workflow run docusaurus.yaml --ref ${{ github.ref }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| # - name: Slack Notification | |
| # uses: act10ns/[email protected] | |
| # with: | |
| # status: ${{ steps.publish.outcome }} | |
| # steps: ${{ toJson(steps) }} | |
| # matrix: ${{ toJson(matrix) }} | |
| # config: ".github/config/slack.yml" | |
| # webhook-url: ${{ secrets.SLACK_WEBHOOK_REPORT }} | |
| # - name: Slack Failed-Cancelled Notification | |
| # if: failure() || cancelled() | |
| # uses: act10ns/[email protected] | |
| # with: | |
| # status: ${{ job.status }} | |
| # steps: ${{ toJson(steps) }} | |
| # config: ".github/config/slack.yml" | |
| # webhook-url: ${{ secrets.SLACK_WEBHOOK_REPORT }} |