Release #273
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release | |
| on: | |
| push: | |
| tags: | |
| - "v*" | |
| schedule: | |
| # Release at 00:00 UTC+8 | |
| - cron: "0 16 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| tags: | |
| description: The tags to be released | |
| required: false | |
| type: string | |
| permissions: | |
| id-token: write | |
| contents: write | |
| jobs: | |
| create_release: | |
| name: create release | |
| runs-on: ubuntu-latest | |
| outputs: | |
| version: ${{ steps.generated-tag.outputs.tag }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get latest tag | |
| id: get-latest-tag | |
| run: | | |
| echo "tag=`gh release list -L 1 | cut -f 1`" >> $GITHUB_OUTPUT | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Bump version | |
| id: generated-tag | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| if (context.ref.startsWith("refs/tags/")) { | |
| let tag = context.ref.replace("refs/tags/", ""); | |
| core.setOutput('tag', tag); | |
| console.log(`This event pushed a tag ${tag}, return directly.`) | |
| return | |
| } | |
| if ("${{ github.event.inputs.tags }}") { | |
| let tag = "${{ github.event.inputs.tags }}"; | |
| core.setOutput('tag', tag); | |
| console.log(`This event triggered by workflow_dispatch with a tag ${tag}, return directly.`) | |
| return | |
| } | |
| let tag = "${{ steps.get-latest-tag.outputs.tag }}"; | |
| let result = /v(\d+)\.(\d+)\.(\d+)/g.exec(tag); | |
| if (result === null) { | |
| throw `The previous tag ${{ steps.get-latest-tag.outputs.tag }} is invalid, ignoring`; | |
| } | |
| let major = result[1]; | |
| let minor = result[2]; | |
| let patch = (parseInt(result[3]) + 1).toString(); | |
| let next_tag = `v${major}.${minor}.${patch}-nightly`; | |
| console.log(`This event is triggered, return generated ${next_tag}.`) | |
| core.setOutput('tag', next_tag) | |
| - name: Create github release if not exist | |
| # Allow this action failure | |
| continue-on-error: true | |
| # Reference: https://cli.github.com/manual/gh_release_create | |
| run: | | |
| echo "Create a release for ${{ steps.generated-tag.outputs.tag }}" | |
| gh release create ${{ steps.generated-tag.outputs.tag }} --generate-notes -p | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Trigger benchmark | |
| continue-on-error: true | |
| run: | | |
| echo "Trigger benchmark for ${{ steps.generated-tag.outputs.tag }}" | |
| gh workflow run benchmark.yml -f tag=${{ steps.generated-tag.outputs.tag }} | |
| env: | |
| GH_TOKEN: ${{ secrets.DATABEND_BOT_TOKEN }} | |
| publish_macos: | |
| name: macos assets | |
| runs-on: macos-11 | |
| needs: [create_release] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: | |
| - x86_64 | |
| - aarch64 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get target | |
| id: target | |
| run: echo 'target=${{ matrix.arch }}-apple-darwin' >> $GITHUB_OUTPUT | |
| - name: Rust setup | |
| run: | | |
| bash ./scripts/setup/dev_setup.sh -yb | |
| - name: Install coreutils for macOS sha256sum | |
| run: brew install coreutils | |
| - name: Cross setup | |
| if: matrix.arch == 'aarch64' | |
| run: | | |
| rustup target add aarch64-apple-darwin | |
| echo "JEMALLOC_SYS_WITH_LG_PAGE=14" >> $GITHUB_ENV | |
| - name: Build Binary | |
| run: | | |
| cargo build --release --target=${{ steps.target.outputs.target }} | |
| - name: Pack binaries | |
| run: | | |
| brew install gnu-tar | |
| sudo /usr/sbin/purge | |
| target=${{ steps.target.outputs.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| mkdir -p release/${target}/{bin,configs,scripts} | |
| cp ./target/${target}/release/databend-* release/${target}/bin/ | |
| rm -f release/${target}/bin/*.d | |
| cp ./scripts/distribution/configs/databend-* release/${target}/configs/ | |
| cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt | |
| cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/ | |
| gtar -C ./release/${target} -czvf databend-${version}-${target}.tar.gz bin configs scripts readme.txt | |
| - name: generate sha256sums | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt | |
| - name: post sha256 | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sha256sums | |
| path: sha256-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt | |
| retention-days: 1 | |
| - name: Publish Binaries | |
| uses: ./.github/actions/publish_binary | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| version: ${{ needs.create_release.outputs.version }} | |
| target: ${{ steps.target.outputs.target }} | |
| repo_role_arn: ${{ secrets.REPO_ROLE_ARN }} | |
| publish_linux: | |
| name: linux assets | |
| runs-on: ubuntu-latest | |
| needs: [create_release] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: | |
| - x86_64 | |
| - aarch64 | |
| platform: | |
| - gnu | |
| - musl | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get the version | |
| id: get_version | |
| run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT | |
| - name: Get target | |
| id: target | |
| run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.platform }}' >> $GITHUB_OUTPUT | |
| - name: Build Release | |
| uses: ./.github/actions/build_linux | |
| with: | |
| profile: release | |
| target: ${{ steps.target.outputs.target }} | |
| upload: false | |
| - name: Pack binaries | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| mkdir -p release/${target}/{bin,configs,systemd,scripts} | |
| cp ./target/${target}/release/databend-* release/${target}/bin/ | |
| rm -f release/${target}/bin/*.d | |
| cp ./scripts/distribution/systemd/databend-* release/${target}/systemd/ | |
| cp ./scripts/distribution/configs/databend-* release/${target}/configs/ | |
| cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt | |
| cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/ | |
| cp -r ./scripts/distribution/package-scripts/* release/${target}/scripts/ | |
| tar -C ./release/${target} -czvf databend-${version}-${target}.tar.gz bin configs systemd scripts readme.txt | |
| - name: generate sha256sums | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt | |
| - name: post sha256 | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sha256sums | |
| path: sha256-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt | |
| retention-days: 1 | |
| - name: Publish Binaries | |
| uses: ./.github/actions/publish_binary | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| version: ${{ needs.create_release.outputs.version }} | |
| target: ${{ steps.target.outputs.target }} | |
| repo_role_arn: ${{ secrets.REPO_ROLE_ARN }} | |
| publish_hive: | |
| name: hive assets | |
| runs-on: ubuntu-latest | |
| needs: [create_release] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: | |
| - x86_64 | |
| platform: | |
| - gnu | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get the version | |
| id: get_version | |
| run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT | |
| - name: Get target | |
| id: target | |
| run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.platform }}' >> $GITHUB_OUTPUT | |
| - name: Setup Build Tool | |
| uses: ./.github/actions/setup_build_tool | |
| with: | |
| image: ${{ steps.target.outputs.target }} | |
| - name: Build Binary | |
| run: | | |
| cargo build --release --target=${{ steps.target.outputs.target }} --features hive | |
| - name: Copyobj zlib for gnu binaries | |
| if: matrix.platform == 'gnu' | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| build-tool /usr/bin/${{ matrix.arch }}-linux-gnu-objcopy --compress-debug-sections=zlib-gnu ./target/${target}/release/databend-query | |
| build-tool /usr/bin/${{ matrix.arch }}-linux-gnu-objcopy --compress-debug-sections=zlib-gnu ./target/${target}/release/databend-meta | |
| build-tool /usr/bin/${{ matrix.arch }}-linux-gnu-objcopy --compress-debug-sections=zlib-gnu ./target/${target}/release/databend-metactl | |
| - name: Pack binaries | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| mkdir -p release/${target}/{bin,configs,systemd,scripts} | |
| cp ./target/${target}/release/databend-* release/${target}/bin/ | |
| rm -f release/${target}/bin/*.d | |
| cp ./scripts/distribution/systemd/databend-* release/${target}/systemd/ | |
| cp ./scripts/distribution/configs/databend-* release/${target}/configs/ | |
| cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt | |
| cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/ | |
| cp -r ./scripts/distribution/package-scripts/* release/${target}/scripts/ | |
| tar -C ./release/${target} -czvf databend-hive-${version}-${target}.tar.gz bin configs systemd scripts readme.txt | |
| - name: generate sha256sums | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| sha256sum databend-hive-${version}-${target}.tar.gz >> sha256-hive-${version}-${target}.txt | |
| - name: post sha256 | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sha256sums | |
| path: sha256-hive-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt | |
| retention-days: 1 | |
| - name: Publish Binaries | |
| uses: ./.github/actions/publish_binary | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| version: ${{ needs.create_release.outputs.version }} | |
| target: ${{ steps.target.outputs.target }} | |
| repo_role_arn: ${{ secrets.REPO_ROLE_ARN }} | |
| mode: hive | |
| publish_sqllogic_testsuites: | |
| name: sqllogic testsuites | |
| runs-on: ubuntu-latest | |
| needs: [create_release] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Get the version | |
| id: get_version | |
| run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT | |
| - name: Upload to github release | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| version=${{ needs.create_release.outputs.version }} | |
| tar -C ./tests/sqllogictests -czvf testsuites-${version}.tar.gz suites | |
| gh release upload ${version} testsuites-${version}.tar.gz --clobber | |
| release_docker_combined: | |
| name: docker image combined | |
| runs-on: ubuntu-latest | |
| needs: [create_release, publish_linux] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Get the version | |
| id: get_version | |
| run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v1 | |
| - uses: ./.github/actions/setup_docker | |
| id: login | |
| with: | |
| repo: databend | |
| ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }} | |
| dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }} | |
| dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Download binaries for usage | |
| id: download_binaries | |
| run: | | |
| version="${{ needs.create_release.outputs.version }}" | |
| declare -A platform_targets=( ["arm64"]="aarch64-unknown-linux-gnu" ["amd64"]="x86_64-unknown-linux-gnu") | |
| mkdir -p ./distro/ | |
| for platform in ${!platform_targets[@]}; do | |
| target=${platform_targets[$platform]} | |
| wget -P distro -qc https://repo.databend.rs/databend/${version}/databend-${version}-${target}.tar.gz | |
| mkdir -p ./target/${target}/release | |
| tar x -C ./target/${target}/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/ | |
| mkdir -p ./distro/linux/${platform} | |
| cp ./target/${target}/release/databend-* ./distro/linux/${platform} | |
| done | |
| - name: Build and push | |
| id: docker_build | |
| uses: docker/build-push-action@v3 | |
| with: | |
| push: true | |
| tags: | | |
| ${{ steps.login.outputs.dockerhub_repo }}:latest | |
| ${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }} | |
| ${{ steps.login.outputs.ecr_repo }}:latest | |
| ${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }} | |
| platforms: linux/amd64,linux/arm64 | |
| context: . | |
| file: ./docker/Dockerfile | |
| build-args: VERSION=${{ needs.create_release.outputs.version }} | |
| - name: Update repo description | |
| uses: peter-evans/dockerhub-description@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| repository: ${{ steps.login.outputs.dockerhub_repo }} | |
| short-description: ${{ github.event.repository.description }} | |
| readme-filepath: ./docker/README.md | |
| release_docker_separate: | |
| name: docker image seperated | |
| runs-on: ubuntu-latest | |
| needs: [create_release, publish_linux] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| service: | |
| - meta | |
| - query | |
| distro: | |
| - debian | |
| - distroless | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Get the version | |
| id: get_version | |
| run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v1 | |
| - uses: ./.github/actions/setup_docker | |
| id: login | |
| with: | |
| repo: databend-${{ matrix.service }} | |
| ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }} | |
| dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }} | |
| dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Download binaries for usage | |
| id: download_binaries | |
| run: | | |
| version="${{ needs.create_release.outputs.version }}" | |
| declare -A platform_targets=( ["arm64"]="aarch64-unknown-linux-gnu" ["amd64"]="x86_64-unknown-linux-gnu") | |
| mkdir -p ./distro/ | |
| for platform in ${!platform_targets[@]}; do | |
| target=${platform_targets[$platform]} | |
| wget -P distro -qc https://repo.databend.rs/databend/${version}/databend-${version}-${target}.tar.gz | |
| mkdir -p ./target/${target}/release | |
| tar x -C ./target/${target}/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/ | |
| mkdir -p ./distro/linux/${platform} | |
| cp ./target/${target}/release/databend-* ./distro/linux/${platform} | |
| done | |
| - name: get image tags | |
| id: get_image_tags | |
| shell: bash | |
| run: | | |
| _tags="${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}-${{ matrix.distro }}" | |
| _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}-${{ matrix.distro }}" | |
| if [[ "${{ matrix.distro }}" == "debian" ]]; then | |
| _tags="${_tags},${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}" | |
| _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}" | |
| _tags="${_tags},${{ steps.login.outputs.dockerhub_repo }}:latest" | |
| _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:latest" | |
| fi | |
| echo "IMAGE_TAGS=${_tags}" >> $GITHUB_OUTPUT | |
| - name: push service image | |
| uses: docker/build-push-action@v3 | |
| with: | |
| push: true | |
| tags: ${{ steps.get_image_tags.outputs.IMAGE_TAGS }} | |
| platforms: linux/amd64,linux/arm64 | |
| context: . | |
| file: ./docker/${{ matrix.distro }}/${{ matrix.service }}.Dockerfile | |
| build-args: VERSION=${{ needs.create_release.outputs.version }} | |
| release_packages: | |
| name: release packages | |
| runs-on: ubuntu-latest | |
| needs: [create_release, publish_linux] | |
| strategy: | |
| matrix: | |
| arch: | |
| - x86_64 | |
| - aarch64 | |
| packager: | |
| - deb | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Install nfpm@latest | |
| run: | | |
| curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz | |
| tar xf nfpm.tar.gz | |
| sudo mv nfpm /usr/local/bin | |
| sudo chmod a+x /usr/local/bin/nfpm | |
| rm nfpm.tar.gz | |
| - name: Get target | |
| id: target | |
| run: | | |
| echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT | |
| - name: Download binaries for usage | |
| id: download_binaries | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| version="${{ needs.create_release.outputs.version }}" | |
| mkdir -p ./distro/ | |
| wget -P distro -qc https://repo.databend.rs/databend/${version}/databend-${version}-${target}.tar.gz | |
| tar x -C distro -f ./distro/databend-${version}-${target}.tar.gz | |
| - name: Build Packages | |
| id: build_packages | |
| run: | | |
| export name="databend" | |
| export version="${{ needs.create_release.outputs.version }}" | |
| export path="distro" | |
| case "${{ matrix.arch }}" in | |
| x86_64) | |
| export arch="amd64" | |
| ;; | |
| aarch64) | |
| export arch="arm64" | |
| ;; | |
| esac | |
| nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml) | |
| - name: Update release to github | |
| shell: bash | |
| # Reference: https://cli.github.com/manual/gh_release_upload | |
| run: | | |
| version="${{ needs.create_release.outputs.version }}" | |
| # name looks like: `databend_0.8.144~nightly_amd64.deb` | |
| gh release upload ${version} databend_*.${{ matrix.packager }} --clobber | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| publish_deb: | |
| name: publish deb | |
| runs-on: ubuntu-latest | |
| needs: [create_release, release_packages] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: ./.github/actions/publish_deb | |
| with: | |
| version: ${{ needs.create_release.outputs.version }} | |
| repo_role_arn: ${{ secrets.REPO_ROLE_ARN }} | |
| gpg_signing_key: ${{ secrets.GPG_KEY_DEB }} | |
| upload-sha256sums-plugin: | |
| needs: [create_release, publish_linux, publish_macos, release_packages] | |
| runs-on: ubuntu-latest | |
| name: upload-sha256sums | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Get Pre-release | |
| uses: cardinalby/git-get-release-action@v1 | |
| id: latest_pre_release | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| prerelease: true | |
| latest: true | |
| - name: download sha256sums | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: sha256sums | |
| - shell: bash | |
| run: | | |
| for file in *.txt | |
| do | |
| cat ${file} >> sha256sums.txt | |
| done | |
| - name: upload checksums | |
| uses: actions/[email protected] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| upload_url: ${{ steps.latest_pre_release.outputs.upload_url }} | |
| asset_path: sha256sums.txt | |
| asset_name: sha256sums.txt | |
| asset_content_type: text/plain | |
| notify_build_result: | |
| name: notify build result | |
| runs-on: ubuntu-latest | |
| if: always() | |
| needs: | |
| - create_release | |
| - publish_linux | |
| - publish_macos | |
| - release_docker_combined | |
| - release_docker_separate | |
| - release_packages | |
| - publish_deb | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: | | |
| status="${{ (contains(needs.*.result, 'failure') && 'failure') || (contains(needs.*.result, 'cancelled') && 'cancelled') || 'success' }}" | |
| jq -n -f .github/release-report.jq \ | |
| --arg title "[Release] ${{ needs.create_release.outputs.version }}" \ | |
| --arg content "Build result: ${status}" \ | |
| --arg link "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ | |
| > /tmp/release-report.json | |
| curl -X POST ${{ secrets.RELEASE_REPORT_WEBHOOK }} \ | |
| -H 'Content-Type: application/json' \ | |
| -H 'cache-control: no-cache' \ | |
| -d @/tmp/release-report.json |