Merge remote-tracking branch 'origin/main' into new-hashtable

databendlabs · Oct 26, 2022 · 74eb282 · 74eb282
2 parents 3c3d00b + 11035e7
commit 74eb282
Show file tree

Hide file tree

Showing 709 changed files with 18,734 additions and 9,379 deletions.
diff --git a/.github/actions/artifact_download/action.yml b/.github/actions/artifact_download/action.yml
@@ -18,6 +18,13 @@ inputs:
     description: "default to ./target/{inputs.profile}/"
     required: false
     default: ""
+outputs:
+  path:
+    description: ""
+    value: ${{ steps.info.outputs.path }}
+  src:
+    description: ""
+    value: ${{ steps.info.outputs.src }}
 runs:
   using: "composite"
   steps:
@@ -30,14 +37,14 @@ runs:
         else
           path="${{ inputs.path }}"
         fi
-        echo "::set-output name=path::${path}"
+        echo "path=${path}" >> $GITHUB_OUTPUT
 
         if [[ "${{ inputs.profile }}" == "debug" && "${{ inputs.target }}" =~ "linux" ]]; then
           src="s3"
         else
           src="github"
         fi
-        echo "::set-output name=src::${src}"
+        echo "src=${src}" >> $GITHUB_OUTPUT
 
     - uses: actions/download-artifact@v2
       if: steps.info.outputs.src == 'github'

diff --git a/.github/actions/artifact_upload/action.yml b/.github/actions/artifact_upload/action.yml
@@ -14,26 +14,23 @@ inputs:
     description: "default/hive"
     required: false
     default: default
+outputs:
+  src:
+    description: ""
+    value: ${{ steps.info.outputs.src }}
 runs:
   using: "composite"
   steps:
     - name: Get Upload Info
       id: info
       shell: bash
       run: |
-        if [[ -z "${{ inputs.path }}" ]]; then
-          path="./target/${{ inputs.profile }}"
-        else
-          path="${{ inputs.path }}"
-        fi
-        echo "::set-output name=path::${path}"
-
         if [[ "${{ inputs.profile }}" == "debug" && "${{ inputs.target }}" =~ "linux" ]]; then
           src="s3"
         else
           src="github"
         fi
-        echo "::set-output name=src::${src}"
+        echo "src=${src}" >> $GITHUB_OUTPUT
 
     - name: Upload artifact to github
       if: steps.info.outputs.src == 'github'

diff --git a/.github/actions/check/action.yml b/.github/actions/check/action.yml
@@ -26,20 +26,19 @@ runs:
       shell: bash
       run: cargo fmt --all -- --check
 
-    - name: Clippy
+    - name: Check unused deps
       shell: bash
-      run: cargo -Z sparse-registry clippy --workspace --all-targets --all-features -- -D warnings
+      run: cargo -Z sparse-registry machete
+
+    - name: Check toml format
+      shell: bash
+      run: taplo fmt --check
 
     - name: Audit dependencies
       shell: bash
       if: "!contains(github.event.head_commit.message, 'skip audit')"
       run: cargo -Z sparse-registry audit --db ./target/advisory-db
 
-    - name: Check udeps
-      shell: bash
-      if: "!contains(github.event.head_commit.message, 'skip udeps')"
-      run: cargo -Z sparse-registry udeps --workspace
-
-    - name: Check toml format
+    - name: Clippy
       shell: bash
-      run: taplo fmt --check
+      run: cargo -Z sparse-registry clippy --workspace --all-targets --all-features -- -D warnings
diff --git a/.github/actions/setup_docker/action.yml b/.github/actions/setup_docker/action.yml
@@ -61,6 +61,6 @@ runs:
       shell: bash
       id: registry
       run: |
-        echo "::set-output name=dockerhub::datafuselabs/${{ inputs.repo }}"
-        echo "::set-output name=ecr::public.ecr.aws/i7g1w5q7/${{ inputs.repo }}"
-        echo "::set-output name=acr::registry.cn-beijing.aliyuncs.com/datafuselabs/${{ inputs.repo }}"
+        echo 'dockerhub=datafuselabs/${{ inputs.repo }}' >> $GITHUB_OUTPUT
+        echo 'ecr=public.ecr.aws/i7g1w5q7/${{ inputs.repo }}' >> $GITHUB_OUTPUT
+        echo 'acr=registry.cn-beijing.aliyuncs.com/datafuselabs/${{ inputs.repo }}' >> $GITHUB_OUTPUT
diff --git a/.github/actions/test_metactl/action.yml b/.github/actions/test_metactl/action.yml
@@ -23,3 +23,8 @@ runs:
       shell: bash
       run: |
         bash ./tests/metactl/test-metactl.sh
+
+    - name: Test metactl restore new cluster
+      shell: bash
+      run: |
+        bash ./tests/metactl/test-metactl-restore-new-cluster.sh
diff --git a/.github/actions/test_sqllogic_cluster_linux/action.yml b/.github/actions/test_sqllogic_cluster_linux/action.yml
@@ -29,6 +29,8 @@ runs:
         docker run --rm --tty --net=host \
           --user $(id -u):$(id -g) \
           --env BUILD_PROFILE \
+          --ulimit nofile=65535:65535 \
+          --ulimit stack=16777216:16777216 \
           --volume "${PWD}:/workspace" \
           --workdir "/workspace" \
           datafuselabs/build-tool:sqllogic \

diff --git a/.github/actions/test_sqllogic_standalone_linux/action.yml b/.github/actions/test_sqllogic_standalone_linux/action.yml
@@ -29,6 +29,8 @@ runs:
         docker run --rm --tty --net=host \
           --user $(id -u):$(id -g) \
           --env BUILD_PROFILE \
+          --ulimit nofile=65535:65535 \
+          --ulimit stack=16777216:16777216 \
           --volume "${PWD}:/workspace" \
           --workdir "/workspace" \
           datafuselabs/build-tool:sqllogic \

diff --git a/.github/mergify.yml b/.github/mergify.yml
@@ -7,13 +7,17 @@ queue_rules:
       # Reference: https://docs.mergify.com/conditions/#validating-all-status-checks
       # We only require linux checks to pass
       - check-success=check
-      - check-success~=^build_(aarch64|x86_64)_musl$
+      - check-success=build_aarch64_musl
+      - check-success=build_x86_64_musl
       - check-success=test_unit
       - check-success=test_metactl
       - check-success=fuse_compat
       - check-success=test_compat
       # state tests
-      - check-success~=^test_(stateless|stateful)_(standalone|cluster)_linux$
+      - check-success=test_stateless_standalone_linux
+      - check-success=test_stateless_cluster_linux
+      - check-success=test_stateful_standalone_linux
+      - check-success=test_stateful_cluster_linux
       # sqllogic tests
       - check-success=test_sqllogic_base_standalone_linux
       - check-success=test_sqllogic_ydb_standalone_linux

diff --git a/.github/workflows/build-tool.yml b/.github/workflows/build-tool.yml
@@ -30,7 +30,7 @@ jobs:
         id: toolchain
         run: |
           version=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' scripts/setup/rust-toolchain.toml)
-          echo "::set-output name=TOOLCHAIN::${version}"
+          echo "TOOLCHAIN=${version}" >> $GITHUB_OUTPUT
 
       - name: Build and publish databend build base image
         uses: docker/build-push-action@v3
@@ -71,7 +71,7 @@ jobs:
         id: toolchain
         run: |
           version=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' scripts/setup/rust-toolchain.toml)
-          echo "::set-output name=TOOLCHAIN::${version}"
+          echo "TOOLCHAIN=${version}" >> $GITHUB_OUTPUT
 
       - name: Build and publish databend build image
         uses: docker/build-push-action@v3
@@ -107,7 +107,7 @@ jobs:
         id: toolchain
         run: |
           version=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' scripts/setup/rust-toolchain.toml)
-          echo "::set-output name=TOOLCHAIN::${version}"
+          echo "TOOLCHAIN=${version}" >> $GITHUB_OUTPUT
 
       - name: Build and publish databend build base image
         uses: docker/build-push-action@v3

diff --git a/.github/workflows/databend-release.yml b/.github/workflows/databend-release.yml
@@ -32,7 +32,7 @@ jobs:
       - name: Get latest tag
         id: get-latest-tag
         run: |
-          echo "::set-output name=tag::`gh release list -L 1 | cut -f 1`"
+          echo "tag=`gh release list -L 1 | cut -f 1`" >> $GITHUB_OUTPUT
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Bump version
@@ -94,7 +94,7 @@ jobs:
           fetch-depth: 0
       - name: Get target
         id: target
-        run: echo ::set-output name=target::${{ matrix.arch }}-apple-darwin
+        run: echo 'target=${{ matrix.arch }}-apple-darwin' >> $GITHUB_OUTPUT
       - name: Rust setup
         run: |
           bash ./scripts/setup/dev_setup.sh -yb
@@ -161,10 +161,10 @@ jobs:
           fetch-depth: 0
       - name: Get the version
         id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+        run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
       - name: Get target
         id: target
-        run: echo ::set-output name=target::${{ matrix.arch }}-unknown-linux-${{ matrix.platform }}
+        run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.platform }}' >> $GITHUB_OUTPUT
       - name: Setup Build Tool
         uses: ./.github/actions/setup_build_tool
         with:
@@ -224,7 +224,7 @@ jobs:
         uses: actions/checkout@v3
       - name: Get the version
         id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+        run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
       - name: Upload to github release
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -243,7 +243,7 @@ jobs:
 
       - name: Get the version
         id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+        run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
@@ -298,7 +298,7 @@ jobs:
 
       - name: Get the version
         id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+        run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
@@ -342,7 +342,7 @@ jobs:
             _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:latest"
             _tags="${_tags},${{ steps.login.outputs.acr_repo }}:latest"
           fi
-          echo ::set-output name=IMAGE_TAGS::${_tags}
+          echo "IMAGE_TAGS=${_tags}" >> $GITHUB_OUTPUT
 
       - name: push service image
         uses: docker/build-push-action@v3