[pull] main from datafuselabs:main

.cargo/audit.toml

@@ -18,5 +18,23 @@ ignore = [
     "RUSTSEC-2021-0139",
     # buf_redux
     # buf_redux is Unmaintained
-    "RUSTSEC-2023-0028"
+    "RUSTSEC-2023-0028",
+    # rsa
+    # Marvin Attack: potential key recovery through timing sidechannels
+    "RUSTSEC-2023-0071",
+    # mach
+    # mach is Unmaintained
+    "RUSTSEC-2020-0168",
+    # atty
+    # Potential unaligned read
+    "RUSTSEC-2021-0145",
+    # borsh
+    # Parsing borsh messages with ZST which are not-copy/clone is unsound
+    "RUSTSEC-2023-0033",
+    # yaml-rust is unmaintained.
+    "RUSTSEC-2024-0320",
+    # Degradation of service in h2 servers with CONTINUATION Flood(Wait for tonic to fix)
+    "RUSTSEC-2024-0332",
+    # `rustls::ConnectionCommon::complete_io` could fall into an infinite loop based on network input
+    "RUSTSEC-2024-0336"
 ]

.gitattributes

@@ -1,3 +1,2 @@
 docker/** -text
-scripts/** -text
-website/databend/** -text
+scripts/** -text

.github/CODEOWNERS

@@ -39,10 +39,6 @@
 /src/common/http/                  @datafuselabs/databend-api-reviewer
 /src/query/service/src/servers/    @datafuselabs/databend-api-reviewer
 
-# Website
-/website/              @datafuselabs/databend-document-reviewer
-/docs/                 @datafuselabs/databend-document-reviewer
-
 # Tool
 /docker/               @datafuselabs/databend-tool-reviewer
 /scripts/              @datafuselabs/databend-tool-reviewer

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,24 @@
-I hereby agree to the terms of the CLA available at: https://databend.rs/dev/policies/cla/
+I hereby agree to the terms of the CLA available at: https://docs.databend.com/dev/policies/cla/
 
 ## Summary
 
-Summary about this PR
+_Briefly describe what this PR aims to solve. Include background context that will help reviewers understand the purpose of the PR._
 
-- Closes #issue
+- Fixes #[Link the issue here]
+
+## Tests
+
+- [ ] Unit Test
+- [ ] Logic Test
+- [ ] Benchmark Test
+- [ ] No Test  - _Explain why_
+
+## Type of change
+
+- [ ] Bug Fix (non-breaking change which fixes an issue)
+- [ ] New Feature (non-breaking change which adds functionality)
+- [ ] Breaking Change (fix or feature that could cause existing functionality not to work as expected)
+- [ ] Documentation Update
+- [ ] Refactoring
+- [ ] Performance Improvement
+- [ ] Other (please describe):

.github/actions/artifact_download/action.yml

@@ -42,7 +42,7 @@ runs:
         fi
         echo "path=${path}" >> $GITHUB_OUTPUT
 
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       if: env.RUNNER_PROVIDER == 'github'
       with:
         name: ${{ env.BUILD_PROFILE }}-${{ inputs.sha }}-${{ inputs.target }}-${{ inputs.category }}

.github/actions/artifact_failure/action.yml

@@ -19,8 +19,12 @@ runs:
             docker logs "$line" > .databend/docker/"$line".log
         done
 
+        touch nohup.out
+        cp nohup.out .databend/nohup.out
+
+        mkdir -p target
         find -type d -name .databend -print0 | xargs -0 tar -zcf target/failure-${{ inputs.name }}.tar.gz
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: ${{ inputs.name }}
         path: target/failure-${{ inputs.name }}.tar.gz

.github/actions/artifact_upload/action.yml

@@ -32,14 +32,14 @@ runs:
 
     - name: Upload artifact to github
       if: env.RUNNER_PROVIDER == 'github'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: ${{ env.BUILD_PROFILE }}-${{ inputs.sha }}-${{ inputs.target }}-${{ inputs.category }}
         path: ${{ steps.info.outputs.path }}/databend-*
 
     - name: Upload open-sharing to github
       if: env.RUNNER_PROVIDER == 'github' && inputs.category == 'default'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: ${{ env.BUILD_PROFILE }}-${{ inputs.sha }}-${{ inputs.target }}-${{ inputs.category }}
         path: ${{ steps.info.outputs.path }}/open-sharing

.github/actions/benchmark_cloud/action.yml

@@ -38,8 +38,23 @@ runs:
     - name: Install script dependencies
       shell: bash
       run: |
-        sudo apt-get update -y
-        sudo apt install -y python3
+        sudo apt-get update -yq
+        sudo apt-get install -yq python3
+
+    - name: Prepare
+      working-directory: benchmark/clickbench
+      shell: bash
+      id: prepare
+      env:
+        BENDSQL_DSN: "databend://${{ inputs.cloud_user }}:${{ inputs.cloud_password }}@${{ inputs.cloud_gateway }}:443"
+      run: |
+        if [[ "${{ inputs.dataset }}" == "load" ]]; then
+          echo "database=load_test_${{ inputs.run_id }}" >> $GITHUB_OUTPUT
+          echo "tries=1" >> $GITHUB_OUTPUT
+        else
+          echo "database=clickbench" >> $GITHUB_OUTPUT
+          echo "tries=3" >> $GITHUB_OUTPUT
+        fi
 
     - name: Run Benchmark
       working-directory: benchmark/clickbench
@@ -48,7 +63,8 @@ runs:
         BENCHMARK_DATASET: ${{ inputs.dataset }}
         BENCHMARK_SIZE: ${{ inputs.size }}
         BENCHMARK_VERSION: ${{ inputs.version }}
-        BENCHMARK_DATABASE: clickbench
+        BENCHMARK_DATABASE: ${{ steps.prepare.outputs.database }}
+        BENCHMARK_TRIES: ${{ steps.prepare.outputs.tries }}
         CLOUD_USER: ${{ inputs.cloud_user }}
         CLOUD_PASSWORD: ${{ inputs.cloud_password }}
         CLOUD_GATEWAY: ${{ inputs.cloud_gateway }}
@@ -76,9 +92,9 @@ runs:
         mv result.json result-${{ inputs.dataset }}-cloud-${{ inputs.size }}.json
 
     - name: Upload artifact
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: benchmark-${{ inputs.dataset }}
+        name: benchmark-${{ inputs.dataset }}-${{ inputs.size }}
         path: benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}.json
 
     - name: Remove warehouse

.github/actions/benchmark_local/action.yml

@@ -23,8 +23,8 @@ runs:
     - name: Install script dependencies
       shell: bash
       run: |
-        sudo apt update -y
-        sudo apt install -y python3
+        sudo apt-get update -yq
+        sudo apt-get install -yq python3
 
     - name: Run Benchmark
       working-directory: benchmark/clickbench
@@ -54,15 +54,7 @@ runs:
         mv result.json result-${{ inputs.dataset }}-local.json
 
     - name: Upload artifact
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: benchmark-${{ inputs.dataset }}
+        name: benchmark-${{ inputs.dataset }}-local
         path: benchmark/clickbench/result-${{ inputs.dataset }}-local.json
-
-    - name: Clean test data
-      working-directory: benchmark/clickbench
-      if: always()
-      continue-on-error: true
-      shell: bash
-      run: |
-        bendsql <"${{ inputs.dataset }}/clear.sql"

.github/actions/build_bindings_python/action.yml

@@ -51,16 +51,18 @@ runs:
         rust-toolchain: ${{ steps.toolchain.outputs.RUST_TOOLCHAIN }}
         working-directory: src/bendpy
         target: ${{ inputs.target }}
-        manylinux: auto
+        manylinux: "2_28"
         # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
         rustup-components: rust-std rustfmt
         args: ${{ steps.opts.outputs.BUILD_ARGS }}
         docker-options: --env RUSTC_WRAPPER=sccache --env SCCACHE_GCS_RW_MODE=READ_WRITE --env SCCACHE_GCS_BUCKET=databend-ci --env SCCACHE_GCS_KEY_PREFIX=cache/sccache/
-        before-script-linux: ../../scripts/setup/dev_setup.sh -yb
+        before-script-linux: |
+          unset RUSTC_WRAPPER
+          ../../scripts/setup/dev_setup.sh -yb
 
     - name: Run tests
       working-directory: src/bendpy
-      if: startsWith(inputs.target, 'x86_64-')
+      if: endsWith(inputs.target, '-gnu')
       shell: bash
       run: |
         pip install dist/*.whl

.github/actions/build_linux/action.yml

@@ -70,9 +70,9 @@ runs:
         artifacts="meta,metactl,query,sqllogictests"
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch,shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact
         done
-        cargo -Zgitoxide=fetch,shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin open-sharing
+        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin open-sharing
         ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-*
 
     - name: Build Debug for specific artifacts
@@ -82,7 +82,7 @@ runs:
         artifacts="${{ inputs.artifacts }}"
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch,shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact
         done
         ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
 
@@ -93,7 +93,7 @@ runs:
         artifacts="meta,metactl,query,sqllogictests"
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch,shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --release --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --release --bin databend-$artifact
         done
         ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
 
@@ -104,12 +104,16 @@ runs:
         artifacts="${{ inputs.artifacts }}"
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch,shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --release --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --release --bin databend-$artifact
         done
         ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
 
-    - shell: bash
-      run: readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query
+    - name: Check Binary Info
+      shell: bash
+      continue-on-error: true
+      run: |
+        readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query
+        ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query
 
     # - name: Compress Binaries with UPX
     #   if: env.BUILD_PROFILE == 'debug'

.github/actions/build_linux_sanitizer/action.yml

@@ -0,0 +1,66 @@
+name: "Build Linux with AddressSanitizer"
+description: "Build with AddressSanitizer"
+inputs:
+  target:
+    description: ""
+    required: true
+  artifacts:
+    description: "Artifacts to build"
+    required: false
+    default: "meta,query"
+  features:
+    description: "Features to build"
+    required: false
+    default: "default"
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Build Tool
+      uses: ./.github/actions/setup_build_tool
+      with:
+        target: ${{ inputs.target }}
+        bypass_env_vars: RUSTFLAGS,RUST_LOG
+
+    - name: Cross setup
+      if: startsWith(inputs.target, 'aarch64-')
+      shell: bash
+      run: |
+        echo "JEMALLOC_SYS_WITH_LG_PAGE=16" >> $GITHUB_ENV
+
+    - name: Set Build Flags
+      shell: bash
+      run: |
+        case ${{ inputs.target }} in
+          x86_64-unknown-linux-gnu)
+            flags="-Z sanitizer=address"
+            ;;
+          aarch64-unknown-linux-gnu)
+            flags="-Z sanitizer=address"
+            ;;
+          *)
+            flags=""
+            ;;
+        esac
+        echo "RUSTFLAGS=${flags} -C link-arg=-Wl,--compress-debug-sections=zlib" >> $GITHUB_ENV
+        target=${{ inputs.target }}
+        echo "BUILD_ARCH=${target/-unknown-linux-*}" >> $GITHUB_ENV
+
+    # build all binaries for debug
+    - name: Build Debug
+      if: env.BUILD_PROFILE == 'debug' && endsWith(inputs.target, '-gnu')
+      shell: bash
+      run: |
+        artifacts="${{ inputs.artifacts }}"
+        for artifact in ${artifacts//,/ }; do
+          echo "==> building databend-$artifact with sanitizer ..."
+          cargo -Zbuild-std -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --manifest-path src/binaries/Cargo.toml --bin databend-$artifact
+        done
+        ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-*
+
+    - name: Check Binary Info
+      if: env.BUILD_PROFILE == 'debug' && endsWith(inputs.target, '-gnu')
+      shell: bash
+      continue-on-error: true
+      run: |
+        readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query
+        ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query

.github/actions/build_macos/action.yml

@@ -21,6 +21,7 @@ runs:
     - name: Rust setup
       shell: bash
       run: |
+        brew unlink python || true
         bash ./scripts/setup/dev_setup.sh -yb
         rustup target add ${{ inputs.target }}
 
@@ -43,18 +44,18 @@ runs:
     - name: Build Debug for all artifacts
       if: env.BUILD_PROFILE == 'debug' && inputs.artifacts == 'all'
       shell: bash
-      run: cargo -Zgitoxide=fetch,shallow-index,shallow-deps build --target ${{ inputs.target }}
+      run: cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }}
 
     - name: Build Debug for specific artifacts
       if: env.BUILD_PROFILE == 'debug' && inputs.artifacts != 'all'
       shell: bash
       run: |
         artifacts="${{ inputs.artifacts }}"
         echo "==> building libs ..."
-        cargo -Zgitoxide=fetch,shallow-index,shallow-deps build --target ${{ inputs.target }} --lib
+        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --lib
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch,shallow-index,shallow-deps build --target ${{ inputs.target }} --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --bin databend-$artifact
         done
 
     - name: Build Release
@@ -63,10 +64,10 @@ runs:
       run: |
         artifacts="${{ inputs.artifacts }}"
         echo "==> building libs ..."
-        cargo -Zgitoxide=fetch,shallow-index,shallow-deps build --target ${{ inputs.target }} --release --lib
+        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --release --lib
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch,shallow-index,shallow-deps build --target ${{ inputs.target }} --release --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --release --bin databend-$artifact
         done
 
     - name: Upload artifact

.github/actions/check/action.yml

@@ -30,7 +30,7 @@ runs:
 
     - name: Check typos
       shell: bash
-      run: typos --exclude docs --exclude website
+      run: typos
 
     - name: Check unused deps
       shell: bash
@@ -47,4 +47,4 @@ runs:
 
     - name: Clippy
       shell: bash
-      run: cargo -Zgitoxide=fetch,shallow-index,shallow-deps clippy --workspace --all-targets --all-features -- -D warnings
+      run: cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps clippy --workspace --all-targets --all-features -- -D warnings