Merge branch 'main' of https://github.com/datafuselabs/databend into …

…mergeinto-runtime-filter
databendlabs · Jan 3, 2024 · 93b76f8 · 93b76f8
2 parents a226452 + fdc22ef
commit 93b76f8
Show file tree

Hide file tree

Showing 588 changed files with 16,982 additions and 5,156 deletions.
diff --git a/.github/actions/test_ee_management_mode_linux/action.yml b/.github/actions/test_ee_management_mode_linux/action.yml
@@ -0,0 +1,21 @@
+name: "Test ee management mode"
+description: "Running stateless tests in management mode"
+inputs:
+  target:
+    description: ""
+    required: true
+    default: "x86_64-unknown-linux-gnu"
+runs:
+  using: "composite"
+  steps:
+    - uses: ./.github/actions/setup_bendsql
+    - name: Download artifact
+      uses: ./.github/actions/artifact_download
+      with:
+        sha: ${{ github.sha }}
+        target: ${{ inputs.target }}
+
+    - name: Run Stateful Tests with Standalone mode
+      shell: bash
+      run: |
+        ./scripts/ci/ci-run-ee-tests-management-mode.sh
diff --git a/.github/actions/test_sqllogic_standalone_linux_minio/action.yml b/.github/actions/test_sqllogic_standalone_linux_minio/action.yml
@@ -0,0 +1,61 @@
+name: "Test sqllogic Standalone(minio)"
+description: "Running sqllogic tests in standalone mode"
+inputs:
+  target:
+    description: ""
+    required: true
+    default: "x86_64-unknown-linux-gnu"
+  dirs:
+    description: "logic test suites dirs"
+    required: true
+    default: ""
+  handlers:
+    description: "logic test handlers, mysql,http,clickhouse"
+    required: true
+    default: ""
+  storage-format:
+    description: "storage format for databend query to test"
+    required: true
+    default: all
+
+runs:
+  using: "composite"
+  steps:
+    - uses: ./.github/actions/setup_bendsql
+    - name: Download artifact
+      uses: ./.github/actions/artifact_download
+      with:
+        sha: ${{ github.sha }}
+        target: ${{ inputs.target }}
+        artifacts: sqllogictests,meta,query
+
+    - name: Minio Setup for (ubuntu-latest only)
+      shell: bash
+      run: |
+        docker run -d --network host --name minio \
+                -e "MINIO_ACCESS_KEY=minioadmin" \
+                -e "MINIO_SECRET_KEY=minioadmin" \
+                -e "MINIO_ADDRESS=:9900" \
+                -v /tmp/data:/data \
+                -v /tmp/config:/root/.minio \
+                minio/minio server /data
+
+        export AWS_ACCESS_KEY_ID=minioadmin
+        export AWS_SECRET_ACCESS_KEY=minioadmin
+        export AWS_EC2_METADATA_DISABLED=true
+
+        aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket
+
+    - name: Run sqllogic Tests with Standalone mode
+      if: inputs.storage-format == 'all' || inputs.storage-format == 'parquet'
+      shell: bash
+      env:
+        TEST_HANDLERS: ${{ inputs.handlers }}
+      run: bash ./scripts/ci/ci-run-sqllogic-tests-minio.sh ${{ inputs.dirs }}
+
+    - name: Run native sqllogic Tests with Standalone mode
+      if: inputs.storage-format == 'all' || inputs.storage-format == 'native'
+      shell: bash
+      env:
+        TEST_HANDLERS: ${{ inputs.handlers }}
+      run: bash ./scripts/ci/ci-run-sqllogic-tests-native-minio.sh ${{ inputs.dirs }}
diff --git a/.github/actions/test_stateless_cluster_linux/action.yml b/.github/actions/test_stateless_cluster_linux/action.yml
@@ -20,6 +20,13 @@ runs:
       run: |
         bash ./scripts/setup/dev_setup.sh -yd
 
+    - name: Start UDF Server
+      shell: bash
+      run: |
+        pip install databend-udf
+        python3 tests/udf/udf_server.py &
+        sleep 2
+
     - name: Run Stateless Tests with Cluster mode
       shell: bash
       run: |

diff --git a/.github/actions/test_stateless_cluster_macos/action.yml b/.github/actions/test_stateless_cluster_macos/action.yml
@@ -15,12 +15,18 @@ runs:
       with:
         sha: ${{ github.sha }}
         target: ${{ inputs.target }}
-
     - name: Test setup
       shell: bash
       run: |
         bash ./scripts/setup/dev_setup.sh -yd
 
+    - name: Start UDF Server
+      shell: bash
+      run: |
+        pip install databend-udf
+        python3 tests/udf/udf_server.py &
+        sleep 2
+
     - name: Run Stateless Tests with Cluster mode
       shell: bash
       run: |

diff --git a/.github/actions/test_stateless_standalone_linux/action.yml b/.github/actions/test_stateless_standalone_linux/action.yml
@@ -21,6 +21,13 @@ runs:
       run: |
         bash ./scripts/setup/dev_setup.sh -yd
 
+    - name: Start UDF Server
+      shell: bash
+      run: |
+        pip install databend-udf
+        python3 tests/udf/udf_server.py &
+        sleep 2
+
     - name: Run Stateless Tests with Standalone mode
       shell: bash
       run: |

diff --git a/.github/actions/test_stateless_standalone_macos/action.yml b/.github/actions/test_stateless_standalone_macos/action.yml
@@ -20,6 +20,13 @@ runs:
       run: |
         bash ./scripts/setup/dev_setup.sh -yd
 
+    - name: Start UDF Server
+      shell: bash
+      run: |
+        pip install databend-udf
+        python3 tests/udf/udf_server.py &
+        sleep 2
+
     - name: Run Stateless Tests with Standalone mode
       shell: bash
       run: |

diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml
@@ -0,0 +1,42 @@
+name: Links
+
+on:
+  repository_dispatch:
+  workflow_dispatch:
+  schedule:
+    - cron: "00 18 * * *"
+
+jobs:
+  linkChecker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Restore lychee cache
+        id: restore-cache
+        uses: actions/cache/restore@v3
+        with:
+          path: .lycheecache
+          key: cache-lychee-${{ github.sha }}
+          restore-keys: cache-lychee-
+
+      - name: Link Checker
+        id: lychee
+        uses: lycheeverse/[email protected]
+        with:
+          args: "--base . --cache --max-cache-age 1d . --exclude 'https?://twitter\\.com(?:/.*$)?$'"
+
+      - name: Save lychee cache
+        uses: actions/cache/save@v3
+        if: always()
+        with:
+          path: .lycheecache
+          key: ${{ steps.restore-cache.outputs.cache-primary-key }}
+
+      - name: Create Issue From File
+        if: env.lychee_exit_code != 0
+        uses: peter-evans/create-issue-from-file@v4
+        with:
+          title: Link Checker Report
+          content-filepath: ./lychee/out.md
+          labels: report, automated issue
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -143,7 +143,7 @@ jobs:
           token: ${{ github.token }}
           identifier: 'pr-assistant-cla'
           body: |
-            Pull request description must contain [CLA](https://docs.databend.com/doc/contributing/good-pr) like the following:
+            Pull request description must contain [CLA](https://docs.databend.com/dev/policies/cla/) like the following:
 
             ```
             I hereby agree to the terms of the CLA available at: https://docs.databend.com/dev/policies/cla/

diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml
@@ -278,6 +278,28 @@ jobs:
         with:
           name: test-stateful-standalone-linux
 
+  test_ee_management_mode:
+    needs: [build, check, license]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    if: needs.license.outputs.license
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup_bendsql
+      - name: Decode license
+        id: license
+        run: |
+          echo "license=$(echo '${{ needs.license.outputs.license }}' | base64 -d)" >> $GITHUB_OUTPUT
+      - uses: ./.github/actions/test_ee_management_mode_linux
+        timeout-minutes: 10
+        env:
+          DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
+          QUERY_DATABEND_ENTERPRISE_LICENSE: ${{ steps.license.outputs.license }}
+      - name: Upload failure
+        if: failure()
+        uses: ./.github/actions/artifact_failure
+        with:
+          name: test-ee-management-mode-linux
+
   sqllogic_ee:
     runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     if: needs.license.outputs.license

diff --git a/.github/workflows/reuse.sqllogic.yml b/.github/workflows/reuse.sqllogic.yml
@@ -153,3 +153,27 @@ jobs:
         uses: ./.github/actions/artifact_failure
         with:
           name: test-sqllogic-stage-${{ matrix.storage }}
+
+  standalone-minio:
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    strategy:
+      fail-fast: false
+      matrix:
+        dirs:
+          - "query"
+        handler:
+          - "mysql"
+          - "http"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/test_sqllogic_standalone_linux_minio
+        timeout-minutes: 15
+        with:
+          dirs: ${{ matrix.dirs }}
+          handlers: ${{ matrix.handler }}
+          storage-format: all
+      - name: Upload failure
+        if: failure()
+        uses: ./.github/actions/artifact_failure
+        with:
+          name: test-sqllogic-standalone-${{ matrix.dirs }}-${{ matrix.handler }}
diff --git a/.gitignore b/.gitignore
@@ -75,3 +75,6 @@ benchmark/clickbench/results
 
 # z3
 **/.z3-trace
+
+# lychee
+.lycheecache