feat: impl opendal::HttpFetch for StorageHttpClient

.github/workflows/dev.yml

@@ -23,7 +23,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Check Source File Changes
-        uses: tj-actions/changed-files@v39
+        uses: tj-actions/changed-files@v45
         id: src
         with:
           files_ignore: |
@@ -50,7 +50,7 @@ jobs:
     uses: ./.github/workflows/reuse.linux.yml
     secrets: inherit
     with:
-      build_profile: debug
+      build_profile: release
       runner_provider: aws
 
   linux_hive:
@@ -59,7 +59,7 @@ jobs:
     uses: ./.github/workflows/reuse.linux.hive.yml
     secrets: inherit
     with:
-      build_profile: debug
+      build_profile: release
       runner_provider: aws
 
   ready:

.github/workflows/production.yml → .github/workflows/main.yml

@@ -1,10 +1,9 @@
-name: Production
+name: Main
 
 on:
   push:
     branches:
       - main
-  merge_group:
 
 jobs:
   changes:
@@ -16,7 +15,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Check Source File Changes
-        uses: tj-actions/changed-files@v39
+        uses: tj-actions/changed-files@v45
         id: src
         with:
           files_ignore: |

.github/workflows/merge_group.yml

@@ -1,4 +1,4 @@
-name: "PR Assistant"
+name: "Merge Group"
 
 on:
   merge_group:
@@ -17,3 +17,66 @@ jobs:
       - name: Skip CLA check
         shell: bash
         run: exit 0
+
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      any_src_changed: ${{ steps.src.outputs.any_changed }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Check Source File Changes
+        uses: tj-actions/changed-files@v45
+        id: src
+        with:
+          files_ignore: |
+            .github/**
+            **.md
+            benchmark/**
+            docker/**
+            scripts/setup/**
+            .devcontainer/**
+      - name: Output Source File Changes
+        run: |
+          if [[ "${{ steps.src.outputs.any_changed }}" == "true" ]]; then
+            echo "these src files changed:" >> $GITHUB_STEP_SUMMARY
+            for line in ${{ steps.src.outputs.all_changed_files }}; do
+              echo "- $line" >> $GITHUB_STEP_SUMMARY
+            done
+          else
+            echo "no src file changes detected" >> $GITHUB_STEP_SUMMARY
+          fi
+
+  linux:
+    needs: changes
+    if: needs.changes.outputs.any_src_changed == 'true'
+    uses: ./.github/workflows/reuse.linux.yml
+    secrets: inherit
+    with:
+      build_profile: release
+      runner_provider: aws
+
+  ready:
+    if: always()
+    runs-on: ubuntu-latest
+    needs:
+      - changes
+      - linux
+    steps:
+      - name: Check Ready to Merge
+        uses: actions/github-script@v7
+        env:
+          SRC_CHANGED: ${{ needs.changes.outputs.any_src_changed }}
+          LINUX_BUILD_RESULT: ${{ needs.linux.result }}
+        with:
+          script: |
+            if (process.env.SRC_CHANGED == 'false') {
+              core.info('No source file changes detected, skipping');
+              return;
+            }
+            if (process.env.LINUX_BUILD_RESULT == 'success') {
+              core.info('Linux build succeeded, ready to merge');
+              return;
+            }
+            core.setFailed('Build failed, not ready to merge');

.github/workflows/reuse.sqllogic.yml

@@ -20,7 +20,7 @@ env:
 
 jobs:
   management_mode:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_sqllogic_management_mode_linux
@@ -30,12 +30,11 @@ jobs:
           handlers: mysql,http
 
   standalone:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     strategy:
       fail-fast: false
       matrix:
         dirs:
-          - "query"
           - "standalone"
           - "crdb"
           - "duckdb"
@@ -60,8 +59,35 @@ jobs:
         with:
           name: test-sqllogic-standalone-${{ matrix.dirs }}-${{ matrix.handler }}
 
+  standalone_query:
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    strategy:
+      fail-fast: false
+      matrix:
+        dirs:
+          - "query"
+        handler:
+          - "mysql"
+          - "http"
+        format:
+          - "native"
+          - "parquet"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/test_sqllogic_standalone_linux
+        timeout-minutes: 15
+        with:
+          dirs: ${{ matrix.dirs }}
+          handlers: ${{ matrix.handler }}
+          storage-format: ${{ matrix.format }}
+      - name: Upload failure
+        if: failure()
+        uses: ./.github/actions/artifact_failure
+        with:
+          name: test-sqllogic-standalone-query-${{ matrix.handler }}-${{ matrix.format }}
+
   standalone_udf_server:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     steps:
       - uses: actions/checkout@v4
       - name: Start UDF Server
@@ -82,7 +108,7 @@ jobs:
           name: test-sqllogic-standalone-udf-server
 
   standalone_cloud:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     steps:
       - uses: actions/checkout@v4
       - name: Start Cloud Control Server
@@ -103,7 +129,7 @@ jobs:
           name: test-sqllogic-standalone-cloud
 
   standalone_minio:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     strategy:
       fail-fast: false
       matrix:
@@ -130,13 +156,13 @@ jobs:
           name: test-sqllogic-standalone-minio-${{ matrix.dirs }}-${{ matrix.handler }}-${{ matrix.format }}
 
   standalone_iceberg_tpch:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-java@v4
         with:
-          distribution: 'temurin'
-          java-version: '17'
+          distribution: "temurin"
+          java-version: "17"
       - uses: ./.github/actions/test_sqllogic_iceberg_tpch
         timeout-minutes: 15
         with:
@@ -149,7 +175,7 @@ jobs:
           name: test-sqllogic-standalone-iceberg-tpch
 
   cluster:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     strategy:
       fail-fast: false
       matrix:
@@ -182,7 +208,7 @@ jobs:
           name: test-sqllogic-cluster-${{ matrix.dirs }}-${{ matrix.handler }}
 
   stage:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     strategy:
       fail-fast: false
       matrix:
@@ -204,7 +230,7 @@ jobs:
           name: test-sqllogic-stage-${{ matrix.storage }}
 
   standalone_no_table_meta_cache:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     strategy:
       fail-fast: false
       matrix:
@@ -228,7 +254,7 @@ jobs:
           name: test-sqllogic-standalone-no-table-meta-cache-${{ matrix.dirs }}-${{ matrix.handler }}
 
   ee:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     strategy:
       fail-fast: false
       matrix:

src/common/storage/Cargo.toml

@@ -24,11 +24,13 @@ databend-common-meta-app = { workspace = true }
 databend-enterprise-storage-encryption = { workspace = true }
 flagset = { workspace = true }
 futures = { workspace = true }
+http = { workspace = true }
 log = { workspace = true }
 opendal = { workspace = true }
 parquet = { workspace = true }
 prometheus-client = { workspace = true }
 regex = { workspace = true }
+reqwest = { workspace = true }
 serde = { workspace = true }
 thiserror = { workspace = true }