From ac4a8ab1577c0b71581814336f7af602f3f810cf Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 31 Oct 2024 11:41:49 +0800 Subject: [PATCH 01/13] feat: impl opendal::HttpFetch for HttpClient --- Cargo.lock | 2 + src/common/base/Cargo.toml | 2 + src/common/base/src/http_client.rs | 96 ++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f73e498dba83b..9a8851f62e8c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3172,6 +3172,7 @@ dependencies = [ "enquote", "fastrace", "futures", + "http 1.1.0", "libc", "log", "logcall", @@ -3179,6 +3180,7 @@ dependencies = [ "num-traits", "num_cpus", "once_cell", + "opendal", "parking_lot 0.12.3", "pin-project-lite", "pprof", diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml index 7cb7347bbb697..d43d521daf2c5 100644 --- a/src/common/base/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -32,6 +32,7 @@ ctrlc = { workspace = true } enquote = { workspace = true } fastrace = { workspace = true } futures = { workspace = true } +http = { workspace = true } libc = { workspace = true } log = { workspace = true } logcall = { workspace = true } @@ -39,6 +40,7 @@ micromarshal = { workspace = true } num-traits = { workspace = true } num_cpus = { workspace = true } once_cell = { workspace = true } +opendal = { workspace = true } parking_lot = { workspace = true } pin-project-lite = { workspace = true } pprof = { workspace = true } diff --git a/src/common/base/src/http_client.rs b/src/common/base/src/http_client.rs index 182c0a46611da..908bc5680ce23 100644 --- a/src/common/base/src/http_client.rs +++ b/src/common/base/src/http_client.rs @@ -13,10 +13,21 @@ // limitations under the License. use std::env; +use std::future; +use std::mem; +use std::str::FromStr; use std::sync::Arc; use std::sync::LazyLock; use std::time::Duration; +use futures::TryStreamExt; +use http::Request; +use http::Response; +use opendal::raw::parse_content_encoding; +use opendal::raw::parse_content_length; +use opendal::raw::HttpBody; +use opendal::raw::HttpFetch; +use opendal::Buffer; use reqwest_hickory_resolver::HickoryResolver; /// Global shared hickory resolver. @@ -86,3 +97,88 @@ impl HttpClient { self.client.clone() } } + +impl HttpFetch for HttpClient { + async fn fetch(&self, req: Request) -> opendal::Result> { + // Uri stores all string alike data in `Bytes` which means + // the clone here is cheap. + let uri = req.uri().clone(); + let is_head = req.method() == http::Method::HEAD; + + let (parts, body) = req.into_parts(); + + let mut req_builder = self + .client + .request( + parts.method, + reqwest::Url::from_str(&uri.to_string()).expect("input request url must be valid"), + ) + .headers(parts.headers); + + req_builder = req_builder.version(parts.version); + // Don't set body if body is empty. + if !body.is_empty() { + req_builder = req_builder.body(reqwest::Body::wrap_stream(body)) + } + + let mut resp = req_builder + .send() + .await + .map_err(|err| to_opendal_unexpected_error(err, &uri, "send http request"))?; + + // Get content length from header so that we can check it. + // + // - If the request method is HEAD, we will ignore content length. + // - If response contains content_encoding, we should omit its content length. + let content_length = if is_head || parse_content_encoding(resp.headers())?.is_some() { + None + } else { + parse_content_length(resp.headers())? + }; + + let mut hr = Response::builder() + .status(resp.status()) + // Insert uri into response extension so that we can fetch + // it later. + .extension(uri.clone()); + + hr = hr.version(resp.version()); + + // Swap headers directly instead of copy the entire map. + mem::swap(hr.headers_mut().unwrap(), resp.headers_mut()); + + let bs = HttpBody::new( + resp.bytes_stream() + .try_filter(|v| future::ready(!v.is_empty())) + .map_ok(Buffer::from) + .map_err(move |err| { + to_opendal_unexpected_error(err, &uri, "read data from http response") + }), + content_length, + ); + + let resp = hr.body(bs).expect("response must build succeed"); + Ok(resp) + } +} + +fn to_opendal_unexpected_error(err: reqwest::Error, uri: &http::Uri, desc: &str) -> opendal::Error { + let mut oe = opendal::Error::new(opendal::ErrorKind::Unexpected, desc) + .with_operation("http_util::Client::send") + .with_context("url", uri.to_string()); + if is_temporary_error(&err) { + oe = oe.set_temporary(); + } + oe = oe.set_source(err); + oe +} + +#[inline] +fn is_temporary_error(err: &reqwest::Error) -> bool { + // error sending request + err.is_request()|| + // request or response body error + err.is_body() || + // error decoding response body, for example, connection reset. + err.is_decode() +} From c75c97623d20b5b454d86cf7c07ae2bc9b35db28 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 31 Oct 2024 11:47:02 +0800 Subject: [PATCH 02/13] z --- src/common/base/src/http_client.rs | 14 +++++++++++++- src/common/storage/src/operator.rs | 14 +++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/common/base/src/http_client.rs b/src/common/base/src/http_client.rs index 908bc5680ce23..b4a77c5608ae8 100644 --- a/src/common/base/src/http_client.rs +++ b/src/common/base/src/http_client.rs @@ -98,7 +98,19 @@ impl HttpClient { } } -impl HttpFetch for HttpClient { +pub struct StorageHttpClient { + client: reqwest::Client, +} + +impl StorageHttpClient { + pub fn new() -> Self { + Self { + client: GLOBAL_HTTP_CLIENT.inner(), + } + } +} + +impl HttpFetch for StorageHttpClient { async fn fetch(&self, req: Request) -> opendal::Result> { // Uri stores all string alike data in `Bytes` which means // the clone here is cheap. diff --git a/src/common/storage/src/operator.rs b/src/common/storage/src/operator.rs index 6cf73e49d1240..3383c333e3e5c 100644 --- a/src/common/storage/src/operator.rs +++ b/src/common/storage/src/operator.rs @@ -20,7 +20,7 @@ use std::time::Duration; use anyhow::anyhow; use databend_common_base::base::GlobalInstance; -use databend_common_base::http_client::GLOBAL_HTTP_CLIENT; +use databend_common_base::http_client::StorageHttpClient; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::TrySpawn; use databend_common_exception::ErrorCode; @@ -180,7 +180,7 @@ pub fn init_azblob_operator(cfg: &StorageAzblobConfig) -> Result { // Credential .account_name(&cfg.account_name) .account_key(&cfg.account_key) - .http_client(HttpClient::with(GLOBAL_HTTP_CLIENT.inner())); + .http_client(HttpClient::with(StorageHttpClient::new())); Ok(builder) } @@ -205,7 +205,7 @@ fn init_gcs_operator(cfg: &StorageGcsConfig) -> Result { .bucket(&cfg.bucket) .root(&cfg.root) .credential(&cfg.credential) - .http_client(HttpClient::with(GLOBAL_HTTP_CLIENT.inner())); + .http_client(HttpClient::with(StorageHttpClient::new())); Ok(builder) } @@ -302,7 +302,7 @@ fn init_s3_operator(cfg: &StorageS3Config) -> Result { builder = builder.enable_virtual_host_style(); } - builder = builder.http_client(HttpClient::with(GLOBAL_HTTP_CLIENT.inner())); + builder = builder.http_client(HttpClient::with(StorageHttpClient::new())); Ok(builder) } @@ -319,7 +319,7 @@ fn init_obs_operator(cfg: &StorageObsConfig) -> Result { // Credential .access_key_id(&cfg.access_key_id) .secret_access_key(&cfg.secret_access_key) - .http_client(HttpClient::with(GLOBAL_HTTP_CLIENT.inner())); + .http_client(HttpClient::with(StorageHttpClient::new())); Ok(builder) } @@ -335,7 +335,7 @@ fn init_oss_operator(cfg: &StorageOssConfig) -> Result { .root(&cfg.root) .server_side_encryption(&cfg.server_side_encryption) .server_side_encryption_key_id(&cfg.server_side_encryption_key_id) - .http_client(HttpClient::with(GLOBAL_HTTP_CLIENT.inner())); + .http_client(HttpClient::with(StorageHttpClient::new())); Ok(builder) } @@ -368,7 +368,7 @@ fn init_cos_operator(cfg: &StorageCosConfig) -> Result { .secret_key(&cfg.secret_key) .bucket(&cfg.bucket) .root(&cfg.root) - .http_client(HttpClient::with(GLOBAL_HTTP_CLIENT.inner())); + .http_client(HttpClient::with(StorageHttpClient::new())); Ok(builder) } From 9525672b3ae45de09dcc60929c9b4a8eab04e558 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 31 Oct 2024 11:57:10 +0800 Subject: [PATCH 03/13] z --- Cargo.lock | 4 +- src/common/base/Cargo.toml | 2 - src/common/base/src/http_client.rs | 108 ---------------------- src/common/storage/Cargo.toml | 2 + src/common/storage/src/http_client.rs | 124 ++++++++++++++++++++++++++ src/common/storage/src/lib.rs | 3 + src/common/storage/src/operator.rs | 2 +- 7 files changed, 132 insertions(+), 113 deletions(-) create mode 100644 src/common/storage/src/http_client.rs diff --git a/Cargo.lock b/Cargo.lock index 9a8851f62e8c3..fc97299b42c9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3172,7 +3172,6 @@ dependencies = [ "enquote", "fastrace", "futures", - "http 1.1.0", "libc", "log", "logcall", @@ -3180,7 +3179,6 @@ dependencies = [ "num-traits", "num_cpus", "once_cell", - "opendal", "parking_lot 0.12.3", "pin-project-lite", "pprof", @@ -4179,11 +4177,13 @@ dependencies = [ "databend-enterprise-storage-encryption", "flagset", "futures", + "http 1.1.0", "log", "opendal", "parquet", "prometheus-client", "regex", + "reqwest", "serde", "thiserror", ] diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml index d43d521daf2c5..7cb7347bbb697 100644 --- a/src/common/base/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -32,7 +32,6 @@ ctrlc = { workspace = true } enquote = { workspace = true } fastrace = { workspace = true } futures = { workspace = true } -http = { workspace = true } libc = { workspace = true } log = { workspace = true } logcall = { workspace = true } @@ -40,7 +39,6 @@ micromarshal = { workspace = true } num-traits = { workspace = true } num_cpus = { workspace = true } once_cell = { workspace = true } -opendal = { workspace = true } parking_lot = { workspace = true } pin-project-lite = { workspace = true } pprof = { workspace = true } diff --git a/src/common/base/src/http_client.rs b/src/common/base/src/http_client.rs index b4a77c5608ae8..182c0a46611da 100644 --- a/src/common/base/src/http_client.rs +++ b/src/common/base/src/http_client.rs @@ -13,21 +13,10 @@ // limitations under the License. use std::env; -use std::future; -use std::mem; -use std::str::FromStr; use std::sync::Arc; use std::sync::LazyLock; use std::time::Duration; -use futures::TryStreamExt; -use http::Request; -use http::Response; -use opendal::raw::parse_content_encoding; -use opendal::raw::parse_content_length; -use opendal::raw::HttpBody; -use opendal::raw::HttpFetch; -use opendal::Buffer; use reqwest_hickory_resolver::HickoryResolver; /// Global shared hickory resolver. @@ -97,100 +86,3 @@ impl HttpClient { self.client.clone() } } - -pub struct StorageHttpClient { - client: reqwest::Client, -} - -impl StorageHttpClient { - pub fn new() -> Self { - Self { - client: GLOBAL_HTTP_CLIENT.inner(), - } - } -} - -impl HttpFetch for StorageHttpClient { - async fn fetch(&self, req: Request) -> opendal::Result> { - // Uri stores all string alike data in `Bytes` which means - // the clone here is cheap. - let uri = req.uri().clone(); - let is_head = req.method() == http::Method::HEAD; - - let (parts, body) = req.into_parts(); - - let mut req_builder = self - .client - .request( - parts.method, - reqwest::Url::from_str(&uri.to_string()).expect("input request url must be valid"), - ) - .headers(parts.headers); - - req_builder = req_builder.version(parts.version); - // Don't set body if body is empty. - if !body.is_empty() { - req_builder = req_builder.body(reqwest::Body::wrap_stream(body)) - } - - let mut resp = req_builder - .send() - .await - .map_err(|err| to_opendal_unexpected_error(err, &uri, "send http request"))?; - - // Get content length from header so that we can check it. - // - // - If the request method is HEAD, we will ignore content length. - // - If response contains content_encoding, we should omit its content length. - let content_length = if is_head || parse_content_encoding(resp.headers())?.is_some() { - None - } else { - parse_content_length(resp.headers())? - }; - - let mut hr = Response::builder() - .status(resp.status()) - // Insert uri into response extension so that we can fetch - // it later. - .extension(uri.clone()); - - hr = hr.version(resp.version()); - - // Swap headers directly instead of copy the entire map. - mem::swap(hr.headers_mut().unwrap(), resp.headers_mut()); - - let bs = HttpBody::new( - resp.bytes_stream() - .try_filter(|v| future::ready(!v.is_empty())) - .map_ok(Buffer::from) - .map_err(move |err| { - to_opendal_unexpected_error(err, &uri, "read data from http response") - }), - content_length, - ); - - let resp = hr.body(bs).expect("response must build succeed"); - Ok(resp) - } -} - -fn to_opendal_unexpected_error(err: reqwest::Error, uri: &http::Uri, desc: &str) -> opendal::Error { - let mut oe = opendal::Error::new(opendal::ErrorKind::Unexpected, desc) - .with_operation("http_util::Client::send") - .with_context("url", uri.to_string()); - if is_temporary_error(&err) { - oe = oe.set_temporary(); - } - oe = oe.set_source(err); - oe -} - -#[inline] -fn is_temporary_error(err: &reqwest::Error) -> bool { - // error sending request - err.is_request()|| - // request or response body error - err.is_body() || - // error decoding response body, for example, connection reset. - err.is_decode() -} diff --git a/src/common/storage/Cargo.toml b/src/common/storage/Cargo.toml index 816392e7f2211..f65fdaa551ee0 100644 --- a/src/common/storage/Cargo.toml +++ b/src/common/storage/Cargo.toml @@ -24,11 +24,13 @@ databend-common-meta-app = { workspace = true } databend-enterprise-storage-encryption = { workspace = true } flagset = "0.4" futures = { workspace = true } +http = { workspace = true } log = { workspace = true } opendal = { workspace = true } parquet = { workspace = true } prometheus-client = { workspace = true } regex = { workspace = true } +reqwest = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } diff --git a/src/common/storage/src/http_client.rs b/src/common/storage/src/http_client.rs new file mode 100644 index 0000000000000..270e39e029028 --- /dev/null +++ b/src/common/storage/src/http_client.rs @@ -0,0 +1,124 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::future; +use std::mem; +use std::str::FromStr; + +use databend_common_base::http_client::GLOBAL_HTTP_CLIENT; +use futures::TryStreamExt; +use http::Request; +use http::Response; +use opendal::raw::parse_content_encoding; +use opendal::raw::parse_content_length; +use opendal::raw::HttpBody; +use opendal::raw::HttpFetch; +use opendal::Buffer; + +pub struct StorageHttpClient { + client: reqwest::Client, +} + +impl StorageHttpClient { + pub fn new() -> Self { + Self { + client: GLOBAL_HTTP_CLIENT.inner(), + } + } +} + +impl HttpFetch for StorageHttpClient { + async fn fetch(&self, req: Request) -> opendal::Result> { + // Uri stores all string alike data in `Bytes` which means + // the clone here is cheap. + let uri = req.uri().clone(); + let is_head = req.method() == http::Method::HEAD; + + let (parts, body) = req.into_parts(); + + let mut req_builder = self + .client + .request( + parts.method, + reqwest::Url::from_str(&uri.to_string()).expect("input request url must be valid"), + ) + .headers(parts.headers); + + req_builder = req_builder.version(parts.version); + // Don't set body if body is empty. + if !body.is_empty() { + req_builder = req_builder.body(reqwest::Body::wrap_stream(body)) + } + + let mut resp = req_builder + .send() + .await + .map_err(|err| to_opendal_unexpected_error(err, &uri, "send http request"))?; + + // Get content length from header so that we can check it. + // + // - If the request method is HEAD, we will ignore content length. + // - If response contains content_encoding, we should omit its content length. + let content_length = if is_head || parse_content_encoding(resp.headers())?.is_some() { + None + } else { + parse_content_length(resp.headers())? + }; + + let mut hr = Response::builder() + .status(resp.status()) + // Insert uri into response extension so that we can fetch + // it later. + .extension(uri.clone()); + + hr = hr.version(resp.version()); + + // Swap headers directly instead of copy the entire map. + mem::swap(hr.headers_mut().unwrap(), resp.headers_mut()); + + let bs = HttpBody::new( + resp.bytes_stream() + .try_filter(|v| future::ready(!v.is_empty())) + .map_ok(Buffer::from) + .map_err(move |err| { + to_opendal_unexpected_error(err, &uri, "read data from http response") + }), + content_length, + ); + + let resp = hr.body(bs).expect("response must build succeed"); + Ok(resp) + } +} + +fn to_opendal_unexpected_error(err: reqwest::Error, uri: &http::Uri, desc: &str) -> opendal::Error { + let mut oe = opendal::Error::new(opendal::ErrorKind::Unexpected, desc) + .with_operation("http_util::Client::send") + .with_context("url", uri.to_string()); + if is_temporary_error(&err) { + oe = oe.set_temporary(); + } + oe = oe.set_source(err); + oe +} + +#[inline] +fn is_temporary_error(err: &reqwest::Error) -> bool { + // error sending request + err.is_request()|| + // request or response body error + err.is_body() || + // error decoding response body, for example, connection reset. + err.is_decode() +} diff --git a/src/common/storage/src/lib.rs b/src/common/storage/src/lib.rs index 60c505d2cc6ac..4777206435530 100644 --- a/src/common/storage/src/lib.rs +++ b/src/common/storage/src/lib.rs @@ -35,6 +35,9 @@ mod config; pub use config::ShareTableConfig; pub use config::StorageConfig; +mod http_client; +pub use http_client::StorageHttpClient; + mod operator; pub use operator::build_operator; pub use operator::init_operator; diff --git a/src/common/storage/src/operator.rs b/src/common/storage/src/operator.rs index 3383c333e3e5c..03b80404df1fb 100644 --- a/src/common/storage/src/operator.rs +++ b/src/common/storage/src/operator.rs @@ -20,7 +20,6 @@ use std::time::Duration; use anyhow::anyhow; use databend_common_base::base::GlobalInstance; -use databend_common_base::http_client::StorageHttpClient; use databend_common_base::runtime::GlobalIORuntime; use databend_common_base::runtime::TrySpawn; use databend_common_exception::ErrorCode; @@ -57,6 +56,7 @@ use opendal::Operator; use crate::metrics_layer::METRICS_LAYER; use crate::runtime_layer::RuntimeLayer; use crate::StorageConfig; +use crate::StorageHttpClient; /// init_operator will init an opendal operator based on storage config. pub fn init_operator(cfg: &StorageParams) -> Result { From 29dcf3a776a8ed2e7efc6edda7ce0e30753844f3 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 31 Oct 2024 12:08:23 +0800 Subject: [PATCH 04/13] z --- src/common/storage/src/http_client.rs | 4 ++-- src/common/storage/src/operator.rs | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/common/storage/src/http_client.rs b/src/common/storage/src/http_client.rs index 270e39e029028..ede8977ad6903 100644 --- a/src/common/storage/src/http_client.rs +++ b/src/common/storage/src/http_client.rs @@ -30,8 +30,8 @@ pub struct StorageHttpClient { client: reqwest::Client, } -impl StorageHttpClient { - pub fn new() -> Self { +impl Default for StorageHttpClient { + fn default() -> Self { Self { client: GLOBAL_HTTP_CLIENT.inner(), } diff --git a/src/common/storage/src/operator.rs b/src/common/storage/src/operator.rs index 03b80404df1fb..582fee54c85e7 100644 --- a/src/common/storage/src/operator.rs +++ b/src/common/storage/src/operator.rs @@ -180,7 +180,7 @@ pub fn init_azblob_operator(cfg: &StorageAzblobConfig) -> Result { // Credential .account_name(&cfg.account_name) .account_key(&cfg.account_key) - .http_client(HttpClient::with(StorageHttpClient::new())); + .http_client(HttpClient::with(StorageHttpClient::default())); Ok(builder) } @@ -205,7 +205,7 @@ fn init_gcs_operator(cfg: &StorageGcsConfig) -> Result { .bucket(&cfg.bucket) .root(&cfg.root) .credential(&cfg.credential) - .http_client(HttpClient::with(StorageHttpClient::new())); + .http_client(HttpClient::with(StorageHttpClient::default())); Ok(builder) } @@ -302,7 +302,7 @@ fn init_s3_operator(cfg: &StorageS3Config) -> Result { builder = builder.enable_virtual_host_style(); } - builder = builder.http_client(HttpClient::with(StorageHttpClient::new())); + builder = builder.http_client(HttpClient::with(StorageHttpClient::default())); Ok(builder) } @@ -319,7 +319,7 @@ fn init_obs_operator(cfg: &StorageObsConfig) -> Result { // Credential .access_key_id(&cfg.access_key_id) .secret_access_key(&cfg.secret_access_key) - .http_client(HttpClient::with(StorageHttpClient::new())); + .http_client(HttpClient::with(StorageHttpClient::default())); Ok(builder) } @@ -335,7 +335,7 @@ fn init_oss_operator(cfg: &StorageOssConfig) -> Result { .root(&cfg.root) .server_side_encryption(&cfg.server_side_encryption) .server_side_encryption_key_id(&cfg.server_side_encryption_key_id) - .http_client(HttpClient::with(StorageHttpClient::new())); + .http_client(HttpClient::with(StorageHttpClient::default())); Ok(builder) } @@ -368,7 +368,7 @@ fn init_cos_operator(cfg: &StorageCosConfig) -> Result { .secret_key(&cfg.secret_key) .bucket(&cfg.bucket) .root(&cfg.root) - .http_client(HttpClient::with(StorageHttpClient::new())); + .http_client(HttpClient::with(StorageHttpClient::default())); Ok(builder) } From 0ffba287a440f272208a03f66b521f8405bc5e87 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 31 Oct 2024 12:09:05 +0800 Subject: [PATCH 05/13] z --- src/common/storage/src/http_client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/storage/src/http_client.rs b/src/common/storage/src/http_client.rs index ede8977ad6903..a8bbfabd7d3ca 100644 --- a/src/common/storage/src/http_client.rs +++ b/src/common/storage/src/http_client.rs @@ -105,11 +105,11 @@ impl HttpFetch for StorageHttpClient { fn to_opendal_unexpected_error(err: reqwest::Error, uri: &http::Uri, desc: &str) -> opendal::Error { let mut oe = opendal::Error::new(opendal::ErrorKind::Unexpected, desc) .with_operation("http_util::Client::send") - .with_context("url", uri.to_string()); + .with_context("url", uri.to_string()) + .set_source(err); if is_temporary_error(&err) { oe = oe.set_temporary(); } - oe = oe.set_source(err); oe } From 02a13cb91c25ca6f9d996ce938743bebad432d32 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 31 Oct 2024 12:10:20 +0800 Subject: [PATCH 06/13] z --- src/common/storage/src/http_client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/storage/src/http_client.rs b/src/common/storage/src/http_client.rs index a8bbfabd7d3ca..ede8977ad6903 100644 --- a/src/common/storage/src/http_client.rs +++ b/src/common/storage/src/http_client.rs @@ -105,11 +105,11 @@ impl HttpFetch for StorageHttpClient { fn to_opendal_unexpected_error(err: reqwest::Error, uri: &http::Uri, desc: &str) -> opendal::Error { let mut oe = opendal::Error::new(opendal::ErrorKind::Unexpected, desc) .with_operation("http_util::Client::send") - .with_context("url", uri.to_string()) - .set_source(err); + .with_context("url", uri.to_string()); if is_temporary_error(&err) { oe = oe.set_temporary(); } + oe = oe.set_source(err); oe } From e10e9a02cce68e76a9a35aeae4a029c08bb748e6 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 31 Oct 2024 15:48:23 +0800 Subject: [PATCH 07/13] z --- .github/workflows/merge_group.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge_group.yml b/.github/workflows/merge_group.yml index 04d3eac06c97e..4d2ede8a84c90 100644 --- a/.github/workflows/merge_group.yml +++ b/.github/workflows/merge_group.yml @@ -1,4 +1,4 @@ -name: "PR Assistant" +name: "Merge Group Helper" on: merge_group: From 1811fc49f72f7c16e24822ac20992a01e3f3734c Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 31 Oct 2024 15:49:06 +0800 Subject: [PATCH 08/13] z --- .github/workflows/dev.yml | 2 +- .github/workflows/production.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index b9e32f21a8a81..c011532ce66c6 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -23,7 +23,7 @@ jobs: with: fetch-depth: 0 - name: Check Source File Changes - uses: tj-actions/changed-files@v39 + uses: tj-actions/changed-files@v45 id: src with: files_ignore: | diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml index 762827ca06fd8..45d0c7a87b42a 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/production.yml @@ -16,7 +16,7 @@ jobs: with: fetch-depth: 0 - name: Check Source File Changes - uses: tj-actions/changed-files@v39 + uses: tj-actions/changed-files@v45 id: src with: files_ignore: | From b1cc139af0f1413868f3d2ac188389625d6bb742 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 31 Oct 2024 16:23:15 +0800 Subject: [PATCH 09/13] z --- .github/workflows/merge_group.yml | 63 +++++++++++++++++++++++++++++++ .github/workflows/production.yml | 1 - 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/.github/workflows/merge_group.yml b/.github/workflows/merge_group.yml index 4d2ede8a84c90..76294f82cbfd6 100644 --- a/.github/workflows/merge_group.yml +++ b/.github/workflows/merge_group.yml @@ -17,3 +17,66 @@ jobs: - name: Skip CLA check shell: bash run: exit 0 + + changes: + runs-on: ubuntu-latest + outputs: + any_src_changed: ${{ steps.src.outputs.any_changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check Source File Changes + uses: tj-actions/changed-files@v45 + id: src + with: + files_ignore: | + .github/** + **.md + benchmark/** + docker/** + scripts/setup/** + .devcontainer/** + - name: Output Source File Changes + run: | + if [[ "${{ steps.src.outputs.any_changed }}" == "true" ]]; then + echo "these src files changed:" >> $GITHUB_STEP_SUMMARY + for line in ${{ steps.src.outputs.all_changed_files }}; do + echo "- $line" >> $GITHUB_STEP_SUMMARY + done + else + echo "no src file changes detected" >> $GITHUB_STEP_SUMMARY + fi + + linux: + needs: changes + if: needs.changes.outputs.any_src_changed == 'true' + uses: ./.github/workflows/reuse.linux.yml + secrets: inherit + with: + build_profile: release + runner_provider: aws + + ready: + if: always() + runs-on: ubuntu-latest + needs: + - changes + - linux + steps: + - name: Check Ready to Merge + uses: actions/github-script@v7 + env: + SRC_CHANGED: ${{ needs.changes.outputs.any_src_changed }} + LINUX_BUILD_RESULT: ${{ needs.linux.result }} + with: + script: | + if (process.env.SRC_CHANGED == 'false') { + core.info('No source file changes detected, skipping'); + return; + } + if (process.env.LINUX_BUILD_RESULT == 'success') { + core.info('Linux build succeeded, ready to merge'); + return; + } + core.setFailed('Build failed, not ready to merge'); diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml index 45d0c7a87b42a..442eda11342ff 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/production.yml @@ -4,7 +4,6 @@ on: push: branches: - main - merge_group: jobs: changes: From 0881c0dd86f2480e9d1af432a5a4a21553b86b7c Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 31 Oct 2024 16:24:22 +0800 Subject: [PATCH 10/13] z --- .github/workflows/{production.yml => main.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{production.yml => main.yml} (99%) diff --git a/.github/workflows/production.yml b/.github/workflows/main.yml similarity index 99% rename from .github/workflows/production.yml rename to .github/workflows/main.yml index 442eda11342ff..e22d76b029d24 100644 --- a/.github/workflows/production.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: Production +name: Main on: push: From 5a34aeb31898bc7fbccdf11dc81e84f379332e82 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 31 Oct 2024 16:26:11 +0800 Subject: [PATCH 11/13] z --- .github/workflows/merge_group.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge_group.yml b/.github/workflows/merge_group.yml index 76294f82cbfd6..bc69873db0686 100644 --- a/.github/workflows/merge_group.yml +++ b/.github/workflows/merge_group.yml @@ -1,4 +1,4 @@ -name: "Merge Group Helper" +name: "Merge Group" on: merge_group: From f4feae8cbc721229d5387d1686f448cc9af24bc9 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Thu, 31 Oct 2024 16:32:35 +0800 Subject: [PATCH 12/13] z --- .github/workflows/reuse.sqllogic.yml | 52 +++++++++++++++++++++------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/.github/workflows/reuse.sqllogic.yml b/.github/workflows/reuse.sqllogic.yml index 358a414f0d8ee..db3697bc5b56c 100644 --- a/.github/workflows/reuse.sqllogic.yml +++ b/.github/workflows/reuse.sqllogic.yml @@ -20,7 +20,7 @@ env: jobs: management_mode: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_sqllogic_management_mode_linux @@ -30,12 +30,11 @@ jobs: handlers: mysql,http standalone: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: dirs: - - "query" - "standalone" - "crdb" - "duckdb" @@ -60,8 +59,35 @@ jobs: with: name: test-sqllogic-standalone-${{ matrix.dirs }}-${{ matrix.handler }} + standalone_query: + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + strategy: + fail-fast: false + matrix: + dirs: + - "query" + handler: + - "mysql" + - "http" + format: + - "native" + - "parquet" + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/test_sqllogic_standalone_linux + timeout-minutes: 15 + with: + dirs: ${{ matrix.dirs }} + handlers: ${{ matrix.handler }} + storage-format: ${{ matrix.format }} + - name: Upload failure + if: failure() + uses: ./.github/actions/artifact_failure + with: + name: test-sqllogic-standalone-query-${{ matrix.handler }}-${{ matrix.format }} + standalone_udf_server: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - name: Start UDF Server @@ -82,7 +108,7 @@ jobs: name: test-sqllogic-standalone-udf-server standalone_cloud: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - name: Start Cloud Control Server @@ -103,7 +129,7 @@ jobs: name: test-sqllogic-standalone-cloud standalone_minio: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: @@ -130,13 +156,13 @@ jobs: name: test-sqllogic-standalone-minio-${{ matrix.dirs }}-${{ matrix.handler }}-${{ matrix.format }} standalone_iceberg_tpch: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: '17' + distribution: "temurin" + java-version: "17" - uses: ./.github/actions/test_sqllogic_iceberg_tpch timeout-minutes: 15 with: @@ -149,7 +175,7 @@ jobs: name: test-sqllogic-standalone-iceberg-tpch cluster: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: @@ -182,7 +208,7 @@ jobs: name: test-sqllogic-cluster-${{ matrix.dirs }}-${{ matrix.handler }} stage: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: @@ -204,7 +230,7 @@ jobs: name: test-sqllogic-stage-${{ matrix.storage }} standalone_no_table_meta_cache: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: @@ -228,7 +254,7 @@ jobs: name: test-sqllogic-standalone-no-table-meta-cache-${{ matrix.dirs }}-${{ matrix.handler }} ee: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: From ef7d1bb58075b1b4f6d93c1b1fe3463171e59aea Mon Sep 17 00:00:00 2001 From: everpcpc Date: Fri, 1 Nov 2024 13:05:42 +0800 Subject: [PATCH 13/13] z --- .github/workflows/dev.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index c011532ce66c6..948829c940043 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -50,7 +50,7 @@ jobs: uses: ./.github/workflows/reuse.linux.yml secrets: inherit with: - build_profile: debug + build_profile: release runner_provider: aws linux_hive: @@ -59,7 +59,7 @@ jobs: uses: ./.github/workflows/reuse.linux.hive.yml secrets: inherit with: - build_profile: debug + build_profile: release runner_provider: aws ready: