From d5865bedf5aee7feb0fa9f7c228a28efb12ceeb3 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Fri, 6 Aug 2021 14:06:27 +0800 Subject: [PATCH 1/7] init: TLS http api tests --- fusequery/query/src/api/http_service_test.rs | 33 ++++++++++++++++++++ fusequery/query/src/api/mod.rs | 2 ++ fusequery/query/src/api/rpc_service.rs | 4 +-- 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 fusequery/query/src/api/http_service_test.rs diff --git a/fusequery/query/src/api/http_service_test.rs b/fusequery/query/src/api/http_service_test.rs new file mode 100644 index 0000000000000..68e65f12dbed9 --- /dev/null +++ b/fusequery/query/src/api/http_service_test.rs @@ -0,0 +1,33 @@ +// Copyright 2020-2021 The Datafuse Authors. +// +// SPDX-License-Identifier: Apache-2.0. + +//use std::net::SocketAddr; +//use std::sync::Arc; +// +use common_exception::Result; +use common_runtime::tokio; +//use common_runtime::tokio::net::TcpListener; +// +//use crate::api::http::router::Router; +//use crate::api::HttpService; +//use crate::clusters::Cluster; +//use crate::clusters::ClusterRef; +//use crate::configs::Config; +//use crate::servers::Server; + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn test_http_service_tls_server() -> Result<()> { + //let mut conf = Config::default(); + //conf.rpc_tls_server_key = "../../tests/data/certs/server.key".to_owned(); + //conf.rpc_tls_server_cert = "../../tests/data/certs/server.pem".to_owned(); + + //let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + //let address = listener.local_addr().unwrap(); + + //let cluster = Cluster::create_global(conf.clone())?; + //let listening = conf.http_api_address.parse::()?; + //let mut srv = HttpService::create(conf.clone(), cluster.clone()); + //let listening = srv.start(address).await?; + Ok(()) +} diff --git a/fusequery/query/src/api/mod.rs b/fusequery/query/src/api/mod.rs index 0bfdea5d96cbc..48aa2dba0599b 100644 --- a/fusequery/query/src/api/mod.rs +++ b/fusequery/query/src/api/mod.rs @@ -17,5 +17,7 @@ mod http_service; mod rpc; mod rpc_service; +#[cfg(test)] +mod http_service_test; #[cfg(test)] mod rpc_service_test; diff --git a/fusequery/query/src/api/rpc_service.rs b/fusequery/query/src/api/rpc_service.rs index 1fbfa7efd739b..0f05436b4e09d 100644 --- a/fusequery/query/src/api/rpc_service.rs +++ b/fusequery/query/src/api/rpc_service.rs @@ -102,8 +102,8 @@ impl FuseQueryServer for RpcService { } async fn start(&mut self, listening: SocketAddr) -> Result { - let (listener_stream, listening) = Self::listener_tcp(listening).await?; + let (listener_stream, listener_addr) = Self::listener_tcp(listening).await?; self.start_with_incoming(listener_stream).await?; - Ok(listening) + Ok(listener_addr) } } From 1b079a2c8ed3ff24a9a233c03090a2d28376e4ee Mon Sep 17 00:00:00 2001 From: dantengsky Date: Fri, 6 Aug 2021 14:11:52 +0800 Subject: [PATCH 2/7] return HttpService instead of trait object --- fusequery/query/src/api/http_service.rs | 2 +- fusequery/query/src/bin/fuse-query.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fusequery/query/src/api/http_service.rs b/fusequery/query/src/api/http_service.rs index b9e6ddc2c4e06..231276fb2652a 100644 --- a/fusequery/query/src/api/http_service.rs +++ b/fusequery/query/src/api/http_service.rs @@ -24,7 +24,7 @@ pub struct HttpService { } impl HttpService { - pub fn create(cfg: Config, cluster: ClusterRef) -> Box { + pub fn create(cfg: Config, cluster: ClusterRef) -> Box { Box::new(HttpService { cfg, cluster, diff --git a/fusequery/query/src/bin/fuse-query.rs b/fusequery/query/src/bin/fuse-query.rs index b2a8c974263ba..587dd1cfea345 100644 --- a/fusequery/query/src/bin/fuse-query.rs +++ b/fusequery/query/src/bin/fuse-query.rs @@ -13,6 +13,7 @@ use fuse_query::configs::Config; use fuse_query::metrics::MetricService; use fuse_query::servers::ClickHouseHandler; use fuse_query::servers::MySQLHandler; +use fuse_query::servers::Server; use fuse_query::servers::ShutdownHandle; use fuse_query::sessions::SessionManager; use log::info; From 6731cd24fae989139e99a0524f3782eabe95b262 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Fri, 6 Aug 2021 15:37:23 +0800 Subject: [PATCH 3/7] unit test for tls api service --- Cargo.lock | 93 +++++++++++++++++++- fusequery/query/Cargo.toml | 1 + fusequery/query/src/api/http_service_test.rs | 54 ++++++++---- fusequery/query/src/configs/config.rs | 8 +- 4 files changed, 134 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2cecedc5518e6..6733ccf9361af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1529,6 +1529,15 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encoding_rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -1867,6 +1876,7 @@ dependencies = [ "prost 0.7.0", "quantiles", "rand 0.8.4", + "reqwest", "serde", "serde_json", "sqlparser", @@ -2337,6 +2347,19 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.2.3" @@ -2433,7 +2456,7 @@ dependencies = [ "socket2 0.3.19", "widestring", "winapi", - "winreg", + "winreg 0.6.2", ] [[package]] @@ -4209,6 +4232,41 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.7.0", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -5238,6 +5296,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.22.0" @@ -5832,6 +5900,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b608ecc8f4198fe8680e2ed18eccab5f0cd4caaf3d83516fa5fb2e927fda2586" dependencies = [ "cfg-if 1.0.0", + "serde", + "serde_json", "wasm-bindgen-macro", ] @@ -5850,6 +5920,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16646b21c3add8e13fdb8f20172f8a28c3dbf62f45406bcff0233188226cfe0c" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.75" @@ -5974,6 +6056,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] + [[package]] name = "wyz" version = "0.4.0" diff --git a/fusequery/query/Cargo.toml b/fusequery/query/Cargo.toml index 535f9edab70ac..dd54f69dc2260 100644 --- a/fusequery/query/Cargo.toml +++ b/fusequery/query/Cargo.toml @@ -90,6 +90,7 @@ lru = "0.6.6" pretty_assertions = "0.7" criterion = "0.3" mysql = "21.0.1" +reqwest = { version = "0.11", features = ["json", "native-tls"] } [build-dependencies] common-building = {path = "../../common/building"} diff --git a/fusequery/query/src/api/http_service_test.rs b/fusequery/query/src/api/http_service_test.rs index 68e65f12dbed9..ccc2b1b2e9ae0 100644 --- a/fusequery/query/src/api/http_service_test.rs +++ b/fusequery/query/src/api/http_service_test.rs @@ -5,29 +5,49 @@ //use std::net::SocketAddr; //use std::sync::Arc; // +use std::fs::File; +use std::io::Read; + use common_exception::Result; use common_runtime::tokio; -//use common_runtime::tokio::net::TcpListener; -// -//use crate::api::http::router::Router; -//use crate::api::HttpService; -//use crate::clusters::Cluster; -//use crate::clusters::ClusterRef; -//use crate::configs::Config; -//use crate::servers::Server; + +use crate::api::HttpService; +use crate::clusters::Cluster; +use crate::configs::Config; +use crate::servers::Server; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn test_http_service_tls_server() -> Result<()> { - //let mut conf = Config::default(); - //conf.rpc_tls_server_key = "../../tests/data/certs/server.key".to_owned(); - //conf.rpc_tls_server_cert = "../../tests/data/certs/server.pem".to_owned(); + let mut conf = Config::default(); + + conf.tls_server_key = "../../tests/data/certs/server.key".to_owned(); + conf.tls_server_cert = "../../tests/data/certs/server.pem".to_owned(); + + let addr_str = "127.0.0.1:0"; + let cluster = Cluster::create_global(conf.clone())?; + let mut srv = HttpService::create(conf.clone(), cluster.clone()); + let listening = srv.start(addr_str.parse()?).await?; + let port = listening.port(); + + // test cert is issued for "localhost" + let url = format!("https://localhost:{}/v1/hello", port); + + // load cert + let mut buf = Vec::new(); + File::open("../../tests/data/certs/ca.pem")?.read_to_end(&mut buf)?; + let cert = reqwest::Certificate::from_pem(&buf).unwrap(); - //let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - //let address = listener.local_addr().unwrap(); + // kick off + let client = reqwest::Client::builder() + .add_root_certificate(cert) + .danger_accept_invalid_hostnames(true) + .build() + .unwrap(); + let resp = client.get(url).send().await; + assert!(resp.is_ok()); + let resp = resp.unwrap(); + assert!(resp.status().is_success()); + assert_eq!("/v1/hello", resp.url().path()); - //let cluster = Cluster::create_global(conf.clone())?; - //let listening = conf.http_api_address.parse::()?; - //let mut srv = HttpService::create(conf.clone(), cluster.clone()); - //let listening = srv.start(address).await?; Ok(()) } diff --git a/fusequery/query/src/configs/config.rs b/fusequery/query/src/configs/config.rs index 7d1cd3207fa35..1cb598a24d4e9 100644 --- a/fusequery/query/src/configs/config.rs +++ b/fusequery/query/src/configs/config.rs @@ -163,14 +163,14 @@ pub struct Config { long, env = "RPC_TLS_SERVER_CERT", default_value = "", - help = "server cert" + help = "rpc server cert" )] pub rpc_tls_server_cert: String, #[structopt( long, env = "RPC_TLS_SERVER_KEY", - default_value = "key for server cert" + default_value = "key for rpc server cert" )] pub rpc_tls_server_key: String, @@ -178,7 +178,7 @@ pub struct Config { long, env = "RPC_TLS_QUERY_SERVER_ROOT_CA_CERT", default_value = "", - help = "Certificate for client to identify query server" + help = "Certificate for client to identify query rpc server" )] pub rpc_tls_query_server_root_ca_cert: String, @@ -193,7 +193,7 @@ pub struct Config { long, env = "RPC_TLS_STORE_SERVER_ROOT_CA_CERT", default_value = "", - help = "Certificate for client to identify query server" + help = "Certificate for client to identify store rpc server" )] pub rpc_tls_store_server_root_ca_cert: String, From b3192fedcc6c6ccf747e8a7923b0e19e550f4a0d Mon Sep 17 00:00:00 2001 From: dantengsky Date: Fri, 6 Aug 2021 15:42:31 +0800 Subject: [PATCH 4/7] rename tsl_server_* to api_tls_server_* --- fusequery/query/src/api/http_service.rs | 4 ++-- fusequery/query/src/api/http_service_test.rs | 4 ++-- fusequery/query/src/configs/config.rs | 21 ++++++++++++-------- fusequery/query/src/configs/config_test.rs | 4 ++-- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/fusequery/query/src/api/http_service.rs b/fusequery/query/src/api/http_service.rs index 231276fb2652a..376c36af23189 100644 --- a/fusequery/query/src/api/http_service.rs +++ b/fusequery/query/src/api/http_service.rs @@ -61,8 +61,8 @@ impl Server for HttpService { let server = warp::serve(router.router()?); let conf = self.cfg.clone(); - let tls_cert = conf.tls_server_cert; - let tls_key = conf.tls_server_key; + let tls_cert = conf.api_tls_server_cert; + let tls_key = conf.api_tls_server_key; if !tls_cert.is_empty() && !tls_key.is_empty() { log::info!("Http API TLS enabled"); diff --git a/fusequery/query/src/api/http_service_test.rs b/fusequery/query/src/api/http_service_test.rs index ccc2b1b2e9ae0..919f51468867f 100644 --- a/fusequery/query/src/api/http_service_test.rs +++ b/fusequery/query/src/api/http_service_test.rs @@ -20,8 +20,8 @@ use crate::servers::Server; async fn test_http_service_tls_server() -> Result<()> { let mut conf = Config::default(); - conf.tls_server_key = "../../tests/data/certs/server.key".to_owned(); - conf.tls_server_cert = "../../tests/data/certs/server.pem".to_owned(); + conf.api_tls_server_key = "../../tests/data/certs/server.key".to_owned(); + conf.api_tls_server_cert = "../../tests/data/certs/server.pem".to_owned(); let addr_str = "127.0.0.1:0"; let cluster = Cluster::create_global(conf.clone())?; diff --git a/fusequery/query/src/configs/config.rs b/fusequery/query/src/configs/config.rs index 1cb598a24d4e9..8e3326173164e 100644 --- a/fusequery/query/src/configs/config.rs +++ b/fusequery/query/src/configs/config.rs @@ -63,8 +63,8 @@ const STORE_API_ADDRESS: &str = "STORE_API_ADDRESS"; const STORE_API_USERNAME: &str = "STORE_API_USERNAME"; const STORE_API_PASSWORD: &str = "STORE_API_PASSWORD"; -const TLS_SERVER_CERT: &str = "TLS_SERVER_CERT"; -const TLS_SERVER_KEY: &str = "TLS_SERVER_KEY"; +const API_TLS_SERVER_CERT: &str = "API_TLS_SERVER_CERT"; +const API_TLS_SERVER_KEY: &str = "API_TLS_SERVER_KEY"; const DISABLE_REMOTE_CATALOG: &str = "DISABLE_REMOTE_CATALOG"; @@ -153,11 +153,11 @@ pub struct Config { #[structopt(long, short = "c", env = CONFIG_FILE, default_value = "")] pub config_file: String, - #[structopt(long, env = TLS_SERVER_CERT, default_value = "")] - pub tls_server_cert: String, + #[structopt(long, env = API_TLS_SERVER_CERT, default_value = "")] + pub api_tls_server_cert: String, - #[structopt(long, env = TLS_SERVER_KEY, default_value = "")] - pub tls_server_key: String, + #[structopt(long, env = API_TLS_SERVER_KEY, default_value = "")] + pub api_tls_server_key: String, #[structopt( long, @@ -297,8 +297,8 @@ impl Config { store_api_password: "root".to_string(), }, config_file: "".to_string(), - tls_server_cert: "".to_string(), - tls_server_key: "".to_string(), + api_tls_server_cert: "".to_string(), + api_tls_server_key: "".to_string(), rpc_tls_server_cert: "".to_string(), rpc_tls_server_key: "".to_string(), rpc_tls_query_server_root_ca_cert: "".to_string(), @@ -363,6 +363,11 @@ impl Config { env_helper!(mut_config, store_api_username, User, STORE_API_USERNAME); env_helper!(mut_config, store_api_password, Password, STORE_API_PASSWORD); + // for api http service + env_helper!(mut_config, api_tls_server_cert, String, API_TLS_SERVER_CERT); + + env_helper!(mut_config, api_tls_server_key, String, API_TLS_SERVER_KEY); + // for query rpc server env_helper!(mut_config, rpc_tls_server_cert, String, RPC_TLS_SERVER_CERT); diff --git a/fusequery/query/src/configs/config_test.rs b/fusequery/query/src/configs/config_test.rs index 3cc7706f0bacd..39f8002f2180d 100644 --- a/fusequery/query/src/configs/config_test.rs +++ b/fusequery/query/src/configs/config_test.rs @@ -32,8 +32,8 @@ fn test_default_config() -> Result<()> { store_api_password: "root".to_string(), }, config_file: "".to_string(), - tls_server_cert: "".to_string(), - tls_server_key: "".to_string(), + api_tls_server_cert: "".to_string(), + api_tls_server_key: "".to_string(), rpc_tls_server_cert: "".to_string(), rpc_tls_server_key: "".to_string(), rpc_tls_query_server_root_ca_cert: "".to_string(), From 96b417c364f32cc37fd9d0ea6a4589debed7b855 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Fri, 6 Aug 2021 16:05:19 +0800 Subject: [PATCH 5/7] move tests/data/certs to tests/certs --- fusequery/query/src/api/http_service_test.rs | 12 ++++++++---- fusequery/query/src/api/rpc_service_test.rs | 14 +++++++++----- fusequery/query/src/tests/mod.rs | 1 + fusequery/query/src/tests/tls_constants.rs | 9 +++++++++ .../store/src/api/rpc/tls_flight_service_test.rs | 15 ++++++++++----- scripts/ci/ci-run-stateless-tests-cluster-tls.sh | 8 ++++---- tests/{data => }/certs/ca.pem | 0 tests/{data => }/certs/server.key | 0 tests/{data => }/certs/server.pem | 0 9 files changed, 41 insertions(+), 18 deletions(-) create mode 100644 fusequery/query/src/tests/tls_constants.rs rename tests/{data => }/certs/ca.pem (100%) rename tests/{data => }/certs/server.key (100%) rename tests/{data => }/certs/server.pem (100%) diff --git a/fusequery/query/src/api/http_service_test.rs b/fusequery/query/src/api/http_service_test.rs index 919f51468867f..e9b81b81b3ae8 100644 --- a/fusequery/query/src/api/http_service_test.rs +++ b/fusequery/query/src/api/http_service_test.rs @@ -15,13 +15,17 @@ use crate::api::HttpService; use crate::clusters::Cluster; use crate::configs::Config; use crate::servers::Server; +use crate::tests::tls_constants::TEST_CA_CERT; +use crate::tests::tls_constants::TEST_CN_NAME; +use crate::tests::tls_constants::TEST_SERVER_CERT; +use crate::tests::tls_constants::TEST_SERVER_KEY; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn test_http_service_tls_server() -> Result<()> { let mut conf = Config::default(); - conf.api_tls_server_key = "../../tests/data/certs/server.key".to_owned(); - conf.api_tls_server_cert = "../../tests/data/certs/server.pem".to_owned(); + conf.api_tls_server_key = TEST_SERVER_KEY.to_owned(); + conf.api_tls_server_cert = TEST_SERVER_CERT.to_owned(); let addr_str = "127.0.0.1:0"; let cluster = Cluster::create_global(conf.clone())?; @@ -30,11 +34,11 @@ async fn test_http_service_tls_server() -> Result<()> { let port = listening.port(); // test cert is issued for "localhost" - let url = format!("https://localhost:{}/v1/hello", port); + let url = format!("https://{}:{}/v1/hello", TEST_CN_NAME, port); // load cert let mut buf = Vec::new(); - File::open("../../tests/data/certs/ca.pem")?.read_to_end(&mut buf)?; + File::open(TEST_CA_CERT)?.read_to_end(&mut buf)?; let cert = reqwest::Certificate::from_pem(&buf).unwrap(); // kick off diff --git a/fusequery/query/src/api/rpc_service_test.rs b/fusequery/query/src/api/rpc_service_test.rs index 67dfd492d2c9d..d2b8737afc0d0 100644 --- a/fusequery/query/src/api/rpc_service_test.rs +++ b/fusequery/query/src/api/rpc_service_test.rs @@ -20,13 +20,17 @@ use crate::clusters::Cluster; use crate::configs::Config; use crate::configs::RpcClientTlsConfig; use crate::sessions::SessionManager; +use crate::tests::tls_constants::TEST_CA_CERT; +use crate::tests::tls_constants::TEST_CN_NAME; +use crate::tests::tls_constants::TEST_SERVER_CERT; +use crate::tests::tls_constants::TEST_SERVER_KEY; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn test_tls_rpc_server() -> Result<()> { // setup let mut conf = Config::default(); - conf.rpc_tls_server_key = "../../tests/data/certs/server.key".to_owned(); - conf.rpc_tls_server_cert = "../../tests/data/certs/server.pem".to_owned(); + conf.rpc_tls_server_key = TEST_SERVER_KEY.to_owned(); + conf.rpc_tls_server_cert = TEST_SERVER_CERT.to_owned(); let cluster = Cluster::create_global(conf.clone())?; let session_manager = SessionManager::from_conf(conf.clone(), cluster.clone())?; @@ -43,8 +47,8 @@ async fn test_tls_rpc_server() -> Result<()> { srv.start_with_incoming(stream).await?; let client_conf = RpcClientTlsConfig { - rpc_tls_server_root_ca_cert: "../../tests/data/certs/ca.pem".to_string(), - domain_name: "localhost".to_string(), + rpc_tls_server_root_ca_cert: TEST_CA_CERT.to_string(), + domain_name: TEST_CN_NAME.to_string(), }; // normal case @@ -95,7 +99,7 @@ async fn test_tls_rpc_server_invalid_client_config() -> Result<()> { // setup, invalid cert locations let client_conf = RpcClientTlsConfig { rpc_tls_server_root_ca_cert: "../../tests/data/certs/nowhere.pem".to_string(), - domain_name: "localhost".to_string(), + domain_name: TEST_CN_NAME.to_string(), }; let r = ConnectionFactory::create_flight_channel("fake:1234", None, Some(client_conf)).await; diff --git a/fusequery/query/src/tests/mod.rs b/fusequery/query/src/tests/mod.rs index e8342cb76a37b..a2187646d3a7b 100644 --- a/fusequery/query/src/tests/mod.rs +++ b/fusequery/query/src/tests/mod.rs @@ -6,6 +6,7 @@ mod context; mod number; mod parse_query; mod sessions; +pub(crate) mod tls_constants; pub use context::try_create_cluster_context; pub use context::try_create_context; diff --git a/fusequery/query/src/tests/tls_constants.rs b/fusequery/query/src/tests/tls_constants.rs new file mode 100644 index 0000000000000..6fe461d5faaa5 --- /dev/null +++ b/fusequery/query/src/tests/tls_constants.rs @@ -0,0 +1,9 @@ +// Copyright 2020-2021 The Datafuse Authors. +// +// SPDX-License-Identifier: Apache-2.0. +// + +pub const TEST_CA_CERT: &'static str = "../../tests/certs/ca.pem"; +pub const TEST_SERVER_CERT: &'static str = "../../tests/certs/server.pem"; +pub const TEST_SERVER_KEY: &'static str = "../../tests/certs/server.key"; +pub const TEST_CN_NAME: &'static str = "localhost"; diff --git a/fusestore/store/src/api/rpc/tls_flight_service_test.rs b/fusestore/store/src/api/rpc/tls_flight_service_test.rs index 69e85db2f4b70..2f376f76c4a8b 100644 --- a/fusestore/store/src/api/rpc/tls_flight_service_test.rs +++ b/fusestore/store/src/api/rpc/tls_flight_service_test.rs @@ -12,14 +12,19 @@ use pretty_assertions::assert_eq; use crate::tests::service::new_test_context; use crate::tests::start_store_server_with_context; +const TEST_CA_CERT: &'static str = "../../tests/certs/ca.pem"; +const TEST_SERVER_CERT: &'static str = "../../tests/certs/server.pem"; +const TEST_SERVER_KEY: &'static str = "../../tests/certs/server.key"; +const TEST_CN_NAME: &'static str = "localhost"; + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn test_flight_tls() -> anyhow::Result<()> { common_tracing::init_default_tracing(); let mut tc = new_test_context(); - tc.config.rpc_tls_server_key = "../../tests/data/certs/server.key".to_owned(); - tc.config.rpc_tls_server_cert = "../../tests/data/certs/server.pem".to_owned(); + tc.config.rpc_tls_server_key = TEST_SERVER_KEY.to_owned(); + tc.config.rpc_tls_server_cert = TEST_SERVER_CERT.to_owned(); let r = start_store_server_with_context(&mut tc).await; assert!(r.is_ok()); @@ -27,8 +32,8 @@ async fn test_flight_tls() -> anyhow::Result<()> { let addr = tc.config.flight_api_address.clone(); let tls_conf = RpcClientTlsConfig { - rpc_tls_server_root_ca_cert: "../../tests/data/certs/ca.pem".to_string(), - domain_name: "localhost".to_string(), + rpc_tls_server_root_ca_cert: TEST_CA_CERT.to_string(), + domain_name: TEST_CN_NAME.to_string(), }; let mut client = @@ -63,7 +68,7 @@ async fn test_flight_tls_client_config_failure() -> anyhow::Result<()> { let tls_conf = RpcClientTlsConfig { rpc_tls_server_root_ca_cert: "../../tests/data/certs/not_exist.pem".to_string(), - domain_name: "localhost".to_string(), + domain_name: TEST_CN_NAME.to_string(), }; let r = StoreClient::with_tls_conf("addr", "root", "xxx", Some(tls_conf)).await; diff --git a/scripts/ci/ci-run-stateless-tests-cluster-tls.sh b/scripts/ci/ci-run-stateless-tests-cluster-tls.sh index 452fd16a734e9..07ba9ae385b7f 100755 --- a/scripts/ci/ci-run-stateless-tests-cluster-tls.sh +++ b/scripts/ci/ci-run-stateless-tests-cluster-tls.sh @@ -4,11 +4,11 @@ echo "EXPORTING TLS RPC CONFIGURATION ENV VARS" set -x -export RPC_TLS_SERVER_CERT="./tests/data/certs/server.pem"; -export RPC_TLS_SERVER_KEY="./tests/data/certs/server.key"; -export RPC_TLS_QUERY_SERVER_ROOT_CA_CERT="./tests/data/certs/ca.pem"; +export RPC_TLS_SERVER_CERT="./tests/certs/server.pem"; +export RPC_TLS_SERVER_KEY="./tests/certs/server.key"; +export RPC_TLS_QUERY_SERVER_ROOT_CA_CERT="./tests/certs/ca.pem"; export RPC_TLS_QUERY_SERVICE_DOMAIN_NAME="localhost"; -export RPC_TLS_STORE_SERVER_ROOT_CA_CERT="./tests/data/certs/ca.pem"; +export RPC_TLS_STORE_SERVER_ROOT_CA_CERT="./tests/certs/ca.pem"; export RPC_TLS_STORE_SERVICE_DOMAIN_NAME="localhost"; set +x diff --git a/tests/data/certs/ca.pem b/tests/certs/ca.pem similarity index 100% rename from tests/data/certs/ca.pem rename to tests/certs/ca.pem diff --git a/tests/data/certs/server.key b/tests/certs/server.key similarity index 100% rename from tests/data/certs/server.key rename to tests/certs/server.key diff --git a/tests/data/certs/server.pem b/tests/certs/server.pem similarity index 100% rename from tests/data/certs/server.pem rename to tests/certs/server.pem From 8a07595583be86bd8bd7829589c7083a7c361e6e Mon Sep 17 00:00:00 2001 From: dantengsky Date: Fri, 6 Aug 2021 16:07:09 +0800 Subject: [PATCH 6/7] remove dir tls --- tls/Makefile | 6 ----- tls/README.md | 47 --------------------------------------- tls/example/test.crt | 31 -------------------------- tls/example/test.key | 52 -------------------------------------------- 4 files changed, 136 deletions(-) delete mode 100644 tls/Makefile delete mode 100644 tls/README.md delete mode 100644 tls/example/test.crt delete mode 100644 tls/example/test.key diff --git a/tls/Makefile b/tls/Makefile deleted file mode 100644 index aff5b26984c81..0000000000000 --- a/tls/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -generate: - mkdir -p certs - openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout certs/private.key -out certs/public.crt - -clean: - rm -rf certs diff --git a/tls/README.md b/tls/README.md deleted file mode 100644 index 17db4431e3a50..0000000000000 --- a/tls/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Enable TLS verification in datafuse -TLS bring security enhancement toward datafuse cluster, and should be supported by default -## Example on how to setup TLS for fuse-query server -1. Generate certificate private key and cert -```bash -make generate -``` -Remember to set up common name as host name(here is 127.0.0.1) -```bash -Country Name (2 letter code) []: -State or Province Name (full name) []: -Locality Name (eg, city) []: -Organization Name (eg, company) []: -Organizational Unit Name (eg, section) []: -Common Name (eg, fully qualified host name) []:127.0.0.1 -Email Address []: -``` -2. start fuse-query server with tls enabled -```bash -./target/release/fuse-query --tls-server-cert ./tls/certs/public.crt --tls-server-key ./tls/certs/private.key -``` -3. test self-signed certificate through curl -Generate pem chain -```bash -echo quit | openssl s_client -showcerts -servername localhost -connect 127.0.0.1:8080 > cacert.pem -``` -Provide pem chain for curl -```bash -curl --cacert cacert.pem https://127.0.0.1:8080/v1/hello -``` -you should see it works -```bash -Config { log_level: "INFO", log_dir: "./_logs", num_cpus: 8, mysql_handler_host: "127.0.0.1", mysql_handler_port: 3307, max_active_sessions: 256, clickhouse_handler_host: "127.0.0.1", clickhouse_handler_port: 9000, flight_api_address: "127.0.0.1:9090", http_api_address: "127.0.0.1:8080", metric_api_address: "127.0.0.1:7070", store_api_address: "127.0.0.1:9191", store_api_username: ******, store_api_password: ******, config_file: "", tls_server_cert: "./tls/certs/public.crt", tls_server_key: "./tls/certs/private.key" } -``` -if we do not provide pem chain -```bash -curl https://127.0.0.1:8080/v1/hello -``` -error message should occur -```bash -curl: (60) SSL certificate problem: self signed certificate -More details here: https://curl.haxx.se/docs/sslcerts.html - -curl failed to verify the legitimacy of the server and therefore could not -establish a secure connection to it. To learn more about this situation and -how to fix it, please visit the web page mentioned above. -``` \ No newline at end of file diff --git a/tls/example/test.crt b/tls/example/test.crt deleted file mode 100644 index 15e76be81eef7..0000000000000 --- a/tls/example/test.crt +++ /dev/null @@ -1,31 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgIUYgBV65+qU1Ehnx2zUibA9RK2kqQwDQYJKoZIhvcNAQEL -BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMTA3MjEwNzQ4NDZaFw0yMjA3 -MjEwNzQ4NDZaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw -HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggIiMA0GCSqGSIb3DQEB -AQUAA4ICDwAwggIKAoICAQDCeeCeSMgNAF8NUTYE5HOgP24/F0kr19BXJBCBL1z2 -D3s4ly7AqcyfP17c7DsgpgWX/bQIw51BfY5fgX7mVTymWYqkZrb0JFEMyYn3tirk -xm2LOohRRP2zMZrMAQhC1uQ5o638mL3tM7CHB9XSDxiT6b4rzZB0TGKf1xJwTRrc -n4JAvwkf4P93FnTj2U9BTL5EENebAyfDLpnqOT6LKC/UCQyA8JVZy9aXmMqBNOLX -Mn7PjddrrjRvZPO9OzsPuBW7PU+y0Qn/xlTS1dKHxzZ1BWziFJqHbEUaeGUT9AZl -dh+MrUbsTV491I731Kq6FxM5IU8GNmJpu9ggGeo1TOnxA+9FIZ28uz0pMmiQDpNJ -10jTymJjrQpBLOR/a7J6akHqOsjo88qp5nhKEORH6npqa1NVpQP4CaMSCAJmLXjY -7FeUuQMK7jH6WNQa9fw/bdQDU+AyMxEFn2CA2gYuTZ4wgKSFp4yrqN0WuygXigrw -qedVbYqkb/xnvw8BuAVNRD4egOz3LngWJL5kh8CMGFO56GNVwQSbBPP2Ctwk6yJY -IRFPKXOisNCx9gxx7u3Ye3CCbEXKi2jmkoDZo/Fk6NeRbfV9bsvOagALQmIRe87/ -fJZrA/f58QVqQvG48BIsUYvoEkrRy/LW0C7spOAPKuwL7NkRZwX5HNeD+fZ0jXOE -5QIDAQABo1MwUTAdBgNVHQ4EFgQUP48HmnzVtlynpgTUHdZ53emOdeYwHwYDVR0j -BBgwFoAUP48HmnzVtlynpgTUHdZ53emOdeYwDwYDVR0TAQH/BAUwAwEB/zANBgkq -hkiG9w0BAQsFAAOCAgEAK4TpdDA/dKb1H934xvmGf7H3BcJ/8sx/fnJ2Z5hyfMQJ -UiAWX36mqTjKpLew1rJQ0qq6GI0Agszex7PnTmLuU7r4lsMfwz5SwiKY/ZWcrPw7 -o2KNEIBEQPSasNBhGSMatAtRqDKAOu1aLUj1SJNeEZOqwNvlXEGR2pRmGGZPtg3E -Il7maCyLsSgilhaoZGust6Lv9fNV4c9zrtOSbo1lzLw+3G22YqvajqBLqrDdyXff -SwT0P5R8EpBzvMy4JuXAz0eYvAp5RFlKY5l4ArV6DkirjZIG7zeEF39IUGxvwlN5 -pgU6Jf/oeimP2QLCdxXDcxetVltZCZLHMO5wmHeo0LBL4ggQoA3691YWV3IY32NX -k5OWo9ALYfN1Q4OnXm/7cgUozVC3BDmwjSXzOG32dALJpSaitP7Sj9YBdnXwv5e8 -afr2zVMzQBTO9q5+X9M/6w1ZT/YQEHcuI+nVoHaKjTUhJKEUQT7gQHtSP2V+mesl -JsDFlQNQU6BSt9rHvsNfMeSEK2QnxNkPkt+V8GjOElT2H7CxB7ZNNua6E0R76zFI -b1JFEo1SVB/KJTvpF4+cR9bdgaxzjYuJByVMbRQ26AvwnrdqZKN8OSW23W/jMuqZ -h+PvbPtsiWRwYYsjNqSMxO7n5G6atgG5iiJLmNqTHDaevwEsTR2xNlV9+NJE1g0= ------END CERTIFICATE----- diff --git a/tls/example/test.key b/tls/example/test.key deleted file mode 100644 index ebe96219dd318..0000000000000 --- a/tls/example/test.key +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDCeeCeSMgNAF8N -UTYE5HOgP24/F0kr19BXJBCBL1z2D3s4ly7AqcyfP17c7DsgpgWX/bQIw51BfY5f -gX7mVTymWYqkZrb0JFEMyYn3tirkxm2LOohRRP2zMZrMAQhC1uQ5o638mL3tM7CH -B9XSDxiT6b4rzZB0TGKf1xJwTRrcn4JAvwkf4P93FnTj2U9BTL5EENebAyfDLpnq -OT6LKC/UCQyA8JVZy9aXmMqBNOLXMn7PjddrrjRvZPO9OzsPuBW7PU+y0Qn/xlTS -1dKHxzZ1BWziFJqHbEUaeGUT9AZldh+MrUbsTV491I731Kq6FxM5IU8GNmJpu9gg -Geo1TOnxA+9FIZ28uz0pMmiQDpNJ10jTymJjrQpBLOR/a7J6akHqOsjo88qp5nhK -EORH6npqa1NVpQP4CaMSCAJmLXjY7FeUuQMK7jH6WNQa9fw/bdQDU+AyMxEFn2CA -2gYuTZ4wgKSFp4yrqN0WuygXigrwqedVbYqkb/xnvw8BuAVNRD4egOz3LngWJL5k -h8CMGFO56GNVwQSbBPP2Ctwk6yJYIRFPKXOisNCx9gxx7u3Ye3CCbEXKi2jmkoDZ -o/Fk6NeRbfV9bsvOagALQmIRe87/fJZrA/f58QVqQvG48BIsUYvoEkrRy/LW0C7s -pOAPKuwL7NkRZwX5HNeD+fZ0jXOE5QIDAQABAoICABfsYkNWdLWUvBypRtnnOIoY -A3njZW7tR4XQu+8IYokqcmANyQSOXsugQg0vSj7D7aA653DEj43E4kTBa1juV12n -WjTLJep2pTnLeRtt32zxM/ySoL5OnF701TQPVHLUeJLWncqbB5Isb6TMw/LpRH69 -nFVLHmAQHXnIPU5vTcSiIqXQWlD/nq2Jp6t3yhYgWK+K2tsI0lViYDC+BgGcWfXd -fxpFfjnjp2xSsWafmOAcKBlkO4HVlkG1y34V0TgnzxSPgERhDrfWMA0XnKRzTzGq -DzDok/Hg0cdl1BlWRYspssvdbQIYqmU0xrmaAb6OpCC9Bjmfaw/Suc2JIpSX95cl -WQCvAz8DYx9BkWWdSXivczIg2PRgsex2VqEhQcweIPE5Ci+BEojs7ThvtrLJb9Pa -mnvcXS9KhSs2dmYno53CJQDzDkgsI9crbd/giReFuFV+VMFGHIUu5i0qB08IE7no -HxJBDZEqPhbJc4g5kFt67BcDJ/UR8CBUPoXCBID5vEopqaUqmDDs79gH9TJi1rY5 -yuL7Niww/+XXCHzdxHkfPfFI7T3DdHTKWfrwJMNYsQJ/m8Ytm/SPEPJ1Y950kJGv -voyzlal6YDTpLnfP8aCFGH7D3ZEUn4z/68AJRZ3O1G5AZsppAnTHvsOcbYJtkF8u -6tGV/UtJBiGIEoK8lqsBAoIBAQD2OM4sR6/hyErNDxdCqx8eH+y6Ckb84IF1VI5e -nTNw9sa792aKeW90skatZcCWIYV9PLkklPbY61TCFfeeKBxqaUA8QDSgPJCrIO1b -qZJ3BjU6duKPR/yGNyf+V7DHMYzcDqPg2RM7OAcU8rBj8BynNb+lB+nFobDKMCQo -NSu1jUiaiylHzyzN3RwwaUQx7eMcT7CjRCTpTDrCkj+gvWL51eiOVahYooa55c0i -0aC6XK4pTnOIu+RRSr9T0QeXNg4JSm1P3ek5nISWKBL8PBod4TFMBRaLRMW0Qm7Z -BZPWTuj5OzgjPAXZ7YmP4ew+DsBwVNxCIUYT6eQePpQV0X3FAoIBAQDKMwCkjnG+ -RVgulcRJHB3+EcyUZ/M6iXtf0sTKLhCtTfQefu6WuWGVFEZsWWyWoUCO4R99y+1y -4NZe1LlOq/Pk2hmM2lwJ4LsuprBqeIZDFNTv3o8evjK23yr/vTDDfq0MAjbfaWlP -wZthi+ktlIocPMvaMtRDLf8nNImWWJPtZ3y8iVflWLdi7AxVJIb2Tawamb+5DiZ+ -2gDLOzqCjmZlkBp+92ggmqHnycEeK/viN5wZ3jKTmK5UFr65WTV8VtHxRFtPT4lb -AzUVq1LzXcnpYHJraUzgv0HyTf8ujuEew2h+k6uznCZPXxDNMXyw83+2XzrlzVn8 -oFFbLVcxKHyhAoIBAALwHpWsrU6WYz4bJf3nqNFlNB/sK3is/R53SdObMJfyeeCY -TSLt3ASac/lxs2CXOc6KwPLMzD0+YZ6HQJMki2JBq0CMy5P15QwFso9bcKH28v+l -l4J6K2s2kUWZqkRWNQYTLqVqMa6NNphrZOtsXAsquwSvGX0ANunStTu9lq3t56Xg -bzeqIzC5qLJzLxshaYpW8Raho4cdH75enF/AosO0Z7f7Ea0tE0p4kYAO3Eoc/P8F -Lp+9DjZiG/JWFi8ZuAUoXDGNxlCnq3VXORLFbNJac4oirWgwA+PovwXb6S2vd1wx -9rOE7dTx3qhW28jJyEYO3Sn01685JT2vJXk1LLkCggEBALBPjM3wj2n0GZJ1cXS2 -mciDSLpXchWm7/d8V7BcPpp/kNBOqkivWVGfHlE4IKGv9hUgZE9S1LgJKaQDJZpe -AguZyhGyDfNnDJD/NHGCqsAvmAWZMX8DJkGjk2DwXWb6PHZuL97bkcLD9XM3hpoC -HfPFZ5PaIpB5uTU8cISytDqctH3J3OTVWQNfBNnxQeXfkKSZUD5TAmo2Od78xGY3 -OT5yS0hO/3L+oxL0L/rZK4f0KZ+8yqk8xzX3p1MDYMoNAn+4tU4Q8ppzz4QXofsn -2vou6VaTuT94Rk68iDjWyaIDjkhhB6VUEWvAX2wtLKc3jDNAEqFzUHYibQ5uGMt6 -nEECggEBALvhYYLwv53Z5+oErR1ZI8beHPzyFtnp5+Chq+m96wbI3NM+yIKchPeG -Icun1baX9DOvaFWQ/BvPEGIIb5cYoTJ9h3Is1iCeodLEDuDF6bdD9Id3eHXFsqfS -/kBLyVjBls8if41FThsG7Od/ETWJuuKIxeIEbHHyOh97Mkh6MIvr0hIFhzxuXJ1b -Oyn1rHjUgItzE7yFeMTN5IH6K7b3wxzzokFERDf+YHSjuFli6OayphG1GrYNmNCC -8DutWtZSdBD5l+oHkPJ1fGas9HI6pmB+zuqBMLfmWBl7ZKVP2Ut7KA1Nqv4g6JJk -X1cjhbrNvlI9+Y8PxePe3m4583b/B1Q= ------END PRIVATE KEY----- From f2a8a41ea36d7c91cf3d3b18c73c5193e6882c09 Mon Sep 17 00:00:00 2001 From: dantengsky Date: Fri, 6 Aug 2021 16:32:40 +0800 Subject: [PATCH 7/7] tweak test case setting --- fusequery/query/Cargo.toml | 2 +- fusequery/query/src/api/http_service_test.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/fusequery/query/Cargo.toml b/fusequery/query/Cargo.toml index dd54f69dc2260..fbe85c7cf556a 100644 --- a/fusequery/query/Cargo.toml +++ b/fusequery/query/Cargo.toml @@ -90,7 +90,7 @@ lru = "0.6.6" pretty_assertions = "0.7" criterion = "0.3" mysql = "21.0.1" -reqwest = { version = "0.11", features = ["json", "native-tls"] } +reqwest = { version = "0.11", features = ["json"] } [build-dependencies] common-building = {path = "../../common/building"} diff --git a/fusequery/query/src/api/http_service_test.rs b/fusequery/query/src/api/http_service_test.rs index e9b81b81b3ae8..9bfacc21a0546 100644 --- a/fusequery/query/src/api/http_service_test.rs +++ b/fusequery/query/src/api/http_service_test.rs @@ -44,7 +44,6 @@ async fn test_http_service_tls_server() -> Result<()> { // kick off let client = reqwest::Client::builder() .add_root_certificate(cert) - .danger_accept_invalid_hostnames(true) .build() .unwrap(); let resp = client.get(url).send().await;