From 4e5cb2e8074aa1307777839bea31a154711a9a4c Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 22 Feb 2023 11:17:15 +0000 Subject: [PATCH 1/6] Add hide_options_in_show_create_table to remove noise fields during copy and paste --- Cargo.lock | 1 + .../interpreter_table_show_create.rs | 14 +++++++++++++- src/query/settings/Cargo.toml | 1 + src/query/settings/src/lib.rs | 17 +++++++++++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 7ff66fcb080f4..48ac640ed79a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1958,6 +1958,7 @@ dependencies = [ "futures", "itertools", "num_cpus", + "storages-common-table-meta", "sys-info", "tracing", ] diff --git a/src/query/service/src/interpreters/interpreter_table_show_create.rs b/src/query/service/src/interpreters/interpreter_table_show_create.rs index abc87b96d82eb..51148e5169c7c 100644 --- a/src/query/service/src/interpreters/interpreter_table_show_create.rs +++ b/src/query/service/src/interpreters/interpreter_table_show_create.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashSet; use std::sync::Arc; use common_exception::Result; @@ -115,11 +116,22 @@ impl Interpreter for ShowCreateTableInterpreter { table_create_sql.push_str(format!(" CLUSTER BY {}", cluster_keys_str).as_str()); } + let settings = self.ctx.get_settings(); + let hide_options_in_show_create_table = settings + .get_hide_options_in_show_create_table() + .and_then(|raw_list| + Ok(raw_list.split(',').map(|x| x.to_string().to_uppercase()).collect::>())) + .unwrap_or(HashSet::new()); + + debug!("Show hide_options_in_show_create_table: {:?}", hide_options_in_show_create_table); + table_create_sql.push_str({ let mut opts = table_info.options().iter().collect::>(); opts.sort_by_key(|(k, _)| *k); opts.iter() - .filter(|(k, _)| !is_internal_opt_key(k)) + .filter(|(k, _)| { + !is_internal_opt_key(k) && !hide_options_in_show_create_table.contains(k.to_uppercase().as_str()) + }) .map(|(k, v)| format!(" {}='{}'", k.to_uppercase(), v)) .collect::>() .join("") diff --git a/src/query/settings/Cargo.toml b/src/query/settings/Cargo.toml index 77f140d90051e..6222c512a4446 100644 --- a/src/query/settings/Cargo.toml +++ b/src/query/settings/Cargo.toml @@ -19,6 +19,7 @@ common-exception = { path = "../../common/exception" } common-meta-app = { path = "../../meta/app" } common-meta-types = { path = "../../meta/types" } common-users = { path = "../users" } +storages-common-table-meta = { path = "../storages/common/table-meta" } dashmap = "5.4" futures = "0.3.24" diff --git a/src/query/settings/src/lib.rs b/src/query/settings/src/lib.rs index 30185c7bab7e3..05665499de43e 100644 --- a/src/query/settings/src/lib.rs +++ b/src/query/settings/src/lib.rs @@ -35,6 +35,7 @@ use common_meta_types::MatchSeq; use common_users::UserApiProvider; use dashmap::DashMap; use itertools::Itertools; +use storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; #[derive(Clone)] pub enum ScopeLevel { @@ -437,6 +438,16 @@ impl Settings { desc: "How many hours will the COPY file metadata expired in the metasrv, default value: 24*7=7days", possible_values: None, }, + SettingValue { + default_value: UserSettingValue::String(OPT_KEY_SNAPSHOT_LOCATION.to_string()), + user_setting: UserSetting::create( + "hide_options_in_show_create_table", + UserSettingValue::String(OPT_KEY_SNAPSHOT_LOCATION.to_string()), + ), + level: ScopeLevel::Session, + desc: "The list of options that needs to be ignored when executing show create table.", + possible_values: None, + }, SettingValue { default_value: UserSettingValue::String("".to_string()), user_setting: UserSetting::create( @@ -829,6 +840,12 @@ impl Settings { .and_then(|v| v.user_setting.value.as_string()) } + pub fn get_hide_options_in_show_create_table(&self) -> Result { + let key = "hide_options_in_show_create_table"; + self.check_and_get_setting_value(key) + .and_then(|v| v.user_setting.value.as_string()) + } + pub fn get_enable_query_result_cache(&self) -> Result { let key = "enable_query_result_cache"; self.try_get_u64(key).map(|v| v != 0) From 8c1fb13bf2e6ad85d86800fb0f6caac7ab4e5acc Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 22 Feb 2023 11:24:57 +0000 Subject: [PATCH 2/6] Fix lint issue --- .../interpreter_table_show_create.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/query/service/src/interpreters/interpreter_table_show_create.rs b/src/query/service/src/interpreters/interpreter_table_show_create.rs index 51148e5169c7c..27943864012a7 100644 --- a/src/query/service/src/interpreters/interpreter_table_show_create.rs +++ b/src/query/service/src/interpreters/interpreter_table_show_create.rs @@ -119,18 +119,26 @@ impl Interpreter for ShowCreateTableInterpreter { let settings = self.ctx.get_settings(); let hide_options_in_show_create_table = settings .get_hide_options_in_show_create_table() - .and_then(|raw_list| - Ok(raw_list.split(',').map(|x| x.to_string().to_uppercase()).collect::>())) + .and_then(|raw_list| { + Ok(raw_list + .split(',') + .map(|x| x.to_string().to_uppercase()) + .collect::>()) + }) .unwrap_or(HashSet::new()); - debug!("Show hide_options_in_show_create_table: {:?}", hide_options_in_show_create_table); + debug!( + "Show hide_options_in_show_create_table: {:?}", + hide_options_in_show_create_table + ); table_create_sql.push_str({ let mut opts = table_info.options().iter().collect::>(); opts.sort_by_key(|(k, _)| *k); opts.iter() .filter(|(k, _)| { - !is_internal_opt_key(k) && !hide_options_in_show_create_table.contains(k.to_uppercase().as_str()) + !is_internal_opt_key(k) + && !hide_options_in_show_create_table.contains(k.to_uppercase().as_str()) }) .map(|(k, v)| format!(" {}='{}'", k.to_uppercase(), v)) .collect::>() From 1048e26d99ec3c2f26dd0b96ac576ec27c8b97dd Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 23 Feb 2023 03:09:57 +0000 Subject: [PATCH 3/6] Update hide_options_in_show_create_table to boolean value --- Cargo.lock | 1 - .../interpreter_table_show_create.rs | 41 +++++++------------ .../it/storages/testdata/settings_table.txt | 1 + src/query/settings/Cargo.toml | 1 - src/query/settings/src/lib.rs | 13 +++--- .../base/06_show/06_0001_show_create_table | 8 ++++ 6 files changed, 29 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48ac640ed79a9..7ff66fcb080f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1958,7 +1958,6 @@ dependencies = [ "futures", "itertools", "num_cpus", - "storages-common-table-meta", "sys-info", "tracing", ] diff --git a/src/query/service/src/interpreters/interpreter_table_show_create.rs b/src/query/service/src/interpreters/interpreter_table_show_create.rs index 27943864012a7..3da9c285435ec 100644 --- a/src/query/service/src/interpreters/interpreter_table_show_create.rs +++ b/src/query/service/src/interpreters/interpreter_table_show_create.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashSet; use std::sync::Arc; use common_exception::Result; @@ -119,32 +118,20 @@ impl Interpreter for ShowCreateTableInterpreter { let settings = self.ctx.get_settings(); let hide_options_in_show_create_table = settings .get_hide_options_in_show_create_table() - .and_then(|raw_list| { - Ok(raw_list - .split(',') - .map(|x| x.to_string().to_uppercase()) - .collect::>()) - }) - .unwrap_or(HashSet::new()); - - debug!( - "Show hide_options_in_show_create_table: {:?}", - hide_options_in_show_create_table - ); - - table_create_sql.push_str({ - let mut opts = table_info.options().iter().collect::>(); - opts.sort_by_key(|(k, _)| *k); - opts.iter() - .filter(|(k, _)| { - !is_internal_opt_key(k) - && !hide_options_in_show_create_table.contains(k.to_uppercase().as_str()) - }) - .map(|(k, v)| format!(" {}='{}'", k.to_uppercase(), v)) - .collect::>() - .join("") - .as_str() - }); + .unwrap_or(false); + + if !hide_options_in_show_create_table { + table_create_sql.push_str({ + let mut opts = table_info.options().iter().collect::>(); + opts.sort_by_key(|(k, _)| *k); + opts.iter() + .filter(|(k, _)| !is_internal_opt_key(k)) + .map(|(k, v)| format!(" {}='{}'", k.to_uppercase(), v)) + .collect::>() + .join("") + .as_str() + }); + } let block = DataBlock::new( vec![ diff --git a/src/query/service/tests/it/storages/testdata/settings_table.txt b/src/query/service/tests/it/storages/testdata/settings_table.txt index 405a2c4cfb508..5ee99dfa0e5b2 100644 --- a/src/query/service/tests/it/storages/testdata/settings_table.txt +++ b/src/query/service/tests/it/storages/testdata/settings_table.txt @@ -14,6 +14,7 @@ DB.Table: 'system'.'settings', Table: settings-table_id:1, ver:0, Engine: System | "enable_query_result_cache" | "0" | "0" | "SESSION" | "Enable the cache result of each query. It's disabled by default." | "UInt64" | | "flight_client_timeout" | "60" | "60" | "SESSION" | "Max duration the flight client request is allowed to take in seconds. By default, it is 60 seconds." | "UInt64" | | "group_by_two_level_threshold" | "10000" | "10000" | "SESSION" | "The threshold of keys to open two-level aggregation, default value: 10000." | "UInt64" | +| "hide_options_in_show_create_table" | "1" | "1" | "SESSION" | "Ignore options while rendering the result of show create table." | "UInt64" | | "input_read_buffer_size" | "1048576" | "1048576" | "SESSION" | "The size of buffer in bytes for input with format. By default, it is 1MB." | "UInt64" | | "load_file_metadata_expire_hours" | "168" | "168" | "SESSION" | "How many hours will the COPY file metadata expired in the metasrv, default value: 24*7=7days" | "UInt64" | | "max_block_size" | "65536" | "65536" | "SESSION" | "Maximum block size for reading, default value: 65536." | "UInt64" | diff --git a/src/query/settings/Cargo.toml b/src/query/settings/Cargo.toml index 6222c512a4446..77f140d90051e 100644 --- a/src/query/settings/Cargo.toml +++ b/src/query/settings/Cargo.toml @@ -19,7 +19,6 @@ common-exception = { path = "../../common/exception" } common-meta-app = { path = "../../meta/app" } common-meta-types = { path = "../../meta/types" } common-users = { path = "../users" } -storages-common-table-meta = { path = "../storages/common/table-meta" } dashmap = "5.4" futures = "0.3.24" diff --git a/src/query/settings/src/lib.rs b/src/query/settings/src/lib.rs index 05665499de43e..214722ff9ba43 100644 --- a/src/query/settings/src/lib.rs +++ b/src/query/settings/src/lib.rs @@ -35,7 +35,6 @@ use common_meta_types::MatchSeq; use common_users::UserApiProvider; use dashmap::DashMap; use itertools::Itertools; -use storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; #[derive(Clone)] pub enum ScopeLevel { @@ -439,13 +438,13 @@ impl Settings { possible_values: None, }, SettingValue { - default_value: UserSettingValue::String(OPT_KEY_SNAPSHOT_LOCATION.to_string()), + default_value: UserSettingValue::UInt64(1), user_setting: UserSetting::create( "hide_options_in_show_create_table", - UserSettingValue::String(OPT_KEY_SNAPSHOT_LOCATION.to_string()), + UserSettingValue::UInt64(1), ), level: ScopeLevel::Session, - desc: "The list of options that needs to be ignored when executing show create table.", + desc: "Ignore options while rendering the result of show create table.", possible_values: None, }, SettingValue { @@ -840,10 +839,10 @@ impl Settings { .and_then(|v| v.user_setting.value.as_string()) } - pub fn get_hide_options_in_show_create_table(&self) -> Result { + pub fn get_hide_options_in_show_create_table(&self) -> Result { let key = "hide_options_in_show_create_table"; - self.check_and_get_setting_value(key) - .and_then(|v| v.user_setting.value.as_string()) + let v = self.try_get_u64(key)?; + Ok(v != 0) } pub fn get_enable_query_result_cache(&self) -> Result { diff --git a/tests/sqllogictests/suites/base/06_show/06_0001_show_create_table b/tests/sqllogictests/suites/base/06_show/06_0001_show_create_table index 0315f02019903..cae4df1f41700 100644 --- a/tests/sqllogictests/suites/base/06_show/06_0001_show_create_table +++ b/tests/sqllogictests/suites/base/06_show/06_0001_show_create_table @@ -15,6 +15,14 @@ a CREATE TABLE `a` ( `a` BIGINT, `b` INT DEFAULT 3, `c` VARCHAR DEFAULT 'x statement ok CREATE TABLE `test`.`b` ( a bigint, b int null default null, c varchar(255), d smallint unsigned null) Engine = Null COMMENT = 'test b' +query TT +SHOW CREATE TABLE `test`.`b` +---- +b CREATE TABLE `b` ( `a` BIGINT, `b` INT NULL DEFAULT NULL, `c` VARCHAR, `d` SMALLINT UNSIGNED NULL ) ENGINE=NULL + +statement ok +set hide_options_in_show_create_table=0 + query TT SHOW CREATE TABLE `test`.`b` ---- From 4dbbc10850ae5e195d7ff54533f4d2daa34250c1 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 23 Feb 2023 04:06:07 +0000 Subject: [PATCH 4/6] Fix clickhouse tests --- .../14_0001_clickhouse_http.result | 5 ++++- .../14_clickhouse_http_handler/14_0001_clickhouse_http.sh | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.result b/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.result index 13f781a5adf22..d808b4453d9ba 100644 --- a/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.result +++ b/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.result @@ -12,7 +12,10 @@ String t2 Table Create Table String String -t2 CREATE TABLE `t2` (\n `a` INT\n) ENGINE=FUSE COMPRESSION=\'zstd\' STORAGE_FORMAT=\'parquet\' +t2 CREATE TABLE `t2` (\n `a` INT\n) ENGINE=FUSE +Table Create Table +String String +t2 CREATE TABLE `t2` (\n `a` INT\n) ENGINE=FUSE Field Type Null Default Extra String String String String String a INT NO 0 diff --git a/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.sh b/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.sh index 6e1fb1008613c..bd583c87dfea4 100755 --- a/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.sh +++ b/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.sh @@ -21,6 +21,8 @@ curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/ curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'create table t2(a int)' curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d "show TABLES LIKE 't%' format TabSeparatedWithNamesAndTypes" curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'show create table t2 format TabSeparatedWithNamesAndTypes' +curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'set hide_options_in_show_create_table=0' +curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'show create table t2 format TabSeparatedWithNamesAndTypes' curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'desc t2 format TabSeparatedWithNamesAndTypes' curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'insert into table t2 values(1)' curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?" -d 'select * from db2.t2' From 87414247b5b38c2c5c18de01df2906079a3bbbc6 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 23 Feb 2023 13:29:30 +0000 Subject: [PATCH 5/6] Revert "Fix clickhouse tests" This reverts commit 4dbbc10850ae5e195d7ff54533f4d2daa34250c1. --- .../14_0001_clickhouse_http.result | 5 +---- .../14_clickhouse_http_handler/14_0001_clickhouse_http.sh | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.result b/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.result index d808b4453d9ba..13f781a5adf22 100644 --- a/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.result +++ b/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.result @@ -12,10 +12,7 @@ String t2 Table Create Table String String -t2 CREATE TABLE `t2` (\n `a` INT\n) ENGINE=FUSE -Table Create Table -String String -t2 CREATE TABLE `t2` (\n `a` INT\n) ENGINE=FUSE +t2 CREATE TABLE `t2` (\n `a` INT\n) ENGINE=FUSE COMPRESSION=\'zstd\' STORAGE_FORMAT=\'parquet\' Field Type Null Default Extra String String String String String a INT NO 0 diff --git a/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.sh b/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.sh index bd583c87dfea4..6e1fb1008613c 100755 --- a/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.sh +++ b/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.sh @@ -21,8 +21,6 @@ curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/ curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'create table t2(a int)' curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d "show TABLES LIKE 't%' format TabSeparatedWithNamesAndTypes" curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'show create table t2 format TabSeparatedWithNamesAndTypes' -curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'set hide_options_in_show_create_table=0' -curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'show create table t2 format TabSeparatedWithNamesAndTypes' curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'desc t2 format TabSeparatedWithNamesAndTypes' curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?database=db2" -d 'insert into table t2 values(1)' curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?" -d 'select * from db2.t2' From ab190d953b72b9212eb608bbca9eea7a64d50393 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 23 Feb 2023 14:02:49 +0000 Subject: [PATCH 6/6] revert ut and fix --- .../14_clickhouse_http_handler/14_0001_clickhouse_http.result | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.result b/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.result index 13f781a5adf22..e955cb19758ce 100644 --- a/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.result +++ b/tests/suites/0_stateless/14_clickhouse_http_handler/14_0001_clickhouse_http.result @@ -12,7 +12,7 @@ String t2 Table Create Table String String -t2 CREATE TABLE `t2` (\n `a` INT\n) ENGINE=FUSE COMPRESSION=\'zstd\' STORAGE_FORMAT=\'parquet\' +t2 CREATE TABLE `t2` (\n `a` INT\n) ENGINE=FUSE Field Type Null Default Extra String String String String String a INT NO 0