From 3a818fd93ebfd6f42e1c13591cfbaef99b1926db Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 5 Sep 2024 23:39:11 +0800 Subject: [PATCH 1/3] feat: add limit push down rule --- Cargo.lock | 1 + .../sql/src/planner/optimizer/rule/factory.rs | 2 + .../src/planner/optimizer/rule/rewrite/mod.rs | 2 + .../rule/rewrite/rule_push_down_limit.rs | 66 +++++++++++++++++++ .../sql/src/planner/optimizer/rule/rule.rs | 2 + 5 files changed, 73 insertions(+) create mode 100644 src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_limit.rs diff --git a/Cargo.lock b/Cargo.lock index 0547f2ed6388f..879b4d1031793 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3634,6 +3634,7 @@ dependencies = [ "databend-common-meta-stoerr", "databend-common-meta-types", "databend-common-proto-conv", + "enumflags2", "fastrace", "futures", "log", diff --git a/src/query/sql/src/planner/optimizer/rule/factory.rs b/src/query/sql/src/planner/optimizer/rule/factory.rs index c1f9dca755096..5805d39b19d40 100644 --- a/src/query/sql/src/planner/optimizer/rule/factory.rs +++ b/src/query/sql/src/planner/optimizer/rule/factory.rs @@ -45,6 +45,7 @@ use crate::optimizer::rule::rewrite::RuleSplitAggregate; use crate::optimizer::rule::transform::RuleCommuteJoinBaseTable; use crate::optimizer::rule::transform::RuleEagerAggregation; use crate::optimizer::rule::transform::RuleLeftExchangeJoin; +use crate::optimizer::rule::rewrite::RulePushDownLimit; use crate::optimizer::rule::RuleID; use crate::optimizer::rule::RulePtr; use crate::MetadataRef; @@ -63,6 +64,7 @@ impl RuleFactory { RuleID::PushDownFilterScan => Ok(Box::new(RulePushDownFilterScan::new(metadata))), RuleID::PushDownFilterSort => Ok(Box::new(RulePushDownFilterSort::new())), RuleID::PushDownFilterProjectSet => Ok(Box::new(RulePushDownFilterProjectSet::new())), + RuleID::PushDownLimit => Ok(Box::new(RulePushDownLimit::new())), RuleID::PushDownLimitUnion => Ok(Box::new(RulePushDownLimitUnion::new())), RuleID::PushDownLimitScan => Ok(Box::new(RulePushDownLimitScan::new())), RuleID::PushDownSortScan => Ok(Box::new(RulePushDownSortScan::new())), diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs index 440b51231f1dc..61546b9519460 100644 --- a/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs @@ -42,6 +42,7 @@ mod rule_push_down_sort_scan; mod rule_semi_to_inner_join; mod rule_split_aggregate; mod rule_try_apply_agg_index; +mod rule_push_down_limit; pub use rule_commute_join::RuleCommuteJoin; pub use rule_eliminate_eval_scalar::RuleEliminateEvalScalar; @@ -72,3 +73,4 @@ pub use rule_push_down_sort_scan::RulePushDownSortScan; pub use rule_semi_to_inner_join::RuleSemiToInnerJoin; pub use rule_split_aggregate::RuleSplitAggregate; pub use rule_try_apply_agg_index::RuleTryApplyAggIndex; +pub use rule_push_down_limit::RulePushDownLimit; diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_limit.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_limit.rs new file mode 100644 index 0000000000000..1fbf7b1182484 --- /dev/null +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_limit.rs @@ -0,0 +1,66 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_exception::Result; + +use crate::optimizer::extract::Matcher; +use crate::optimizer::rule::Rule; +use crate::optimizer::rule::TransformResult; +use crate::optimizer::RuleID; +use crate::optimizer::SExpr; +use crate::plans::Limit; +use crate::plans::RelOp; +use crate::plans::RelOperator; +use crate::plans::DummyTableScan; + +pub struct RulePushDownLimit { + id: RuleID, + matchers: Vec, +} + +impl RulePushDownLimit { + pub fn new() -> Self { + Self { + id: RuleID::PushDownLimit, + matchers: vec![Matcher::MatchOp { + op_type: RelOp::Limit, + children: vec![Matcher::Leaf], + }], + } + } +} + +impl Rule for RulePushDownLimit { + fn id(&self) -> RuleID { + self.id + } + + fn apply(&self, s_expr: &SExpr, state: &mut TransformResult) -> Result<()> { + let limit: Limit = s_expr.plan().clone().try_into()?; + + if let Some(limit_val) = limit.limit && limit_val == 0{ + let dummy_scan = DummyTableScan; + let result = SExpr::create_leaf(Arc::new(RelOperator::DummyTableScan(dummy_scan))); + state.add_result(result); + } + + Ok(()) + } + + fn matchers(&self) -> &[Matcher] { + &self.matchers + } +} \ No newline at end of file diff --git a/src/query/sql/src/planner/optimizer/rule/rule.rs b/src/query/sql/src/planner/optimizer/rule/rule.rs index 32e3c6646a9f6..40fa2037544a6 100644 --- a/src/query/sql/src/planner/optimizer/rule/rule.rs +++ b/src/query/sql/src/planner/optimizer/rule/rule.rs @@ -34,6 +34,7 @@ pub static DEFAULT_REWRITE_RULES: LazyLock> = LazyLock::new(|| { RuleID::PushDownFilterUnion, RuleID::PushDownFilterAggregate, RuleID::PushDownFilterWindow, + RuleID::PushDownLimit, RuleID::PushDownLimitUnion, RuleID::PushDownLimitEvalScalar, RuleID::PushDownLimitSort, @@ -83,6 +84,7 @@ pub enum RuleID { PushDownFilterSort, PushDownFilterProjectSet, PushDownFilterWindow, + PushDownLimit, PushDownLimitUnion, PushDownLimitOuterJoin, PushDownLimitEvalScalar, From 734a2bab159c1ad9d91c5bbc49d866481cafdd0a Mon Sep 17 00:00:00 2001 From: xudong963 Date: Fri, 6 Sep 2024 01:15:26 +0800 Subject: [PATCH 2/3] fix --- .../sql/src/planner/optimizer/rule/factory.rs | 4 +- .../src/planner/optimizer/rule/rewrite/mod.rs | 4 +- .../rule/rewrite/rule_push_down_limit.rs | 40 +++++++++++++++---- .../sql/src/planner/optimizer/rule/rule.rs | 1 + .../suites/mode/standalone/explain/limit.test | 9 +++++ 5 files changed, 47 insertions(+), 11 deletions(-) diff --git a/src/query/sql/src/planner/optimizer/rule/factory.rs b/src/query/sql/src/planner/optimizer/rule/factory.rs index 5805d39b19d40..753cb6a431915 100644 --- a/src/query/sql/src/planner/optimizer/rule/factory.rs +++ b/src/query/sql/src/planner/optimizer/rule/factory.rs @@ -34,6 +34,7 @@ use crate::optimizer::rule::rewrite::RulePushDownFilterProjectSet; use crate::optimizer::rule::rewrite::RulePushDownFilterScan; use crate::optimizer::rule::rewrite::RulePushDownFilterSort; use crate::optimizer::rule::rewrite::RulePushDownFilterUnion; +use crate::optimizer::rule::rewrite::RulePushDownLimit; use crate::optimizer::rule::rewrite::RulePushDownLimitOuterJoin; use crate::optimizer::rule::rewrite::RulePushDownLimitScan; use crate::optimizer::rule::rewrite::RulePushDownLimitSort; @@ -45,7 +46,6 @@ use crate::optimizer::rule::rewrite::RuleSplitAggregate; use crate::optimizer::rule::transform::RuleCommuteJoinBaseTable; use crate::optimizer::rule::transform::RuleEagerAggregation; use crate::optimizer::rule::transform::RuleLeftExchangeJoin; -use crate::optimizer::rule::rewrite::RulePushDownLimit; use crate::optimizer::rule::RuleID; use crate::optimizer::rule::RulePtr; use crate::MetadataRef; @@ -64,7 +64,7 @@ impl RuleFactory { RuleID::PushDownFilterScan => Ok(Box::new(RulePushDownFilterScan::new(metadata))), RuleID::PushDownFilterSort => Ok(Box::new(RulePushDownFilterSort::new())), RuleID::PushDownFilterProjectSet => Ok(Box::new(RulePushDownFilterProjectSet::new())), - RuleID::PushDownLimit => Ok(Box::new(RulePushDownLimit::new())), + RuleID::PushDownLimit => Ok(Box::new(RulePushDownLimit::new(metadata))), RuleID::PushDownLimitUnion => Ok(Box::new(RulePushDownLimitUnion::new())), RuleID::PushDownLimitScan => Ok(Box::new(RulePushDownLimitScan::new())), RuleID::PushDownSortScan => Ok(Box::new(RulePushDownSortScan::new())), diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs index 61546b9519460..a24e3bc2e0182 100644 --- a/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/mod.rs @@ -30,6 +30,7 @@ mod rule_push_down_filter_scan; mod rule_push_down_filter_sort; mod rule_push_down_filter_union; mod rule_push_down_filter_window; +mod rule_push_down_limit; mod rule_push_down_limit_aggregate; mod rule_push_down_limit_expression; mod rule_push_down_limit_join; @@ -42,7 +43,6 @@ mod rule_push_down_sort_scan; mod rule_semi_to_inner_join; mod rule_split_aggregate; mod rule_try_apply_agg_index; -mod rule_push_down_limit; pub use rule_commute_join::RuleCommuteJoin; pub use rule_eliminate_eval_scalar::RuleEliminateEvalScalar; @@ -61,6 +61,7 @@ pub use rule_push_down_filter_scan::RulePushDownFilterScan; pub use rule_push_down_filter_sort::RulePushDownFilterSort; pub use rule_push_down_filter_union::RulePushDownFilterUnion; pub use rule_push_down_filter_window::RulePushDownFilterWindow; +pub use rule_push_down_limit::RulePushDownLimit; pub use rule_push_down_limit_aggregate::RulePushDownLimitAggregate; pub use rule_push_down_limit_expression::RulePushDownLimitEvalScalar; pub use rule_push_down_limit_join::RulePushDownLimitOuterJoin; @@ -73,4 +74,3 @@ pub use rule_push_down_sort_scan::RulePushDownSortScan; pub use rule_semi_to_inner_join::RuleSemiToInnerJoin; pub use rule_split_aggregate::RuleSplitAggregate; pub use rule_try_apply_agg_index::RuleTryApplyAggIndex; -pub use rule_push_down_limit::RulePushDownLimit; diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_limit.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_limit.rs index 1fbf7b1182484..7177a9ee7fa0e 100644 --- a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_limit.rs +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_push_down_limit.rs @@ -15,26 +15,34 @@ use std::sync::Arc; use databend_common_exception::Result; +use databend_common_expression::Column; +use databend_common_expression::DataField; +use databend_common_expression::DataSchemaRefExt; use crate::optimizer::extract::Matcher; use crate::optimizer::rule::Rule; use crate::optimizer::rule::TransformResult; +use crate::optimizer::RelExpr; use crate::optimizer::RuleID; use crate::optimizer::SExpr; +use crate::plans::ConstantTableScan; use crate::plans::Limit; +use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::RelOperator; -use crate::plans::DummyTableScan; +use crate::MetadataRef; pub struct RulePushDownLimit { id: RuleID, + metadata: MetadataRef, matchers: Vec, } impl RulePushDownLimit { - pub fn new() -> Self { + pub fn new(metadata: MetadataRef) -> Self { Self { id: RuleID::PushDownLimit, + metadata, matchers: vec![Matcher::MatchOp { op_type: RelOp::Limit, children: vec![Matcher::Leaf], @@ -50,10 +58,28 @@ impl Rule for RulePushDownLimit { fn apply(&self, s_expr: &SExpr, state: &mut TransformResult) -> Result<()> { let limit: Limit = s_expr.plan().clone().try_into()?; - - if let Some(limit_val) = limit.limit && limit_val == 0{ - let dummy_scan = DummyTableScan; - let result = SExpr::create_leaf(Arc::new(RelOperator::DummyTableScan(dummy_scan))); + if let Some(limit_val) = limit.limit + && limit_val == 0 + { + let output_columns = limit + .derive_relational_prop(&RelExpr::with_s_expr(s_expr))? + .output_columns + .clone(); + let metadata = self.metadata.read(); + let mut fields = Vec::with_capacity(output_columns.len()); + for col in output_columns.iter() { + fields.push(DataField::new( + &col.to_string(), + metadata.column(*col).data_type(), + )); + } + let empty_scan = ConstantTableScan { + values: vec![Column::Null { len: 0 }; output_columns.len()], + num_rows: 0, + schema: DataSchemaRefExt::create(fields), + columns: output_columns, + }; + let result = SExpr::create_leaf(Arc::new(RelOperator::ConstantTableScan(empty_scan))); state.add_result(result); } @@ -63,4 +89,4 @@ impl Rule for RulePushDownLimit { fn matchers(&self) -> &[Matcher] { &self.matchers } -} \ No newline at end of file +} diff --git a/src/query/sql/src/planner/optimizer/rule/rule.rs b/src/query/sql/src/planner/optimizer/rule/rule.rs index 40fa2037544a6..7a4043e86ba46 100644 --- a/src/query/sql/src/planner/optimizer/rule/rule.rs +++ b/src/query/sql/src/planner/optimizer/rule/rule.rs @@ -120,6 +120,7 @@ impl Display for RuleID { RuleID::PushDownFilterScan => write!(f, "PushDownFilterScan"), RuleID::PushDownFilterSort => write!(f, "PushDownFilterSort"), RuleID::PushDownFilterProjectSet => write!(f, "PushDownFilterProjectSet"), + RuleID::PushDownLimit => write!(f, "PushDownLimit"), RuleID::PushDownLimitUnion => write!(f, "PushDownLimitUnion"), RuleID::PushDownLimitOuterJoin => write!(f, "PushDownLimitOuterJoin"), RuleID::PushDownLimitEvalScalar => write!(f, "PushDownLimitEvalScalar"), diff --git a/tests/sqllogictests/suites/mode/standalone/explain/limit.test b/tests/sqllogictests/suites/mode/standalone/explain/limit.test index d4b71378cce69..f3508317dd9e9 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/limit.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/limit.test @@ -220,3 +220,12 @@ Limit ├── partitions scanned: 1 ├── push downs: [filters: [], limit: NONE] └── estimated rows: 2.00 + + +query T +explain select * from (select count(t1.number) as c1 from numbers(1) as t1 group by number) as t3 left join (select count(t.number) as c from numbers(2) as t group by number) as t4 on t3.c1=t4.c order by t3.c1 limit 0 +---- +ConstantTableScan +├── output columns: [count(t1.number) (#2), count(t.number) (#5)] +├── column 0: [] +└── column 1: [] From 21636db46a9ab1c5fb35c303701910c92472b5dc Mon Sep 17 00:00:00 2001 From: xudong963 Date: Fri, 6 Sep 2024 10:21:42 +0800 Subject: [PATCH 3/3] fix test --- .../suites/mode/standalone/explain/limit.test | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/sqllogictests/suites/mode/standalone/explain/limit.test b/tests/sqllogictests/suites/mode/standalone/explain/limit.test index f3508317dd9e9..1c1a01eb88f22 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/limit.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/limit.test @@ -223,9 +223,8 @@ Limit query T -explain select * from (select count(t1.number) as c1 from numbers(1) as t1 group by number) as t3 left join (select count(t.number) as c from numbers(2) as t group by number) as t4 on t3.c1=t4.c order by t3.c1 limit 0 +explain select c1 from (select count(t1.number) as c1 from numbers(1) as t1 group by number) as t3 left join (select count(t.number) as c from numbers(2) as t group by number) as t4 on t3.c1=t4.c order by t3.c1 limit 0 ---- ConstantTableScan -├── output columns: [count(t1.number) (#2), count(t.number) (#5)] -├── column 0: [] -└── column 1: [] +├── output columns: [count(t1.number) (#2)] +└── column 0: []