Merge pull request #2256 from zhyass/feature_index

ISSUE-2344: Add RangeFilter for index pruning
databendlabs · Oct 20, 2021 · 67ab365 · 67ab365
2 parents 618efc9 + 7eab895
commit 67ab365
Show file tree

Hide file tree

Showing 30 changed files with 751 additions and 48 deletions.
diff --git a/common/datavalues/src/data_value.rs b/common/datavalues/src/data_value.rs
@@ -248,6 +248,27 @@ impl DataValue {
             ))),
         }
     }
+
+    pub fn as_bool(&self) -> Result<bool> {
+        match self {
+            DataValue::Null => Ok(false),
+            DataValue::Boolean(v) => Ok(v.map_or(false, |v| v)),
+            DataValue::Int8(v) => Ok(v.map_or(false, |v| v != 0)),
+            DataValue::Int16(v) => Ok(v.map_or(false, |v| v != 0)),
+            DataValue::Int32(v) => Ok(v.map_or(false, |v| v != 0)),
+            DataValue::Int64(v) => Ok(v.map_or(false, |v| v != 0)),
+            DataValue::UInt8(v) => Ok(v.map_or(false, |v| v != 0)),
+            DataValue::UInt16(v) => Ok(v.map_or(false, |v| v != 0)),
+            DataValue::UInt32(v) => Ok(v.map_or(false, |v| v != 0)),
+            DataValue::UInt64(v) => Ok(v.map_or(false, |v| v != 0)),
+            DataValue::Float32(v) => Ok(v.map_or(false, |v| v != 0f32)),
+            DataValue::Float64(v) => Ok(v.map_or(false, |v| v != 0f64)),
+            other => Result::Err(ErrorCode::BadDataValueType(format!(
+                "Unexpected type:{:?} to get boolean",
+                other.data_type()
+            ))),
+        }
+    }
 }
 
 // Did not use std::convert:TryFrom

diff --git a/common/functions/src/scalars/comparisons/comparison_eq.rs b/common/functions/src/scalars/comparisons/comparison_eq.rs
@@ -31,7 +31,8 @@ impl ComparisonEqFunction {
         FunctionDescription::creator(Box::new(Self::try_create_func)).features(
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function("<>"),
+                .negative_function("<>")
+                .bool_function(),
         )
     }
 }
diff --git a/common/functions/src/scalars/comparisons/comparison_gt.rs b/common/functions/src/scalars/comparisons/comparison_gt.rs
@@ -31,7 +31,8 @@ impl ComparisonGtFunction {
         FunctionDescription::creator(Box::new(Self::try_create_func)).features(
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function("<="),
+                .negative_function("<=")
+                .bool_function(),
         )
     }
 }
diff --git a/common/functions/src/scalars/comparisons/comparison_gt_eq.rs b/common/functions/src/scalars/comparisons/comparison_gt_eq.rs
@@ -31,7 +31,8 @@ impl ComparisonGtEqFunction {
         FunctionDescription::creator(Box::new(Self::try_create_func)).features(
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function("<"),
+                .negative_function("<")
+                .bool_function(),
         )
     }
 }
diff --git a/common/functions/src/scalars/comparisons/comparison_like.rs b/common/functions/src/scalars/comparisons/comparison_like.rs
@@ -31,7 +31,8 @@ impl ComparisonLikeFunction {
         FunctionDescription::creator(Box::new(Self::try_create_func)).features(
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function("not like"),
+                .negative_function("not like")
+                .bool_function(),
         )
     }
 }
diff --git a/common/functions/src/scalars/comparisons/comparison_lt.rs b/common/functions/src/scalars/comparisons/comparison_lt.rs
@@ -31,7 +31,8 @@ impl ComparisonLtFunction {
         FunctionDescription::creator(Box::new(Self::try_create_func)).features(
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function(">="),
+                .negative_function(">=")
+                .bool_function(),
         )
     }
 }
diff --git a/common/functions/src/scalars/comparisons/comparison_lt_eq.rs b/common/functions/src/scalars/comparisons/comparison_lt_eq.rs
@@ -31,7 +31,8 @@ impl ComparisonLtEqFunction {
         FunctionDescription::creator(Box::new(Self::try_create_func)).features(
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function(">"),
+                .negative_function(">")
+                .bool_function(),
         )
     }
 }
diff --git a/common/functions/src/scalars/comparisons/comparison_not_eq.rs b/common/functions/src/scalars/comparisons/comparison_not_eq.rs
@@ -31,7 +31,8 @@ impl ComparisonNotEqFunction {
         FunctionDescription::creator(Box::new(Self::try_create_func)).features(
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function("="),
+                .negative_function("=")
+                .bool_function(),
         )
     }
 }
diff --git a/common/functions/src/scalars/comparisons/comparison_not_like.rs b/common/functions/src/scalars/comparisons/comparison_not_like.rs
@@ -31,7 +31,8 @@ impl ComparisonNotLikeFunction {
         FunctionDescription::creator(Box::new(Self::try_create_func)).features(
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function("like"),
+                .negative_function("like")
+                .bool_function(),
         )
     }
 }
diff --git a/common/functions/src/scalars/expressions/expression.rs b/common/functions/src/scalars/expressions/expression.rs
@@ -24,12 +24,16 @@ pub struct ToCastFunction;
 
 impl ToCastFunction {
     fn cast_function_creator(to_type: DataType) -> FunctionDescription {
+        let mut features = FunctionFeatures::default().deterministic();
+        if to_type == DataType::Boolean {
+            features = features.bool_function();
+        }
+
         let function_creator: FactoryCreator = Box::new(move |display_name| {
             CastFunction::create(display_name.to_string(), to_type.clone())
         });
 
-        FunctionDescription::creator(function_creator)
-            .features(FunctionFeatures::default().deterministic())
+        FunctionDescription::creator(function_creator).features(features)
     }
 
     pub fn register(factory: &mut FunctionFactory) {

diff --git a/common/functions/src/scalars/function_factory.rs b/common/functions/src/scalars/function_factory.rs
@@ -38,13 +38,15 @@ pub type FactoryCreator = Box<dyn Fn(&str) -> Result<Box<dyn Function>> + Send +
 pub struct FunctionFeatures {
     pub is_deterministic: bool,
     pub negative_function_name: Option<String>,
+    pub is_bool_func: bool,
 }
 
 impl FunctionFeatures {
     pub fn default() -> FunctionFeatures {
         FunctionFeatures {
             is_deterministic: false,
             negative_function_name: None,
+            is_bool_func: false,
         }
     }
 
@@ -57,6 +59,11 @@ impl FunctionFeatures {
         self.negative_function_name = Some(negative_name.to_string());
         self
     }
+
+    pub fn bool_function(mut self) -> FunctionFeatures {
+        self.is_bool_func = true;
+        self
+    }
 }
 
 pub struct FunctionDescription {

diff --git a/common/functions/src/scalars/logics/logic_and.rs b/common/functions/src/scalars/logics/logic_and.rs
@@ -31,7 +31,8 @@ impl LogicAndFunction {
         FunctionDescription::creator(Box::new(Self::try_create_func)).features(
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function("or"),
+                .negative_function("or")
+                .bool_function(),
         )
     }
 }
diff --git a/common/functions/src/scalars/logics/logic_not.rs b/common/functions/src/scalars/logics/logic_not.rs
@@ -32,7 +32,8 @@ impl LogicNotFunction {
             // We should need remove expression if negative function is empty
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function(""),
+                .negative_function("")
+                .bool_function(),
         )
     }
 }
diff --git a/common/functions/src/scalars/logics/logic_or.rs b/common/functions/src/scalars/logics/logic_or.rs
@@ -31,7 +31,8 @@ impl LogicOrFunction {
         FunctionDescription::creator(Box::new(Self::try_create_func)).features(
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function("and"),
+                .negative_function("and")
+                .bool_function(),
         )
     }
 }
diff --git a/common/functions/src/scalars/nullables/is_not_null.rs b/common/functions/src/scalars/nullables/is_not_null.rs
@@ -40,7 +40,8 @@ impl IsNotNullFunction {
         FunctionDescription::creator(Box::new(Self::try_create_func)).features(
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function("isnull"),
+                .negative_function("isnull")
+                .bool_function(),
         )
     }
 }

diff --git a/common/functions/src/scalars/nullables/is_null.rs b/common/functions/src/scalars/nullables/is_null.rs
@@ -40,7 +40,8 @@ impl IsNullFunction {
         FunctionDescription::creator(Box::new(Self::try_create_func)).features(
             FunctionFeatures::default()
                 .deterministic()
-                .negative_function("isnotnull"),
+                .negative_function("isnotnull")
+                .bool_function(),
         )
     }
 }

diff --git a/common/functions/src/scalars/udfs/exists.rs b/common/functions/src/scalars/udfs/exists.rs
@@ -36,7 +36,7 @@ impl ExistsFunction {
 
     pub fn desc() -> FunctionDescription {
         FunctionDescription::creator(Box::new(Self::try_create))
-            .features(FunctionFeatures::default())
+            .features(FunctionFeatures::default().bool_function())
     }
 }
 

diff --git a/common/functions/src/scalars/udfs/udf_example.rs b/common/functions/src/scalars/udfs/udf_example.rs
@@ -39,7 +39,7 @@ impl UdfExampleFunction {
 
     pub fn desc() -> FunctionDescription {
         FunctionDescription::creator(Box::new(Self::try_create))
-            .features(FunctionFeatures::default().deterministic())
+            .features(FunctionFeatures::default().deterministic().bool_function())
     }
 }
 

diff --git a/common/planners/src/lib.rs b/common/planners/src/lib.rs
@@ -125,6 +125,7 @@ pub use plan_expression_common::unwrap_alias_exprs;
 pub use plan_expression_function::add;
 pub use plan_expression_function::avg;
 pub use plan_expression_function::modular;
+pub use plan_expression_function::neg;
 pub use plan_expression_function::not;
 pub use plan_expression_function::sum;
 pub use plan_expression_literal::lit;

diff --git a/common/planners/src/plan_expression_function.rs b/common/planners/src/plan_expression_function.rs
@@ -36,6 +36,14 @@ pub fn not(other: Expression) -> Expression {
     }
 }
 
+// Neg.
+pub fn neg(other: Expression) -> Expression {
+    Expression::UnaryExpression {
+        op: "-".to_string(),
+        expr: Box::new(other),
+    }
+}
+
 /// Mod binary function.
 pub fn modular(left: Expression, right: Expression) -> Expression {
     binary_expr(left, "%", right)

diff --git a/common/planners/src/plan_rewriter.rs b/common/planners/src/plan_rewriter.rs
@@ -674,6 +674,10 @@ impl RewriteHelper {
                 op: op.clone(),
                 right: Box::new(expressions[1].clone()),
             },
+            Expression::UnaryExpression { op, .. } => Expression::UnaryExpression {
+                op: op.clone(),
+                expr: Box::new(expressions[0].clone()),
+            },
             Expression::ScalarFunction { op, .. } => Expression::ScalarFunction {
                 op: op.clone(),
                 args: expressions.to_vec(),
@@ -769,4 +773,22 @@ impl RewriteHelper {
 
         Ok(true)
     }
+
+    pub fn rewrite_column_expr(
+        expr: &Expression,
+        column_old: &str,
+        column_new: &str,
+    ) -> Result<Expression> {
+        let expressions = Self::expression_plan_children(expr)?;
+        let expressions = expressions
+            .iter()
+            .map(|e| Self::rewrite_column_expr(e, column_old, column_new))
+            .collect::<Result<Vec<_>>>()?;
+        if let Expression::Column(name) = expr {
+            if name.eq(column_old) {
+                return Ok(Expression::Column(column_new.to_string()));
+            }
+        }
+        Ok(Self::rebuild_from_exprs(expr, &expressions))
+    }
 }
diff --git a/query/src/datasources/index/mod.rs b/query/src/datasources/index/mod.rs
@@ -17,13 +17,18 @@
 mod index_min_max_test;
 #[cfg(test)]
 mod index_sparse_test;
+#[cfg(test)]
+mod range_filter_test;
 
 mod index_min_max;
 mod index_sparse;
+#[allow(dead_code)]
+pub mod range_filter;
 
 pub use index_min_max::MinMaxIndex;
 pub use index_sparse::SparseIndex;
 pub use index_sparse::SparseIndexValue;
+pub use range_filter::RangeFilter;
 
 #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
 pub enum IndexSchemaVersion {