From 774ea0d1b8ef567e0c6f612031c37e0ce9b06802 Mon Sep 17 00:00:00 2001 From: TCeason Date: Wed, 29 Jan 2025 18:28:09 +0800 Subject: [PATCH 1/4] feat(query): support extract EPOCH from timestamp --- src/query/ast/src/ast/expr.rs | 2 + src/query/ast/src/parser/expr.rs | 127 +++++++++++------- .../src/scalars/timestamp/src/datetime.rs | 13 ++ .../it/scalars/testdata/function_list.txt | 2 + .../sql/src/planner/semantic/type_check.rs | 3 + src/tests/sqlsmith/src/sql_gen/expr.rs | 3 +- .../functions/02_0012_function_datetimes.test | 13 ++ 7 files changed, 114 insertions(+), 49 deletions(-) diff --git a/src/query/ast/src/ast/expr.rs b/src/query/ast/src/ast/expr.rs index 9f4c5fb3515f3..a87e8509d6731 100644 --- a/src/query/ast/src/ast/expr.rs +++ b/src/query/ast/src/ast/expr.rs @@ -828,6 +828,7 @@ pub enum IntervalKind { Doy, Week, Dow, + Epoch, } impl Display for IntervalKind { @@ -843,6 +844,7 @@ impl Display for IntervalKind { IntervalKind::Doy => "DOY", IntervalKind::Dow => "DOW", IntervalKind::Week => "WEEK", + IntervalKind::Epoch => "EPOCH", }) } } diff --git a/src/query/ast/src/parser/expr.rs b/src/query/ast/src/parser/expr.rs index 551dae1108388..ac2e7351715df 100644 --- a/src/query/ast/src/parser/expr.rs +++ b/src/query/ast/src/parser/expr.rs @@ -1821,56 +1821,87 @@ pub fn weekday(i: Input) -> IResult { } pub fn interval_kind(i: Input) -> IResult { + let year = value(IntervalKind::Year, rule! { YEAR }); + let quarter = value(IntervalKind::Quarter, rule! { QUARTER }); + let month = value(IntervalKind::Month, rule! { MONTH }); + let day = value(IntervalKind::Day, rule! { DAY }); + let hour = value(IntervalKind::Hour, rule! { HOUR }); + let minute = value(IntervalKind::Minute, rule! { MINUTE }); + let second = value(IntervalKind::Second, rule! { SECOND }); + let doy = value(IntervalKind::Doy, rule! { DOY }); + let dow = value(IntervalKind::Dow, rule! { DOW }); + let week = value(IntervalKind::Week, rule! { WEEK }); + let epoch = value(IntervalKind::Epoch, rule! { EPOCH }); + let year_str = value( + IntervalKind::Year, + rule! { #literal_string_eq_ignore_case("YEAR") }, + ); + let quarter_str = value( + IntervalKind::Quarter, + rule! { #literal_string_eq_ignore_case("QUARTER") }, + ); + let month_str = value( + IntervalKind::Month, + rule! { #literal_string_eq_ignore_case("MONTH") }, + ); + let day_str = value( + IntervalKind::Day, + rule! { #literal_string_eq_ignore_case("DAY") }, + ); + let hour_str = value( + IntervalKind::Hour, + rule! { #literal_string_eq_ignore_case("HOUR") }, + ); + let minute_str = value( + IntervalKind::Minute, + rule! { #literal_string_eq_ignore_case("MINUTE") }, + ); + let second_str = value( + IntervalKind::Second, + rule! { #literal_string_eq_ignore_case("SECOND") }, + ); + let doy_str = value( + IntervalKind::Doy, + rule! { #literal_string_eq_ignore_case("DOY") }, + ); + let dow_str = value( + IntervalKind::Dow, + rule! { #literal_string_eq_ignore_case("DOW") }, + ); + let week_str = value( + IntervalKind::Week, + rule! { #literal_string_eq_ignore_case("WEEK") }, + ); + let epoch_str = value( + IntervalKind::Epoch, + rule! { #literal_string_eq_ignore_case("EPOCH") }, + ); alt(( - value(IntervalKind::Year, rule! { YEAR }), - value(IntervalKind::Quarter, rule! { QUARTER }), - value(IntervalKind::Month, rule! { MONTH }), - value(IntervalKind::Day, rule! { DAY }), - value(IntervalKind::Hour, rule! { HOUR }), - value(IntervalKind::Minute, rule! { MINUTE }), - value(IntervalKind::Second, rule! { SECOND }), - value(IntervalKind::Doy, rule! { DOY }), - value(IntervalKind::Dow, rule! { DOW }), - value(IntervalKind::Week, rule! { WEEK }), - value( - IntervalKind::Year, - rule! { #literal_string_eq_ignore_case("YEAR") }, + rule!( + #year + | #quarter + | #month + | #day + | #hour + | #minute + | #second + | #doy + | #dow + | #week + | #epoch ), - value( - IntervalKind::Quarter, - rule! { #literal_string_eq_ignore_case("QUARTER") }, - ), - value( - IntervalKind::Month, - rule! { #literal_string_eq_ignore_case("MONTH") }, - ), - value( - IntervalKind::Day, - rule! { #literal_string_eq_ignore_case("DAY") }, - ), - value( - IntervalKind::Hour, - rule! { #literal_string_eq_ignore_case("HOUR") }, - ), - value( - IntervalKind::Minute, - rule! { #literal_string_eq_ignore_case("MINUTE") }, - ), - value( - IntervalKind::Second, - rule! { #literal_string_eq_ignore_case("SECOND") }, - ), - value( - IntervalKind::Doy, - rule! { #literal_string_eq_ignore_case("DOY") }, - ), - value( - IntervalKind::Dow, - rule! { #literal_string_eq_ignore_case("DOW") }, - ), - value( - IntervalKind::Week, - rule! { #literal_string_eq_ignore_case("WEEK") }, + rule!( + #year_str + | #quarter_str + | #month_str + | #day_str + | #hour_str + | #minute_str + | #second_str + | #doy_str + | #dow_str + | #week_str + | #epoch_str ), ))(i) } diff --git a/src/query/functions/src/scalars/timestamp/src/datetime.rs b/src/query/functions/src/scalars/timestamp/src/datetime.rs index 22618c5e82232..98336b28a46ac 100644 --- a/src/query/functions/src/scalars/timestamp/src/datetime.rs +++ b/src/query/functions/src/scalars/timestamp/src/datetime.rs @@ -1524,6 +1524,19 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { }), ); + registry.register_passthrough_nullable_1_arg::( + "epoch_microsecond", + |_, _| FunctionDomain::Full, + vectorize_1_arg::(|val, ctx| { + (val.to_timestamp(ctx.func_ctx.tz.clone()) + .with_time_zone(TimeZone::UTC) + .timestamp() + .as_microsecond() as f64 + / 1_000_000f64) + .into() + }), + ); + registry.register_passthrough_nullable_1_arg::( "to_hour", |_, _| FunctionDomain::Full, diff --git a/src/query/functions/tests/it/scalars/testdata/function_list.txt b/src/query/functions/tests/it/scalars/testdata/function_list.txt index 551098cd4b29e..28e1becb9633d 100644 --- a/src/query/functions/tests/it/scalars/testdata/function_list.txt +++ b/src/query/functions/tests/it/scalars/testdata/function_list.txt @@ -1673,6 +1673,8 @@ Functions overloads: 0 divnull(Float64 NULL, Float64 NULL) :: Float64 NULL 0 epoch(Int64) :: Interval 1 epoch(Int64 NULL) :: Interval NULL +0 epoch_microsecond(Timestamp) :: Float64 +1 epoch_microsecond(Timestamp NULL) :: Float64 NULL 0 eq(Variant, Variant) :: Boolean 1 eq(Variant NULL, Variant NULL) :: Boolean NULL 2 eq(String, String) :: Boolean diff --git a/src/query/sql/src/planner/semantic/type_check.rs b/src/query/sql/src/planner/semantic/type_check.rs index e128f3a89be9c..bc63c99807b59 100644 --- a/src/query/sql/src/planner/semantic/type_check.rs +++ b/src/query/sql/src/planner/semantic/type_check.rs @@ -2930,6 +2930,9 @@ impl<'a> TypeChecker<'a> { ASTIntervalKind::Doy => self.resolve_function(span, "to_day_of_year", vec![], &[arg]), ASTIntervalKind::Dow => self.resolve_function(span, "to_day_of_week", vec![], &[arg]), ASTIntervalKind::Week => self.resolve_function(span, "to_week_of_year", vec![], &[arg]), + ASTIntervalKind::Epoch => { + self.resolve_function(span, "epoch_microsecond", vec![], &[arg]) + } } } diff --git a/src/tests/sqlsmith/src/sql_gen/expr.rs b/src/tests/sqlsmith/src/sql_gen/expr.rs index b6cd20d275c9f..bb5d2b3e25d58 100644 --- a/src/tests/sqlsmith/src/sql_gen/expr.rs +++ b/src/tests/sqlsmith/src/sql_gen/expr.rs @@ -495,7 +495,7 @@ impl SqlGenerator<'_, R> { DataType::Timestamp }; let expr = self.gen_expr(&expr_ty); - let kind = match self.rng.gen_range(0..=9) { + let kind = match self.rng.gen_range(0..=10) { 0 => IntervalKind::Year, 1 => IntervalKind::Quarter, 2 => IntervalKind::Month, @@ -506,6 +506,7 @@ impl SqlGenerator<'_, R> { 7 => IntervalKind::Doy, 8 => IntervalKind::Dow, 9 => IntervalKind::Week, + 10 => IntervalKind::Epoch, _ => unreachable!(), }; Expr::Extract { diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test index 9964799bdff60..a6a3d6809dde3 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test @@ -1124,7 +1124,20 @@ select EXTRACT(YEAR FROM to_datetime('2022-03-04 22:32:09')) = 2022 ---- 1 +query B +select EXTRACT(EPOCH FROM to_datetime('2022-03-04 22:32:09')) = to_unix_timestamp('2022-03-04 22:32:09'); +---- +1 + +query B +select EXTRACT(EPOCH FROM now()) = epoch_microsecond(now()) +---- +1 +query B +SELECT (to_timestamp('2023-11-12 09:38:18.965575') - to_timestamp(0)) / 1000000.0 = epoch_microsecond('2023-11-12 09:38:18.965575') +---- +1 query B select EXTRACT(MONTH FROM to_datetime('2022-03-04 22:32:09')) = 3 From cf1cf38fff9560e3821ac3f75c711a00ead25259 Mon Sep 17 00:00:00 2001 From: TCeason Date: Fri, 31 Jan 2025 16:24:26 +0800 Subject: [PATCH 2/4] use val directly --- .../functions/src/scalars/timestamp/src/datetime.rs | 9 +-------- .../query/functions/02_0012_function_datetimes.test | 10 ++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/query/functions/src/scalars/timestamp/src/datetime.rs b/src/query/functions/src/scalars/timestamp/src/datetime.rs index 98336b28a46ac..3fee840462ea0 100644 --- a/src/query/functions/src/scalars/timestamp/src/datetime.rs +++ b/src/query/functions/src/scalars/timestamp/src/datetime.rs @@ -1527,14 +1527,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "epoch_microsecond", |_, _| FunctionDomain::Full, - vectorize_1_arg::(|val, ctx| { - (val.to_timestamp(ctx.func_ctx.tz.clone()) - .with_time_zone(TimeZone::UTC) - .timestamp() - .as_microsecond() as f64 - / 1_000_000f64) - .into() - }), + vectorize_1_arg::(|val, _| (val as f64 / 1_000_000f64).into()), ); registry.register_passthrough_nullable_1_arg::( diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test index a6a3d6809dde3..8c0b1d32fd265 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test @@ -1124,6 +1124,16 @@ select EXTRACT(YEAR FROM to_datetime('2022-03-04 22:32:09')) = 2022 ---- 1 +query T +select EXTRACT(EPOCH FROM to_datetime('1969-12-31 23:59:59')) +---- +-1.0 + +query T +select EXTRACT(EPOCH FROM to_datetime('1969-12-31 23:59:59.999999')) +---- +-1.0e-6 + query B select EXTRACT(EPOCH FROM to_datetime('2022-03-04 22:32:09')) = to_unix_timestamp('2022-03-04 22:32:09'); ---- From 18358b955a430cabcac18b0957e5799b008a9303 Mon Sep 17 00:00:00 2001 From: TCeason Date: Fri, 31 Jan 2025 19:14:31 +0800 Subject: [PATCH 3/4] add domain --- .../src/scalars/timestamp/src/datetime.rs | 7 ++++++- .../functions/02_0012_function_datetimes.test | 20 ++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/query/functions/src/scalars/timestamp/src/datetime.rs b/src/query/functions/src/scalars/timestamp/src/datetime.rs index 3fee840462ea0..d7b72bf394840 100644 --- a/src/query/functions/src/scalars/timestamp/src/datetime.rs +++ b/src/query/functions/src/scalars/timestamp/src/datetime.rs @@ -1526,7 +1526,12 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "epoch_microsecond", - |_, _| FunctionDomain::Full, + |_, domain| { + FunctionDomain::Domain(SimpleDomain:: { + min: (domain.min as f64 / 1_000_000f64).into(), + max: (domain.max as f64 / 1_000_000f64).into(), + }) + }, vectorize_1_arg::(|val, _| (val as f64 / 1_000_000f64).into()), ); diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test index 8c0b1d32fd265..4176fb2e22512 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test @@ -1129,15 +1129,25 @@ select EXTRACT(EPOCH FROM to_datetime('1969-12-31 23:59:59')) ---- -1.0 +statement ok +create or replace table t_epoch(c timestamp); + +statement ok +insert into t_epoch values('1969-12-31 23:59:59'), ('1970-01-01 00:00:01.123456789'); + query T -select EXTRACT(EPOCH FROM to_datetime('1969-12-31 23:59:59.999999')) +select EXTRACT(EPOCH FROM c) from t_epoch order by c; ---- --1.0e-6 +-1.0 +1.123456 -query B -select EXTRACT(EPOCH FROM to_datetime('2022-03-04 22:32:09')) = to_unix_timestamp('2022-03-04 22:32:09'); +statement ok +drop table if exists t_epoch; + +query T +select EXTRACT(EPOCH FROM to_datetime('1969-12-31 23:59:59.999999')) ---- -1 +-1.0e-6 query B select EXTRACT(EPOCH FROM now()) = epoch_microsecond(now()) From 840c58bfe0c3bc30ee2e99c8014e50a76bdf3dd0 Mon Sep 17 00:00:00 2001 From: TCeason Date: Sat, 1 Feb 2025 06:51:08 +0800 Subject: [PATCH 4/4] function rename to epoch --- src/query/functions/src/scalars/timestamp/src/datetime.rs | 2 +- .../functions/tests/it/scalars/testdata/function_list.txt | 8 ++++---- src/query/sql/src/planner/semantic/type_check.rs | 4 +--- .../query/functions/02_0012_function_datetimes.test | 4 ++-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/query/functions/src/scalars/timestamp/src/datetime.rs b/src/query/functions/src/scalars/timestamp/src/datetime.rs index d7b72bf394840..5eec3013e8975 100644 --- a/src/query/functions/src/scalars/timestamp/src/datetime.rs +++ b/src/query/functions/src/scalars/timestamp/src/datetime.rs @@ -1525,7 +1525,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { ); registry.register_passthrough_nullable_1_arg::( - "epoch_microsecond", + "epoch", |_, domain| { FunctionDomain::Domain(SimpleDomain:: { min: (domain.min as f64 / 1_000_000f64).into(), diff --git a/src/query/functions/tests/it/scalars/testdata/function_list.txt b/src/query/functions/tests/it/scalars/testdata/function_list.txt index 28e1becb9633d..6cd915227622e 100644 --- a/src/query/functions/tests/it/scalars/testdata/function_list.txt +++ b/src/query/functions/tests/it/scalars/testdata/function_list.txt @@ -1671,10 +1671,10 @@ Functions overloads: 199 divide(Float64, Float64) :: Float64 200 divide(Float64 NULL, Float64 NULL) :: Float64 NULL 0 divnull(Float64 NULL, Float64 NULL) :: Float64 NULL -0 epoch(Int64) :: Interval -1 epoch(Int64 NULL) :: Interval NULL -0 epoch_microsecond(Timestamp) :: Float64 -1 epoch_microsecond(Timestamp NULL) :: Float64 NULL +0 epoch(Timestamp) :: Float64 +1 epoch(Timestamp NULL) :: Float64 NULL +2 epoch(Int64) :: Interval +3 epoch(Int64 NULL) :: Interval NULL 0 eq(Variant, Variant) :: Boolean 1 eq(Variant NULL, Variant NULL) :: Boolean NULL 2 eq(String, String) :: Boolean diff --git a/src/query/sql/src/planner/semantic/type_check.rs b/src/query/sql/src/planner/semantic/type_check.rs index bc63c99807b59..94d8c5c1c089b 100644 --- a/src/query/sql/src/planner/semantic/type_check.rs +++ b/src/query/sql/src/planner/semantic/type_check.rs @@ -2930,9 +2930,7 @@ impl<'a> TypeChecker<'a> { ASTIntervalKind::Doy => self.resolve_function(span, "to_day_of_year", vec![], &[arg]), ASTIntervalKind::Dow => self.resolve_function(span, "to_day_of_week", vec![], &[arg]), ASTIntervalKind::Week => self.resolve_function(span, "to_week_of_year", vec![], &[arg]), - ASTIntervalKind::Epoch => { - self.resolve_function(span, "epoch_microsecond", vec![], &[arg]) - } + ASTIntervalKind::Epoch => self.resolve_function(span, "epoch", vec![], &[arg]), } } diff --git a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test index 4176fb2e22512..7268e9e7029ba 100644 --- a/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test +++ b/tests/sqllogictests/suites/query/functions/02_0012_function_datetimes.test @@ -1150,12 +1150,12 @@ select EXTRACT(EPOCH FROM to_datetime('1969-12-31 23:59:59.999999')) -1.0e-6 query B -select EXTRACT(EPOCH FROM now()) = epoch_microsecond(now()) +select EXTRACT(EPOCH FROM now()) = epoch(now()) ---- 1 query B -SELECT (to_timestamp('2023-11-12 09:38:18.965575') - to_timestamp(0)) / 1000000.0 = epoch_microsecond('2023-11-12 09:38:18.965575') +SELECT (to_timestamp('2023-11-12 09:38:18.965575') - to_timestamp(0)) / 1000000.0 = epoch('2023-11-12 09:38:18.965575') ---- 1