From 3dd12e0ca2e040c71faab8ab57750959ee01abdd Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 7 Sep 2022 20:41:19 -0700 Subject: [PATCH 01/13] Add implementation for `FROM_UNIXTIME` and `UNIX_TIMESTAMP` functions, UT and IT. Signed-off-by: Yury-Fridlyand --- .../expression/datetime/DateTimeFunction.java | 160 +++++++++++ .../function/BuiltinFunctionName.java | 2 + .../expression/datetime/FromUnixTimeTest.java | 165 +++++++++++ .../datetime/UnixTimeStampTest.java | 265 ++++++++++++++++++ .../datetime/UnixTwoWayConversionTest.java | 136 +++++++++ .../sql/ppl/DateTimeFunctionIT.java | 26 ++ .../sql/sql/DateTimeFunctionIT.java | 26 ++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 +- sql/src/main/antlr/OpenSearchSQLLexer.g4 | 2 + sql/src/main/antlr/OpenSearchSQLParser.g4 | 2 +- 11 files changed, 786 insertions(+), 2 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java create mode 100644 core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java create mode 100644 core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 0fccacd136..819c689d4c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -6,6 +6,12 @@ package org.opensearch.sql.expression.datetime; +import static java.time.temporal.ChronoField.DAY_OF_MONTH; +import static java.time.temporal.ChronoField.HOUR_OF_DAY; +import static java.time.temporal.ChronoField.MINUTE_OF_HOUR; +import static java.time.temporal.ChronoField.MONTH_OF_YEAR; +import static java.time.temporal.ChronoField.SECOND_OF_MINUTE; +import static java.time.temporal.ChronoField.YEAR; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; @@ -19,15 +25,30 @@ import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.time.Instant; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.format.DateTimeParseException; +import java.time.format.ResolverStyle; +import java.time.format.SignStyle; import java.time.format.TextStyle; +import java.time.temporal.ChronoField; import java.util.Locale; +import java.util.TimeZone; import java.util.concurrent.TimeUnit; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDatetimeValue; +import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprIntegerValue; import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprNullValue; @@ -35,6 +56,7 @@ import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.FunctionName; @@ -67,6 +89,7 @@ public void register(BuiltinFunctionRepository repository) { repository.register(dayOfWeek()); repository.register(dayOfYear()); repository.register(from_days()); + repository.register(from_unixtime()); repository.register(hour()); repository.register(makedate()); repository.register(maketime()); @@ -82,6 +105,7 @@ public void register(BuiltinFunctionRepository repository) { repository.register(timestamp()); repository.register(date_format()); repository.register(to_days()); + repository.register(unix_timestamp()); repository.register(week()); repository.register(year()); } @@ -230,6 +254,14 @@ private FunctionResolver from_days() { impl(nullMissingHandling(DateTimeFunction::exprFromDays), DATE, LONG)); } + private FunctionResolver from_unixtime() { + return define(BuiltinFunctionName.FROM_UNIXTIME.getName(), + impl(nullMissingHandling(DateTimeFunction::exprFromUnixTime), DATETIME, LONG), + impl(nullMissingHandling(DateTimeFunction::exprFromUnixTime), DATETIME, DOUBLE), + impl(nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat), STRING, LONG, STRING), + impl(nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat), STRING, DOUBLE, STRING)); + } + /** * HOUR(STRING/TIME/DATETIME/TIMESTAMP). return the hour value for time. */ @@ -378,6 +410,16 @@ private FunctionResolver to_days() { impl(nullMissingHandling(DateTimeFunction::exprToDays), LONG, DATETIME)); } + private FunctionResolver unix_timestamp() { + return define(BuiltinFunctionName.UNIX_TIMESTAMP.getName(), + impl(DateTimeFunction::unixTimeStamp, LONG), + impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DATE), + impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DATETIME), + impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, TIMESTAMP), + impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DOUBLE) + ); + } + /** * WEEK(DATE[,mode]). return the week number for date. */ @@ -518,6 +560,36 @@ private ExprValue exprFromDays(ExprValue exprValue) { return new ExprDateValue(LocalDate.ofEpochDay(exprValue.longValue() - DAYS_0000_TO_1970)); } + private ExprValue exprFromUnixTime(ExprValue time) { + if (0 > time.doubleValue()) + return ExprNullValue.of(); + // According to MySQL documentation: + // effective maximum is 32536771199.999999, which returns '3001-01-18 23:59:59.999999' UTC. + // Regardless of platform or version, a greater value for first argument than the effective + // maximum returns 0. + if (32536771200d <= time.doubleValue()) + return ExprNullValue.of(); + return new ExprDatetimeValue(exprFromUnixTimeImpl(time)); + } + + private LocalDateTime exprFromUnixTimeImpl(ExprValue time) { + if (LONG == time.type()) { + return LocalDateTime.ofInstant(Instant.ofEpochSecond(time.longValue()), + ZoneId.of("UTC")); + } + return LocalDateTime.ofInstant( + Instant.ofEpochSecond((long)Math.floor(time.doubleValue())), + ZoneId.of("UTC")) + .withNano((int)((time.doubleValue() % 1) * 1E9)); + } + + private ExprValue exprFromUnixTimeFormat(ExprValue time, ExprValue format) { + var value = exprFromUnixTime(time); + if (value.equals(ExprNullValue.of())) + return ExprNullValue.of(); + return DateTimeFormatterUtil.getFormattedDate(value, format); + } + /** * Hour implementation for ExprValue. * @@ -719,6 +791,94 @@ private ExprValue exprWeek(ExprValue date, ExprValue mode) { CalendarLookup.getWeekNumber(mode.integerValue(), date.dateValue())); } + private ExprValue unixTimeStamp() { + return new ExprLongValue(Instant.now().getEpochSecond()); + } + + private ExprValue unixTimeStampOf(ExprValue value) { + var res = unixTimeStampOfImpl(value); + if (res == null) + return ExprNullValue.of(); + if (res < 0) + // According to MySQL returns 0 if year < 1970, don't return negative values as java does. + return new ExprDoubleValue(0); + if (res >= 32536771200d) + // Return 0 also for dates > '3001-01-19 03:14:07.999999' UTC (32536771199.999999 sec) + return new ExprDoubleValue(0); + return new ExprDoubleValue(res); + } + + private Double unixTimeStampOfImpl(ExprValue value) { + // Also, according to MySQL documentation: + // The date argument may be a DATE, DATETIME, or TIMESTAMP ... + switch ((ExprCoreType)value.type()) { + case DATE: return value.dateValue().toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d; + case DATETIME: return value.datetimeValue().toEpochSecond(ZoneOffset.UTC) + value.datetimeValue().getNano() / 1E9; + case TIMESTAMP: return value.timestampValue().getEpochSecond() + value.timestampValue().getNano() / 1E9; + default: + // ... or a number in YYMMDD, YYMMDDhhmmss, YYYYMMDD, or YYYYMMDDhhmmss format. + // If the argument includes a time part, it may optionally include a fractional + // seconds part. + + var dateFormatShortYear = new DateTimeFormatterBuilder() + .appendValueReduced(YEAR, 2, 2, 1970) + .appendPattern("MMdd") + .toFormatter() + .withResolverStyle(ResolverStyle.STRICT); + + var dateFormatLongYear = new DateTimeFormatterBuilder() + .appendValue(YEAR, 4) + .appendPattern("MMdd") + .toFormatter() + .withResolverStyle(ResolverStyle.STRICT); + + var dateTimeFormatShortYear = new DateTimeFormatterBuilder() + .appendValueReduced(YEAR, 2, 2, 1970) + .appendPattern("MMddHHmmss") + .toFormatter() + .withResolverStyle(ResolverStyle.STRICT); + + var dateTimeFormatLongYear = new DateTimeFormatterBuilder() + .appendValue(YEAR,4) + .appendPattern("MMddHHmmss") + .toFormatter() + .withResolverStyle(ResolverStyle.STRICT); + + var format = new DecimalFormat("0.#"); + format.setMinimumFractionDigits(0); + format.setMaximumFractionDigits(6); + String input = format.format(value.doubleValue()); + double fraction = 0; + if (input.contains(".")) { + try { + // Keeping fraction second part and adding it to the result, don't parse it as part of datetime + // Because `toEpochSecond` returns only `long` + // input = 12345.6789 becomes input = 12345 and fraction = 0.6789 + fraction = Double.parseDouble(input.substring(input.indexOf('.'))); + } catch (NumberFormatException ignored) {} + input = input.substring(0, input.indexOf('.')); + } + try { + var res = LocalDateTime.parse(input, dateTimeFormatShortYear); + return res.toEpochSecond(ZoneOffset.UTC) + fraction; + } catch (DateTimeParseException ignored) {} + try { + var res = LocalDateTime.parse(input, dateTimeFormatLongYear); + return res.toEpochSecond(ZoneOffset.UTC) + fraction; + } catch (DateTimeParseException ignored) {} + try { + var res = LocalDate.parse(input, dateFormatShortYear); + return res.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d; + } catch (DateTimeParseException ignored) {} + try { + var res = LocalDate.parse(input, dateFormatLongYear); + return res.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d; + } catch (DateTimeParseException ignored) {} + + return null; + } + } + /** * Week for date implementation for ExprValue. * When mode is not specified default value mode 0 is used for default_week_format. diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index b3821d6e41..ee978c6815 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -67,6 +67,7 @@ public enum BuiltinFunctionName { DAYOFWEEK(FunctionName.of("dayofweek")), DAYOFYEAR(FunctionName.of("dayofyear")), FROM_DAYS(FunctionName.of("from_days")), + FROM_UNIXTIME(FunctionName.of("from_unixtime")), HOUR(FunctionName.of("hour")), MAKEDATE(FunctionName.of("makedate")), MAKETIME(FunctionName.of("maketime")), @@ -82,6 +83,7 @@ public enum BuiltinFunctionName { TIMESTAMP(FunctionName.of("timestamp")), DATE_FORMAT(FunctionName.of("date_format")), TO_DAYS(FunctionName.of("to_days")), + UNIX_TIMESTAMP(FunctionName.of("unix_timestamp")), WEEK(FunctionName.of("week")), YEAR(FunctionName.of("year")), diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java new file mode 100644 index 0000000000..d03be2ac11 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java @@ -0,0 +1,165 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.datetime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.text.DecimalFormat; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionSignature; + +public class FromUnixTimeTest { + + Environment env; + + private FunctionExpression fromUnixTime(Expression value) { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), + List.of(value.type()))); + return (FunctionExpression)func.apply(List.of(value)); + } + + private FunctionExpression fromUnixTime(Expression value, Expression format) { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), + List.of(value.type(), format.type()))); + return (FunctionExpression)func.apply(List.of(value, format)); + } + + private LocalDateTime fromUnixTime(Long value) { + return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); + } + + private LocalDateTime fromUnixTime(Double value) { + return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); + } + + private String fromUnixTime(Long value, String format) { + return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); + } + + private String fromUnixTime(Double value, String format) { + return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); + } + + private ExprValue eval(Expression expression) { + return expression.valueOf(env); + } + + private static Stream getLongSamples() { + return Stream.of( + Arguments.of(0L), + Arguments.of(1L), + Arguments.of(1447430881L), + Arguments.of(2147483647L), + Arguments.of(1662577241L) + ); + } + + @ParameterizedTest + @MethodSource("getLongSamples") + public void checkOfLong(Long value) { + assertEquals(LocalDateTime.of(1970, 1, 1, 0, 0, 0).atZone(ZoneId.of("UTC")).plus(value, ChronoUnit.SECONDS).toLocalDateTime(), fromUnixTime(value)); + assertEquals(LocalDateTime.of(1970, 1, 1, 0, 0, 0).plus(value, ChronoUnit.SECONDS), eval(fromUnixTime(DSL.literal(new ExprLongValue(value)))).datetimeValue()); + } + + private static Stream getDoubleSamples() { + return Stream.of( + Arguments.of(0.123d), + Arguments.of(100500.100500d), + Arguments.of(1447430881.564d), + Arguments.of(2147483647.451232d), + Arguments.of(1662577241.d) + ); + } + + @ParameterizedTest + @MethodSource("getDoubleSamples") + public void checkOfDouble(Double value) { + var intPart = Math.round(Math.floor(value)); + var fracPart = value - intPart; + + assertEquals(LocalDateTime.ofEpochSecond(intPart, (int)Math.round(fracPart * 1E9), ZoneOffset.UTC), fromUnixTime(value), new DecimalFormat("0.#").format(value)); + assertEquals(LocalDateTime.ofEpochSecond(intPart, (int)Math.round(fracPart * 1E9), ZoneOffset.UTC), eval(fromUnixTime(DSL.literal(new ExprDoubleValue(value)))).datetimeValue(), new DecimalFormat("0.#").format(value)); + } + + private static Stream getLongSamplesWithFormat() { + return Stream.of( + Arguments.of(0L, "%c", "01"), // 1970-01-01 00:00:00, %c - 2 digit month + Arguments.of(1L, "%Y", "1970"), // 1970-01-01 00:00:01, %Y - 4 digit year + Arguments.of(1447430881L, "%s", "01"), // 2015-11-13 16:08:01, %s - second + Arguments.of(2147483647L, "%T", "03:14:07"), // 2038-01-19 03:14:07, %T - time + Arguments.of(1662577241L, "%d", "07") // 1662577241, %d - day of the month + ); + } + + @ParameterizedTest + @MethodSource("getLongSamplesWithFormat") + public void checkOfLong(Long value, String format, String expected) { + assertEquals(expected, fromUnixTime(value, format)); + assertEquals(expected, eval(fromUnixTime(DSL.literal(new ExprLongValue(value)), DSL.literal(new ExprStringValue(format)))).stringValue()); + } + + private static Stream getDoubleSamplesWithFormat() { + return Stream.of( + Arguments.of(0.123d, "%f", "123000"), // 1970-01-01 00:00:00.123, %f - microseconds + Arguments.of(100500.1005d, "%W", "Friday"), // 1970-01-02 03:55:00.1005, %W - Weekday name + Arguments.of(1447430881.56d, "%M", "November"), // 2015-11-13 16:08:01.56, %M - Month name + Arguments.of(2147483647.42d, "%j", "019"), // 2038-01-19 03:14:07.42, %j - day of the year + Arguments.of(1662577241.d, "%l", "7") // 2022-09-07 19:00:41, %l - 12 hour clock, no 0 pad + ); + } + + @ParameterizedTest + @MethodSource("getDoubleSamplesWithFormat") + public void checkOfLong(Double value, String format, String expected) { + assertEquals(expected, fromUnixTime(value, format)); + assertEquals(expected, eval(fromUnixTime(DSL.literal(new ExprDoubleValue(value)), DSL.literal(new ExprStringValue(format)))).stringValue()); + } + + @Test + public void checkInvalidValues() { + assertEquals(new ExprStringValue("q"), fromUnixTime(DSL.literal(0L), DSL.literal("%q")).valueOf(null)); + assertEquals(new ExprStringValue(""), fromUnixTime(DSL.literal(0L), DSL.literal("")).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1L)).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1.5d)).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200L)).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200d)).valueOf(null)); + assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199L)).valueOf(null)); + assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199d)).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200L), DSL.literal("%d")).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200d), DSL.literal("%d")).valueOf(null)); + assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199L), DSL.literal("%d")).valueOf(null)); + assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199d), DSL.literal("%d")).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of())).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1L), DSL.literal("%d")).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1.5d), DSL.literal("%d")).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(42L), DSL.literal(ExprNullValue.of())).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of()), DSL.literal("%d")).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of()), DSL.literal(ExprNullValue.of())).valueOf(null)); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java new file mode 100644 index 0000000000..5db22f1a1f --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java @@ -0,0 +1,265 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.datetime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.math.BigDecimal; +import java.text.DecimalFormat; +import java.time.format.DateTimeFormatter; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.temporal.ChronoField; +import java.time.ZoneOffset; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.data.model.ExprDateValue; +import org.opensearch.sql.data.model.ExprDatetimeValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ExpressionTestBase; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionSignature; + +public class UnixTimeStampTest extends ExpressionTestBase { + + Environment env; + + private FunctionExpression unixTimeStampExpr() { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), List.of())); + return (FunctionExpression)func.apply(List.of()); + } + + private Long unixTimeStamp() { + return unixTimeStampExpr().valueOf(null).longValue(); + } + + private FunctionExpression unixTimeStampOf(Expression value) { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), + List.of(value.type()))); + return (FunctionExpression)func.apply(List.of(value)); + } + + private Double unixTimeStampOf(Double value) { + return unixTimeStampOf(DSL.literal(value)).valueOf(null).doubleValue(); + } + + private Double unixTimeStampOf(LocalDate value) { + return unixTimeStampOf(DSL.literal(new ExprDateValue(value))).valueOf(null).doubleValue(); + } + + private Double unixTimeStampOf(LocalDateTime value) { + return unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))).valueOf(null).doubleValue(); + } + + private Double unixTimeStampOf(Instant value) { + return unixTimeStampOf(DSL.literal(new ExprTimestampValue(value))).valueOf(null).doubleValue(); + } + + private ExprValue eval(Expression expression) { + return expression.valueOf(env); + } + + @Test + public void checkNoArgs() { + assertEquals(System.currentTimeMillis() / 1000L, unixTimeStamp()); + assertEquals(System.currentTimeMillis() / 1000L, eval(unixTimeStampExpr()).longValue()); + } + + private static Stream getDateSamples() { + return Stream.of( + Arguments.of(LocalDate.of(1984, 1, 1)), + Arguments.of(LocalDate.of(2000, 2, 29)), + Arguments.of(LocalDate.of(1999, 12, 31)), + Arguments.of(LocalDate.of(2004, 2, 29)), + Arguments.of(LocalDate.of(2100, 2, 28)), + Arguments.of(LocalDate.of(2012, 2, 21)) + ); + } + + @ParameterizedTest + @MethodSource("getDateSamples") + public void checkOfDate(LocalDate date) { + assertEquals(date.getLong(ChronoField.EPOCH_DAY) * 24 * 3600, unixTimeStampOf(date)); + assertEquals(date.getLong(ChronoField.EPOCH_DAY) * 24 * 3600, eval(unixTimeStampOf(DSL.literal(new ExprDateValue(date)))).longValue()); + } + + private static Stream getDateTimeSamples() { + return Stream.of( + Arguments.of(LocalDateTime.of(1984, 1, 1, 1, 1)), + Arguments.of(LocalDateTime.of(2000, 2, 29, 22, 54)), + Arguments.of(LocalDateTime.of(1999, 12, 31, 23, 59)), + Arguments.of(LocalDateTime.of(2004, 2, 29, 7, 40)), + Arguments.of(LocalDateTime.of(2100, 2, 28, 13, 14)), + Arguments.of(LocalDateTime.of(2012, 2, 21, 0, 0)) + ); + } + + @ParameterizedTest + @MethodSource("getDateTimeSamples") + public void checkOfDateTime(LocalDateTime dateTime) { + assertEquals(dateTime.toEpochSecond(ZoneOffset.UTC), unixTimeStampOf(dateTime)); + assertEquals(dateTime.toEpochSecond(ZoneOffset.UTC), eval(unixTimeStampOf(DSL.literal(new ExprDatetimeValue(dateTime)))).longValue()); + } + + private static Stream getInstantSamples() { + return getDateTimeSamples().map(v -> Arguments.of(((LocalDateTime)v.get()[0]).toInstant(ZoneOffset.UTC))); + } + + @ParameterizedTest + @MethodSource("getInstantSamples") + public void checkOfDateTime(Instant instant) { + assertEquals(instant.getEpochSecond(), unixTimeStampOf(instant)); + assertEquals(instant.getEpochSecond(), eval(unixTimeStampOf(DSL.literal(new ExprTimestampValue(instant)))).longValue()); + } + + // formats: YYMMDD, YYMMDDhhmmss[.uuuuuu], YYYYMMDD, or YYYYMMDDhhmmss[.uuuuuu] + // use BigDecimal, because double can't fit such big values + private static Stream getDoubleSamples() { + return Stream.of( + Arguments.of(new BigDecimal("840101.")), + Arguments.of(new BigDecimal("840101112233.")), + Arguments.of(new BigDecimal("840101112233.123456")), + Arguments.of(new BigDecimal("19840101.")), + Arguments.of(new BigDecimal("19840101000000.")), + Arguments.of(new BigDecimal("19840101112233.")), + Arguments.of(new BigDecimal("19840101112233.123456")) + ); + } + + @ParameterizedTest + @MethodSource("getDoubleSamples") + public void checkOfDoubleFormats(BigDecimal value) { + LocalDateTime valueDt = LocalDateTime.MIN; + var format = new DecimalFormat("0.#"); + format.setMinimumFractionDigits(0); + format.setMaximumFractionDigits(6); + var valueStr = format.format(value); + switch (valueStr.length()) { + case 6: valueStr = "19" + valueStr; + case 8: valueStr += "000000"; + case 14: valueDt = LocalDateTime.parse(valueStr, DateTimeFormatter.ofPattern("yyyyMMddHHmmss")); break; + case 12: + case 19: valueStr = "19" + valueStr; + case 21: valueDt = LocalDateTime.parse(valueStr, DateTimeFormatter.ofPattern("yyyyMMddHHmmss[.SSSSSS]")); break; + } + assertEquals(valueDt.toEpochSecond(ZoneOffset.UTC), unixTimeStampOf(value.doubleValue()), 1d, format.format(value)); + assertEquals(valueDt.toEpochSecond(ZoneOffset.UTC), eval(unixTimeStampOf(DSL.literal(new ExprDoubleValue(value.doubleValue())))).longValue(), 1d, format.format(value)); + } + + @Test + public void checkOfDouble() { + // 19991231235959.99 passed ok, but 19991231235959.999999 rounded to ...60.0 which is incorrect + // It is a double type limitation + var valueDt = LocalDateTime.of(1999, 12, 31, 23, 59, 59, 999_999_999); + assertEquals(valueDt.toEpochSecond(ZoneOffset.UTC), unixTimeStampOf(19991231235959.99d), 1d); + } + + @Test + public void checkYearLessThan1970() { + assertNotEquals(0, unixTimeStamp()); + assertEquals(0, unixTimeStampOf(LocalDate.of(1961, 4, 12))); + assertEquals(0, unixTimeStampOf(LocalDateTime.of(1961, 4, 12, 9, 7, 0))); + assertEquals(0, unixTimeStampOf(Instant.ofEpochMilli(-1))); + assertEquals(0, unixTimeStampOf(LocalDateTime.of(1970, 1, 1, 0, 0, 0))); + assertEquals(1, unixTimeStampOf(LocalDateTime.of(1970, 1, 1, 0, 0, 1))); + assertEquals(0, unixTimeStampOf(19610412d)); + assertEquals(0, unixTimeStampOf(19610412090700d)); + } + + @Test + public void checkMaxValue() { + // MySQL returns 0 for values above + // '3001-01-19 03:14:07.999999' UTC (corresponding to 32536771199.999999 seconds). + assertEquals(0, unixTimeStampOf(LocalDate.of(3001, 1, 20))); + assertNotEquals(0d, unixTimeStampOf(LocalDate.of(3001, 1, 18))); + assertEquals(0, unixTimeStampOf(LocalDateTime.of(3001, 1, 20, 3, 14, 8))); + assertNotEquals(0d, unixTimeStampOf(LocalDateTime.of(3001, 1, 18, 3, 14, 7))); + assertEquals(0, unixTimeStampOf(Instant.ofEpochSecond(32536771199L + 1))); + assertNotEquals(0d, unixTimeStampOf(Instant.ofEpochSecond(32536771199L))); + assertEquals(0, unixTimeStampOf(30010120d)); + assertNotEquals(0d, unixTimeStampOf(30010118d)); + } + + private static Stream getInvalidDoubleSamples() { + return Stream.of( + //invalid dates + Arguments.of(19990231.d), + Arguments.of(19991320.d), + Arguments.of(19991232.d), + Arguments.of(19990013.d), + Arguments.of(19990931.d), + Arguments.of(990231.d), + Arguments.of(991320.d), + Arguments.of(991232.d), + Arguments.of(990013.d), + Arguments.of(990931.d), + Arguments.of(9990102.d), + Arguments.of(99102.d), + Arguments.of(9912.d), + Arguments.of(199912.d), + Arguments.of(1999102.d), + //same as above, but with valid time + Arguments.of(19990231112233.d), + Arguments.of(19991320112233.d), + Arguments.of(19991232112233.d), + Arguments.of(19990013112233.d), + Arguments.of(19990931112233.d), + Arguments.of(990231112233.d), + Arguments.of(991320112233.d), + Arguments.of(991232112233.d), + Arguments.of(990013112233.d), + Arguments.of(990931112233.d), + Arguments.of(9990102112233.d), + Arguments.of(99102112233.d), + Arguments.of(9912112233.d), + Arguments.of(199912112233.d), + Arguments.of(1999102112233.d), + //invalid time + Arguments.of(19840101242233.d), + Arguments.of(19840101116033.d), + Arguments.of(19840101112260.d), + Arguments.of(1984010111223.d), + Arguments.of(198401011123.d), + Arguments.of(19840101123.d), + Arguments.of(1984010113.d), + Arguments.of(198401011.d), + //same, but with short date + Arguments.of(840101242233.d), + Arguments.of(840101116033.d), + Arguments.of(840101112260.d), + Arguments.of(84010111223.d), + Arguments.of(8401011123.d), + Arguments.of(840101123.d), + Arguments.of(8401011.d), + //misc + Arguments.of(0d), + Arguments.of(-1d), + Arguments.of(42d) + ); + } + + @ParameterizedTest + @MethodSource("getInvalidDoubleSamples") + public void checkInvalidDoubleCausesNull(Double value) { + assertEquals(ExprNullValue.of(), unixTimeStampOf(DSL.literal(new ExprDoubleValue(value))).valueOf(null), new DecimalFormat("0.#").format(value)); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java new file mode 100644 index 0000000000..3c05ca1f5f --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java @@ -0,0 +1,136 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.expression.datetime; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.data.model.ExprDateValue; +import org.opensearch.sql.data.model.ExprDatetimeValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionSignature; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class UnixTwoWayConversionTest { + + Environment env; + + private FunctionExpression unixTimeStampExpr() { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), List.of())); + return (FunctionExpression)func.apply(List.of()); + } + + private Long unixTimeStamp() { + return unixTimeStampExpr().valueOf(null).longValue(); + } + + private FunctionExpression unixTimeStampOf(Expression value) { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), + List.of(value.type()))); + return (FunctionExpression)func.apply(List.of(value)); + } + + private Double unixTimeStampOf(Double value) { + return unixTimeStampOf(DSL.literal(value)).valueOf(null).doubleValue(); + } + + private Double unixTimeStampOf(LocalDate value) { + return unixTimeStampOf(DSL.literal(new ExprDateValue(value))).valueOf(null).doubleValue(); + } + + private Double unixTimeStampOf(LocalDateTime value) { + return unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))).valueOf(null).doubleValue(); + } + + private Double unixTimeStampOf(Instant value) { + return unixTimeStampOf(DSL.literal(new ExprTimestampValue(value))).valueOf(null).doubleValue(); + } + + private FunctionExpression fromUnixTime(Expression value) { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), + List.of(value.type()))); + return (FunctionExpression)func.apply(List.of(value)); + } + + private LocalDateTime fromUnixTime(Long value) { + return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); + } + + private LocalDateTime fromUnixTime(Double value) { + return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); + } + + private ExprValue eval(Expression expression) { + return expression.valueOf(env); + } + + @Test + public void checkConvertNow() { + assertEquals(LocalDateTime.now(ZoneId.of("UTC")).withNano(0), fromUnixTime(unixTimeStamp())); + assertEquals(LocalDateTime.now(ZoneId.of("UTC")).withNano(0), eval(fromUnixTime(unixTimeStampExpr())).datetimeValue()); + } + + private static Stream getDoubleSamples() { + return Stream.of( + Arguments.of(0.123d), + Arguments.of(100500.100500d), + Arguments.of(1447430881.564d), + Arguments.of(2147483647.451232d), + Arguments.of(1662577241.d) + ); + } + + @ParameterizedTest + @MethodSource("getDoubleSamples") + public void convertEpoch2DateTime2Epoch(Double value) { + assertEquals(value, unixTimeStampOf(fromUnixTime(value))); + assertEquals(value, eval(unixTimeStampOf(fromUnixTime(DSL.literal(new ExprDoubleValue(value))))).doubleValue()); + + assertEquals(Math.round(value) + 0d, unixTimeStampOf(fromUnixTime(Math.round(value)))); + assertEquals(Math.round(value) + 0d, eval(unixTimeStampOf(fromUnixTime(DSL.literal(new ExprLongValue(Math.round(value)))))).doubleValue()); + } + + private static Stream getDateTimeSamples() { + return Stream.of( + Arguments.of(LocalDateTime.of(1984, 1, 1, 1, 1)), + Arguments.of(LocalDateTime.of(2000, 2, 29, 22, 54)), + Arguments.of(LocalDateTime.of(1999, 12, 31, 23, 59, 59)), + Arguments.of(LocalDateTime.of(2004, 2, 29, 7, 40)), + Arguments.of(LocalDateTime.of(2100, 2, 28, 13, 14, 15)), + Arguments.of(LocalDateTime.of(2012, 2, 21, 0, 0, 17)) + ); + } + + @ParameterizedTest + @MethodSource("getDateTimeSamples") + public void convertDateTime2Epoch2DateTime(LocalDateTime value) { + assertEquals(value, fromUnixTime(unixTimeStampOf(value))); + assertEquals(value, eval(fromUnixTime(unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))))).datetimeValue()); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index 7e0169d174..d6e6bc28c1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -480,4 +480,30 @@ public void testMakeDate() throws IOException { verifySchema(result, schema("f1", null, "date"), schema("f2", null, "date")); verifySome(result.getJSONArray("datarows"), rows("1945-01-06", "1989-06-06")); } + + @Test + public void testFromUnixTime() throws IOException { + var result = executeQuery(String.format( + "source=%s | eval f1 = FROM_UNIXTIME(200300400), f2 = FROM_UNIXTIME(12224.12), " + + "f3 = FROM_UNIXTIME(1662601316, '%%T') | fields f1, f2, f3", TEST_INDEX_DATE)); + verifySchema(result, + schema("f1", null, "datetime"), + schema("f2", null, "datetime"), + schema("f3", null, "string")); + verifySome(result.getJSONArray("datarows"), + rows("1976-05-07 07:00:00", "1970-01-01 03:23:44.12", "01:41:56")); + } + + @Test + public void testUnixTimeStamp() throws IOException { + var result = executeQuery(String.format( + "source=%s | eval f1 = UNIX_TIMESTAMP(MAKEDATE(1984, 1984)), " + + "f2 = UNIX_TIMESTAMP(TIMESTAMP('2003-12-31 12:00:00')), " + + "f3 = UNIX_TIMESTAMP(20771122143845) | fields f1, f2, f3", TEST_INDEX_DATE)); + verifySchema(result, + schema("f1", null, "double"), + schema("f2", null, "double"), + schema("f3", null, "double")); + verifySome(result.getJSONArray("datarows"), rows(613094400d, 1072872000d, 3404817525d)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 7c6bd7efe2..6fd28a1784 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -471,6 +471,32 @@ public void testMakeDate() throws IOException { verifySome(result.getJSONArray("datarows"), rows("1945-01-06", "1989-06-06")); } + @Test + public void testFromUnixTime() throws IOException { + var result = executeQuery( + "select FROM_UNIXTIME(200300400) f1, FROM_UNIXTIME(12224.12) f2, " + + "FROM_UNIXTIME(1662601316, '%T') f3"); + verifySchema(result, + schema("FROM_UNIXTIME(200300400)", "f1", "datetime"), + schema("FROM_UNIXTIME(12224.12)", "f2", "datetime"), + schema("FROM_UNIXTIME(1662601316, '%T')", "f3", "keyword")); + verifySome(result.getJSONArray("datarows"), + rows("1976-05-07 07:00:00", "1970-01-01 03:23:44.12", "01:41:56")); + } + + @Test + public void testUnixTimeStamp() throws IOException { + var result = executeQuery( + "select UNIX_TIMESTAMP(MAKEDATE(1984, 1984)) f1, " + + "UNIX_TIMESTAMP(TIMESTAMP('2003-12-31 12:00:00')) f2, " + + "UNIX_TIMESTAMP(20771122143845) f3"); + verifySchema(result, + schema("UNIX_TIMESTAMP(MAKEDATE(1984, 1984))", "f1", "double"), + schema("UNIX_TIMESTAMP(TIMESTAMP('2003-12-31 12:00:00'))", "f2", "double"), + schema("UNIX_TIMESTAMP(20771122143845)", "f3", "double")); + verifySome(result.getJSONArray("datarows"), rows(613094400d, 1072872000d, 3404817525d)); + } + protected JSONObject executeQuery(String query) throws IOException { Request request = new Request("POST", QUERY_API_ENDPOINT); request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 93df64d0b3..dc2fe1d4c5 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -231,6 +231,7 @@ DAYOFWEEK: 'DAYOFWEEK'; DAYOFYEAR: 'DAYOFYEAR'; DAYNAME: 'DAYNAME'; FROM_DAYS: 'FROM_DAYS'; +FROM_UNIXTIME: 'FROM_UNIXTIME'; MAKEDATE: 'MAKEDATE'; MAKETIME: 'MAKETIME'; MONTHNAME: 'MONTHNAME'; @@ -240,6 +241,7 @@ TIME_TO_SEC: 'TIME_TO_SEC'; TIMESTAMP: 'TIMESTAMP'; DATE_FORMAT: 'DATE_FORMAT'; TO_DAYS: 'TO_DAYS'; +UNIX_TIMESTAMP: 'UNIX_TIMESTAMP'; // TEXT FUNCTIONS SUBSTR: 'SUBSTR'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index c83297459d..0e28f5aa44 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -375,7 +375,7 @@ trigonometricFunctionName dateAndTimeFunctionBase : ADDDATE | DATE | DATE_ADD | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK | DAYOFYEAR | FROM_DAYS | HOUR | MICROSECOND | MINUTE | MONTH | MONTHNAME | QUARTER | SECOND | SUBDATE | TIME | TIME_TO_SEC - | TIMESTAMP | TO_DAYS | YEAR | WEEK | DATE_FORMAT | MAKETIME | MAKEDATE + | TIMESTAMP | TO_DAYS | YEAR | WEEK | DATE_FORMAT | MAKETIME | MAKEDATE | FROM_UNIXTIME | UNIX_TIMESTAMP ; /** condition function return boolean value */ diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index 6d2d7d8a64..7c8ccb74fa 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -205,6 +205,7 @@ EXP: 'EXP'; EXPM1: 'EXPM1'; FLOOR: 'FLOOR'; FROM_DAYS: 'FROM_DAYS'; +FROM_UNIXTIME: 'FROM_UNIXTIME'; IF: 'IF'; IFNULL: 'IFNULL'; ISNULL: 'ISNULL'; @@ -245,6 +246,7 @@ TIME_TO_SEC: 'TIME_TO_SEC'; TIMESTAMP: 'TIMESTAMP'; TRUNCATE: 'TRUNCATE'; TO_DAYS: 'TO_DAYS'; +UNIX_TIMESTAMP: 'UNIX_TIMESTAMP'; UPPER: 'UPPER'; D: 'D'; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 40207df82a..4a9e79e40b 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -385,7 +385,7 @@ trigonometricFunctionName dateTimeFunctionName : ADDDATE | DATE | DATE_ADD | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK | DAYOFYEAR | FROM_DAYS | HOUR | MICROSECOND | MINUTE | MONTH | MONTHNAME | QUARTER | SECOND | SUBDATE | TIME | TIME_TO_SEC - | TIMESTAMP | TO_DAYS | YEAR | WEEK | DATE_FORMAT | MAKETIME | MAKEDATE + | TIMESTAMP | TO_DAYS | YEAR | WEEK | DATE_FORMAT | MAKETIME | MAKEDATE | FROM_UNIXTIME | UNIX_TIMESTAMP ; textFunctionName From c954d07a7307bbdf616fe1f32885f000a2c6dba2 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 8 Sep 2022 10:06:35 -0700 Subject: [PATCH 02/13] Fix checkstyle. Signed-off-by: Yury-Fridlyand --- .../expression/datetime/DateTimeFunction.java | 55 +++++++----- .../datetime/DateTimeFunctionTest.java | 12 --- .../expression/datetime/FromUnixTimeTest.java | 84 ++++++++++++----- .../datetime/UnixTimeStampTest.java | 90 +++++++++++-------- .../datetime/UnixTwoWayConversionTest.java | 28 ++++-- 5 files changed, 171 insertions(+), 98 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 819c689d4c..f39cc092ab 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -259,7 +259,8 @@ private FunctionResolver from_unixtime() { impl(nullMissingHandling(DateTimeFunction::exprFromUnixTime), DATETIME, LONG), impl(nullMissingHandling(DateTimeFunction::exprFromUnixTime), DATETIME, DOUBLE), impl(nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat), STRING, LONG, STRING), - impl(nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat), STRING, DOUBLE, STRING)); + impl(nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat), + STRING, DOUBLE, STRING)); } /** @@ -561,14 +562,16 @@ private ExprValue exprFromDays(ExprValue exprValue) { } private ExprValue exprFromUnixTime(ExprValue time) { - if (0 > time.doubleValue()) + if (0 > time.doubleValue()) { return ExprNullValue.of(); + } // According to MySQL documentation: // effective maximum is 32536771199.999999, which returns '3001-01-18 23:59:59.999999' UTC. // Regardless of platform or version, a greater value for first argument than the effective // maximum returns 0. - if (32536771200d <= time.doubleValue()) + if (32536771200d <= time.doubleValue()) { return ExprNullValue.of(); + } return new ExprDatetimeValue(exprFromUnixTimeImpl(time)); } @@ -585,8 +588,9 @@ private LocalDateTime exprFromUnixTimeImpl(ExprValue time) { private ExprValue exprFromUnixTimeFormat(ExprValue time, ExprValue format) { var value = exprFromUnixTime(time); - if (value.equals(ExprNullValue.of())) + if (value.equals(ExprNullValue.of())) { return ExprNullValue.of(); + } return DateTimeFormatterUtil.getFormattedDate(value, format); } @@ -797,14 +801,17 @@ private ExprValue unixTimeStamp() { private ExprValue unixTimeStampOf(ExprValue value) { var res = unixTimeStampOfImpl(value); - if (res == null) + if (res == null) { return ExprNullValue.of(); - if (res < 0) + } + if (res < 0) { // According to MySQL returns 0 if year < 1970, don't return negative values as java does. return new ExprDoubleValue(0); - if (res >= 32536771200d) + } + if (res >= 32536771200d) { // Return 0 also for dates > '3001-01-19 03:14:07.999999' UTC (32536771199.999999 sec) return new ExprDoubleValue(0); + } return new ExprDoubleValue(res); } @@ -813,8 +820,10 @@ private Double unixTimeStampOfImpl(ExprValue value) { // The date argument may be a DATE, DATETIME, or TIMESTAMP ... switch ((ExprCoreType)value.type()) { case DATE: return value.dateValue().toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d; - case DATETIME: return value.datetimeValue().toEpochSecond(ZoneOffset.UTC) + value.datetimeValue().getNano() / 1E9; - case TIMESTAMP: return value.timestampValue().getEpochSecond() + value.timestampValue().getNano() / 1E9; + case DATETIME: return value.datetimeValue().toEpochSecond(ZoneOffset.UTC) + + value.datetimeValue().getNano() / 1E9; + case TIMESTAMP: return value.timestampValue().getEpochSecond() + + value.timestampValue().getNano() / 1E9; default: // ... or a number in YYMMDD, YYMMDDhhmmss, YYYYMMDD, or YYYYMMDDhhmmss format. // If the argument includes a time part, it may optionally include a fractional @@ -850,32 +859,36 @@ private Double unixTimeStampOfImpl(ExprValue value) { String input = format.format(value.doubleValue()); double fraction = 0; if (input.contains(".")) { - try { - // Keeping fraction second part and adding it to the result, don't parse it as part of datetime - // Because `toEpochSecond` returns only `long` - // input = 12345.6789 becomes input = 12345 and fraction = 0.6789 - fraction = Double.parseDouble(input.substring(input.indexOf('.'))); - } catch (NumberFormatException ignored) {} + // Keeping fraction second part and adding it to the result, don't parse it + // Because `toEpochSecond` returns only `long` + // input = 12345.6789 becomes input = 12345 and fraction = 0.6789 + fraction = value.doubleValue() - Math.round(Math.ceil(value.doubleValue())); input = input.substring(0, input.indexOf('.')); } try { var res = LocalDateTime.parse(input, dateTimeFormatShortYear); return res.toEpochSecond(ZoneOffset.UTC) + fraction; - } catch (DateTimeParseException ignored) {} + } catch (DateTimeParseException ignored) { + // nothing to do, try another format + } try { var res = LocalDateTime.parse(input, dateTimeFormatLongYear); return res.toEpochSecond(ZoneOffset.UTC) + fraction; - } catch (DateTimeParseException ignored) {} + } catch (DateTimeParseException ignored) { + // nothing to do, try another format + } try { var res = LocalDate.parse(input, dateFormatShortYear); return res.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d; - } catch (DateTimeParseException ignored) {} + } catch (DateTimeParseException ignored) { + // nothing to do, try another format + } try { var res = LocalDate.parse(input, dateFormatLongYear); return res.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d; - } catch (DateTimeParseException ignored) {} - - return null; + } catch (DateTimeParseException ignored) { + return null; + } } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 89415e0560..79efa2a015 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -16,7 +16,6 @@ import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; -import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.INTERVAL; import static org.opensearch.sql.data.type.ExprCoreType.LONG; @@ -25,15 +24,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import com.google.common.collect.ImmutableList; -import java.time.Duration; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.Year; -import java.util.HashSet; import java.util.List; -import java.util.Random; -import java.util.Set; -import java.util.stream.IntStream; import lombok.AllArgsConstructor; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -52,10 +43,7 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionTestBase; import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.env.Environment; -import org.opensearch.sql.expression.function.FunctionName; -import org.opensearch.sql.expression.function.FunctionSignature; @ExtendWith(MockitoExtension.class) class DateTimeFunctionTest extends ExpressionTestBase { diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java index d03be2ac11..6434916f12 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java @@ -10,7 +10,6 @@ import java.text.DecimalFormat; import java.time.LocalDateTime; -import java.time.ZoneId; import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; import java.util.List; @@ -80,11 +79,17 @@ private static Stream getLongSamples() { ); } + /** + * Test processing different Long values. + * @param value a value + */ @ParameterizedTest @MethodSource("getLongSamples") public void checkOfLong(Long value) { - assertEquals(LocalDateTime.of(1970, 1, 1, 0, 0, 0).atZone(ZoneId.of("UTC")).plus(value, ChronoUnit.SECONDS).toLocalDateTime(), fromUnixTime(value)); - assertEquals(LocalDateTime.of(1970, 1, 1, 0, 0, 0).plus(value, ChronoUnit.SECONDS), eval(fromUnixTime(DSL.literal(new ExprLongValue(value)))).datetimeValue()); + assertEquals(LocalDateTime.of(1970, 1, 1, 0, 0, 0).plus(value, ChronoUnit.SECONDS), + fromUnixTime(value)); + assertEquals(LocalDateTime.of(1970, 1, 1, 0, 0, 0).plus(value, ChronoUnit.SECONDS), + eval(fromUnixTime(DSL.literal(new ExprLongValue(value)))).datetimeValue()); } private static Stream getDoubleSamples() { @@ -97,16 +102,29 @@ private static Stream getDoubleSamples() { ); } + /** + * Test processing different Double values. + * @param value a value + */ @ParameterizedTest @MethodSource("getDoubleSamples") public void checkOfDouble(Double value) { var intPart = Math.round(Math.floor(value)); var fracPart = value - intPart; + var valueAsString = new DecimalFormat("0.#").format(value); - assertEquals(LocalDateTime.ofEpochSecond(intPart, (int)Math.round(fracPart * 1E9), ZoneOffset.UTC), fromUnixTime(value), new DecimalFormat("0.#").format(value)); - assertEquals(LocalDateTime.ofEpochSecond(intPart, (int)Math.round(fracPart * 1E9), ZoneOffset.UTC), eval(fromUnixTime(DSL.literal(new ExprDoubleValue(value)))).datetimeValue(), new DecimalFormat("0.#").format(value)); + assertEquals( + LocalDateTime.ofEpochSecond(intPart, (int)Math.round(fracPart * 1E9), ZoneOffset.UTC), + fromUnixTime(value), + valueAsString); + assertEquals( + LocalDateTime.ofEpochSecond(intPart, (int)Math.round(fracPart * 1E9), ZoneOffset.UTC), + eval(fromUnixTime(DSL.literal(new ExprDoubleValue(value)))).datetimeValue(), + valueAsString); } + // We are testing all different formats and combinations, because they are already tested + private static Stream getLongSamplesWithFormat() { return Stream.of( Arguments.of(0L, "%c", "01"), // 1970-01-01 00:00:00, %c - 2 digit month @@ -117,11 +135,18 @@ private static Stream getLongSamplesWithFormat() { ); } + /** + * Test processing different Long values with format. + * @param value a value + * @param format a format + * @param expected expected result + */ @ParameterizedTest @MethodSource("getLongSamplesWithFormat") - public void checkOfLong(Long value, String format, String expected) { + public void checkOfLongWithFormat(Long value, String format, String expected) { assertEquals(expected, fromUnixTime(value, format)); - assertEquals(expected, eval(fromUnixTime(DSL.literal(new ExprLongValue(value)), DSL.literal(new ExprStringValue(format)))).stringValue()); + assertEquals(expected, eval(fromUnixTime(DSL.literal(new ExprLongValue(value)), + DSL.literal(new ExprStringValue(format)))).stringValue()); } private static Stream getDoubleSamplesWithFormat() { @@ -134,32 +159,51 @@ private static Stream getDoubleSamplesWithFormat() { ); } + /** + * Test processing different Double values with format. + * @param value a value + * @param format a format + * @param expected expected result + */ @ParameterizedTest @MethodSource("getDoubleSamplesWithFormat") - public void checkOfLong(Double value, String format, String expected) { + public void checkOfDoubleWithFormat(Double value, String format, String expected) { assertEquals(expected, fromUnixTime(value, format)); - assertEquals(expected, eval(fromUnixTime(DSL.literal(new ExprDoubleValue(value)), DSL.literal(new ExprStringValue(format)))).stringValue()); + assertEquals(expected, eval(fromUnixTime(DSL.literal(new ExprDoubleValue(value)), + DSL.literal(new ExprStringValue(format)))).stringValue()); } @Test public void checkInvalidValues() { - assertEquals(new ExprStringValue("q"), fromUnixTime(DSL.literal(0L), DSL.literal("%q")).valueOf(null)); - assertEquals(new ExprStringValue(""), fromUnixTime(DSL.literal(0L), DSL.literal("")).valueOf(null)); + assertEquals(new ExprStringValue("q"), + fromUnixTime(DSL.literal(0L), DSL.literal("%q")).valueOf(null)); + assertEquals(new ExprStringValue(""), + fromUnixTime(DSL.literal(0L), DSL.literal("")).valueOf(null)); assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1L)).valueOf(null)); assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1.5d)).valueOf(null)); assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200L)).valueOf(null)); assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200d)).valueOf(null)); assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199L)).valueOf(null)); assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199d)).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200L), DSL.literal("%d")).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200d), DSL.literal("%d")).valueOf(null)); - assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199L), DSL.literal("%d")).valueOf(null)); - assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199d), DSL.literal("%d")).valueOf(null)); + assertEquals(ExprNullValue.of(), + fromUnixTime(DSL.literal(32536771200L), DSL.literal("%d")).valueOf(null)); + assertEquals(ExprNullValue.of(), + fromUnixTime(DSL.literal(32536771200d), DSL.literal("%d")).valueOf(null)); + assertNotEquals(ExprNullValue.of(), + fromUnixTime(DSL.literal(32536771199L), DSL.literal("%d")).valueOf(null)); + assertNotEquals(ExprNullValue.of(), + fromUnixTime(DSL.literal(32536771199d), DSL.literal("%d")).valueOf(null)); assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of())).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1L), DSL.literal("%d")).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1.5d), DSL.literal("%d")).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(42L), DSL.literal(ExprNullValue.of())).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of()), DSL.literal("%d")).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of()), DSL.literal(ExprNullValue.of())).valueOf(null)); + assertEquals(ExprNullValue.of(), + fromUnixTime(DSL.literal(-1L), DSL.literal("%d")).valueOf(null)); + assertEquals(ExprNullValue.of(), + fromUnixTime(DSL.literal(-1.5d), DSL.literal("%d")).valueOf(null)); + assertEquals(ExprNullValue.of(), + fromUnixTime(DSL.literal(42L), DSL.literal(ExprNullValue.of())).valueOf(null)); + assertEquals(ExprNullValue.of(), + fromUnixTime(DSL.literal(ExprNullValue.of()), DSL.literal("%d")).valueOf(null)); + assertEquals(ExprNullValue.of(), + fromUnixTime(DSL.literal(ExprNullValue.of()), DSL.literal(ExprNullValue.of())) + .valueOf(null)); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java index 5db22f1a1f..980fa624db 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java @@ -8,14 +8,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import java.math.BigDecimal; import java.text.DecimalFormat; -import java.time.format.DateTimeFormatter; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; -import java.time.temporal.ChronoField; import java.time.ZoneOffset; +import java.time.temporal.ChronoField; import java.util.List; import java.util.stream.Stream; import org.junit.jupiter.api.Test; @@ -95,11 +93,16 @@ private static Stream getDateSamples() { ); } + /** + * Check processing valid values of type LocalDate. + * @param value a value + */ @ParameterizedTest @MethodSource("getDateSamples") - public void checkOfDate(LocalDate date) { - assertEquals(date.getLong(ChronoField.EPOCH_DAY) * 24 * 3600, unixTimeStampOf(date)); - assertEquals(date.getLong(ChronoField.EPOCH_DAY) * 24 * 3600, eval(unixTimeStampOf(DSL.literal(new ExprDateValue(date)))).longValue()); + public void checkOfDate(LocalDate value) { + assertEquals(value.getLong(ChronoField.EPOCH_DAY) * 24 * 3600, unixTimeStampOf(value)); + assertEquals(value.getLong(ChronoField.EPOCH_DAY) * 24 * 3600, + eval(unixTimeStampOf(DSL.literal(new ExprDateValue(value)))).longValue()); } private static Stream getDateTimeSamples() { @@ -113,56 +116,63 @@ private static Stream getDateTimeSamples() { ); } + /** + * Check processing valid values of type LocalDateTime. + * @param value a value + */ @ParameterizedTest @MethodSource("getDateTimeSamples") - public void checkOfDateTime(LocalDateTime dateTime) { - assertEquals(dateTime.toEpochSecond(ZoneOffset.UTC), unixTimeStampOf(dateTime)); - assertEquals(dateTime.toEpochSecond(ZoneOffset.UTC), eval(unixTimeStampOf(DSL.literal(new ExprDatetimeValue(dateTime)))).longValue()); + public void checkOfDateTime(LocalDateTime value) { + assertEquals(value.toEpochSecond(ZoneOffset.UTC), unixTimeStampOf(value)); + assertEquals(value.toEpochSecond(ZoneOffset.UTC), + eval(unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value)))).longValue()); } private static Stream getInstantSamples() { - return getDateTimeSamples().map(v -> Arguments.of(((LocalDateTime)v.get()[0]).toInstant(ZoneOffset.UTC))); + return getDateTimeSamples() + .map(v -> Arguments.of(((LocalDateTime)v.get()[0]).toInstant(ZoneOffset.UTC))); } + /** + * Check processing valid values of type Instant. + * @param value a value + */ @ParameterizedTest @MethodSource("getInstantSamples") - public void checkOfDateTime(Instant instant) { - assertEquals(instant.getEpochSecond(), unixTimeStampOf(instant)); - assertEquals(instant.getEpochSecond(), eval(unixTimeStampOf(DSL.literal(new ExprTimestampValue(instant)))).longValue()); + public void checkOfInstant(Instant value) { + assertEquals(value.getEpochSecond(), unixTimeStampOf(value)); + assertEquals(value.getEpochSecond(), + eval(unixTimeStampOf(DSL.literal(new ExprTimestampValue(value)))).longValue()); } // formats: YYMMDD, YYMMDDhhmmss[.uuuuuu], YYYYMMDD, or YYYYMMDDhhmmss[.uuuuuu] // use BigDecimal, because double can't fit such big values private static Stream getDoubleSamples() { return Stream.of( - Arguments.of(new BigDecimal("840101.")), - Arguments.of(new BigDecimal("840101112233.")), - Arguments.of(new BigDecimal("840101112233.123456")), - Arguments.of(new BigDecimal("19840101.")), - Arguments.of(new BigDecimal("19840101000000.")), - Arguments.of(new BigDecimal("19840101112233.")), - Arguments.of(new BigDecimal("19840101112233.123456")) + Arguments.of(840101d, LocalDateTime.of(1984, 1, 1, 0, 0, 0)), + Arguments.of(840101112233d, LocalDateTime.of(1984, 1, 1, 11,22,33)), + Arguments.of(840101112233.123456, LocalDateTime.of(1984, 1, 1, 11, 22, 33, 123456000)), + Arguments.of(19840101d, LocalDateTime.of(1984, 1, 1, 0, 0, 0)), + Arguments.of(19840101000000d, LocalDateTime.of(1984, 1, 1, 0, 0, 0)), + Arguments.of(19840101112233d, LocalDateTime.of(1984, 1, 1, 11,22,33)), + Arguments.of(19840101112233.123456, LocalDateTime.of(1984, 1, 1, 11, 22, 33, 123456000)) ); } + /** + * Check processing valid Double values. + * @param valueAsDouble a value + * @param valueAsLDT the value as LocalDateTime + */ @ParameterizedTest @MethodSource("getDoubleSamples") - public void checkOfDoubleFormats(BigDecimal value) { - LocalDateTime valueDt = LocalDateTime.MIN; - var format = new DecimalFormat("0.#"); - format.setMinimumFractionDigits(0); - format.setMaximumFractionDigits(6); - var valueStr = format.format(value); - switch (valueStr.length()) { - case 6: valueStr = "19" + valueStr; - case 8: valueStr += "000000"; - case 14: valueDt = LocalDateTime.parse(valueStr, DateTimeFormatter.ofPattern("yyyyMMddHHmmss")); break; - case 12: - case 19: valueStr = "19" + valueStr; - case 21: valueDt = LocalDateTime.parse(valueStr, DateTimeFormatter.ofPattern("yyyyMMddHHmmss[.SSSSSS]")); break; - } - assertEquals(valueDt.toEpochSecond(ZoneOffset.UTC), unixTimeStampOf(value.doubleValue()), 1d, format.format(value)); - assertEquals(valueDt.toEpochSecond(ZoneOffset.UTC), eval(unixTimeStampOf(DSL.literal(new ExprDoubleValue(value.doubleValue())))).longValue(), 1d, format.format(value)); + public void checkOfDoubleFormats(Double valueAsDouble, LocalDateTime valueAsLDT) { + var valueAsStr = new DecimalFormat("0.#").format(valueAsDouble); + assertEquals(valueAsLDT.toEpochSecond(ZoneOffset.UTC), + unixTimeStampOf(valueAsDouble), 1d, valueAsStr); + assertEquals(valueAsLDT.toEpochSecond(ZoneOffset.UTC), + eval(unixTimeStampOf(DSL.literal(new ExprDoubleValue(valueAsDouble)))).longValue(), + 1d, valueAsStr); } @Test @@ -257,9 +267,15 @@ private static Stream getInvalidDoubleSamples() { ); } + /** + * Check processing invalid Double values. + * @param value a value + */ @ParameterizedTest @MethodSource("getInvalidDoubleSamples") public void checkInvalidDoubleCausesNull(Double value) { - assertEquals(ExprNullValue.of(), unixTimeStampOf(DSL.literal(new ExprDoubleValue(value))).valueOf(null), new DecimalFormat("0.#").format(value)); + assertEquals(ExprNullValue.of(), + unixTimeStampOf(DSL.literal(new ExprDoubleValue(value))).valueOf(null), + new DecimalFormat("0.#").format(value)); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java index 3c05ca1f5f..0aa88689d5 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java @@ -6,14 +6,14 @@ package org.opensearch.sql.expression.datetime; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; -import java.time.temporal.ChronoUnit; import java.util.List; import java.util.stream.Stream; - import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -32,8 +32,6 @@ import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.expression.function.FunctionSignature; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class UnixTwoWayConversionTest { Environment env; @@ -93,7 +91,8 @@ private ExprValue eval(Expression expression) { @Test public void checkConvertNow() { assertEquals(LocalDateTime.now(ZoneId.of("UTC")).withNano(0), fromUnixTime(unixTimeStamp())); - assertEquals(LocalDateTime.now(ZoneId.of("UTC")).withNano(0), eval(fromUnixTime(unixTimeStampExpr())).datetimeValue()); + assertEquals(LocalDateTime.now(ZoneId.of("UTC")).withNano(0), + eval(fromUnixTime(unixTimeStampExpr())).datetimeValue()); } private static Stream getDoubleSamples() { @@ -106,14 +105,21 @@ private static Stream getDoubleSamples() { ); } + /** + * Test converting valid Double values EpochTime -> DateTime -> EpochTime. + * @param value a value + */ @ParameterizedTest @MethodSource("getDoubleSamples") public void convertEpoch2DateTime2Epoch(Double value) { assertEquals(value, unixTimeStampOf(fromUnixTime(value))); - assertEquals(value, eval(unixTimeStampOf(fromUnixTime(DSL.literal(new ExprDoubleValue(value))))).doubleValue()); + assertEquals(value, + eval(unixTimeStampOf(fromUnixTime(DSL.literal(new ExprDoubleValue(value))))).doubleValue()); assertEquals(Math.round(value) + 0d, unixTimeStampOf(fromUnixTime(Math.round(value)))); - assertEquals(Math.round(value) + 0d, eval(unixTimeStampOf(fromUnixTime(DSL.literal(new ExprLongValue(Math.round(value)))))).doubleValue()); + assertEquals(Math.round(value) + 0d, + eval(unixTimeStampOf(fromUnixTime(DSL.literal(new ExprLongValue(Math.round(value)))))) + .doubleValue()); } private static Stream getDateTimeSamples() { @@ -127,10 +133,16 @@ private static Stream getDateTimeSamples() { ); } + /** + * Test converting valid values DateTime -> EpochTime -> DateTime. + * @param value a value + */ @ParameterizedTest @MethodSource("getDateTimeSamples") public void convertDateTime2Epoch2DateTime(LocalDateTime value) { assertEquals(value, fromUnixTime(unixTimeStampOf(value))); - assertEquals(value, eval(fromUnixTime(unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))))).datetimeValue()); + assertEquals(value, + eval(fromUnixTime(unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))))) + .datetimeValue()); } } From ab766f2134d9e82fc8aaef629c80b00a58765154 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 8 Sep 2022 11:15:32 -0700 Subject: [PATCH 03/13] Optimize `FROM_UNIXTIME` signature. Signed-off-by: Yury-Fridlyand --- .../opensearch/sql/expression/datetime/DateTimeFunction.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index f39cc092ab..ca52313478 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -256,9 +256,7 @@ private FunctionResolver from_days() { private FunctionResolver from_unixtime() { return define(BuiltinFunctionName.FROM_UNIXTIME.getName(), - impl(nullMissingHandling(DateTimeFunction::exprFromUnixTime), DATETIME, LONG), impl(nullMissingHandling(DateTimeFunction::exprFromUnixTime), DATETIME, DOUBLE), - impl(nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat), STRING, LONG, STRING), impl(nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat), STRING, DOUBLE, STRING)); } From 6872cf0bb515dea74d89121ee128d7c3c5fbdaa1 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 8 Sep 2022 11:15:47 -0700 Subject: [PATCH 04/13] Add doctests. Signed-off-by: Yury-Fridlyand --- docs/user/dql/functions.rst | 53 ++++++++++++++++++++++++++++ docs/user/ppl/functions/datetime.rst | 52 +++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 736b1f148e..f05381cdf5 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1196,6 +1196,34 @@ Example:: +---------------------+ +FROM_UNIXTIME +------------- + +Description +>>>>>>>>>>> + +Usage: Returns a representation of the argument given as a datetime or character string value. Perform reverse conversion for `UNIX_TIMESTAMP`_ function. +If second argument is provided, it is used to format the result in the same way as the format string used for the `DATE_FORMAT`_ function. + +Argument type: DOUBLE, STRING + +Return type map: + +DOUBLE -> DATETIME + +DOUBLE, STRING -> STRING + +Example:: + + os> select FROM_UNIXTIME(1220249547), FROM_UNIXTIME(1220249547, '%T') + fetched rows / total rows = 1/1 + +-----------------------------+-----------------------------------+ + | FROM_UNIXTIME(1220249547) | FROM_UNIXTIME(1220249547, '%T') | + |-----------------------------+-----------------------------------| + | 2008-09-01 06:12:27 | 06:12:27 | + +-----------------------------+-----------------------------------+ + + HOUR ---- @@ -1557,6 +1585,31 @@ Example:: +------------------------------+ +UNIX_TIMESTAMP +-------------- + +Description +>>>>>>>>>>> + +Usage: Converts given argument to Unix time (seconds since Epoch - very beginning of year 1970). If no argument given, it current Unix time. +The date argument may be a DATE, DATETIME, or TIMESTAMP string, or a number in YYMMDD, YYMMDDhhmmss, YYYYMMDD, or YYYYMMDDhhmmss format. If the argument includes a time part, it may optionally include a fractional seconds part. +You can use `FROM_UNIXTIME`_ to do reverse conversion. + +Argument type: /DOUBLE/DATE/DATETIME/TIMESTAMP + +Return type: DOUBLE + +Example:: + + os> select UNIX_TIMESTAMP(20771122143845), UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) + fetched rows / total rows = 1/1 + +----------------------------------+----------------------------------------------------+ + | UNIX_TIMESTAMP(20771122143845) | UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | + |----------------------------------+----------------------------------------------------| + | 3404817525.0 | 848077542.0 | + +----------------------------------+----------------------------------------------------+ + + WEEK ---- diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 3680dc2272..fb3a1428f7 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -365,6 +365,33 @@ Example:: +---------------------+ +FROM_UNIXTIME +------------- + +Description +>>>>>>>>>>> + +Usage: Returns a representation of the argument given as a datetime or character string value. Perform reverse conversion for `UNIX_TIMESTAMP`_ function. +If second argument is provided, it is used to format the result in the same way as the format string used for the `DATE_FORMAT`_ function. + +Argument type: DOUBLE, STRING + +Return type map: + +DOUBLE -> DATETIME + +DOUBLE, STRING -> STRING + +Example:: + + os> source=people | eval `FROM_UNIXTIME(1220249547)` = FROM_UNIXTIME(1220249547), `FROM_UNIXTIME(1220249547, '%T')` = FROM_UNIXTIME(1220249547, '%T') | fields `FROM_UNIXTIME(1220249547)`, `FROM_UNIXTIME(1220249547, '%T')` + fetched rows / total rows = 1/1 + +-----------------------------+-----------------------------------+ + | FROM_UNIXTIME(1220249547) | FROM_UNIXTIME(1220249547, '%T') | + |-----------------------------+-----------------------------------| + | 2008-09-01 06:12:27 | 06:12:27 | + +-----------------------------+-----------------------------------+ + HOUR ---- @@ -726,6 +753,31 @@ Example:: +-------------------------------+ +UNIX_TIMESTAMP +-------------- + +Description +>>>>>>>>>>> + +Usage: Converts given argument to Unix time (seconds since Epoch - very beginning of year 1970). If no argument given, it current Unix time. +The date argument may be a DATE, DATETIME, or TIMESTAMP string, or a number in YYMMDD, YYMMDDhhmmss, YYYYMMDD, or YYYYMMDDhhmmss format. If the argument includes a time part, it may optionally include a fractional seconds part. +You can use `FROM_UNIXTIME`_ to do reverse conversion. + +Argument type: /DOUBLE/DATE/DATETIME/TIMESTAMP + +Return type: DOUBLE + +Example:: + + os> source=people | eval `UNIX_TIMESTAMP(20771122143845)` = UNIX_TIMESTAMP(20771122143845), `UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42'))` = UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | fields `UNIX_TIMESTAMP(20771122143845)`, `UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42'))` + fetched rows / total rows = 1/1 + +----------------------------------+----------------------------------------------------+ + | UNIX_TIMESTAMP(20771122143845) | UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | + |----------------------------------+----------------------------------------------------| + | 3404817525.0 | 848077542.0 | + +----------------------------------+----------------------------------------------------+ + + WEEK ---- From 73ca29e89216fa352fd2f561042ad6f0b412b4ef Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 8 Sep 2022 12:43:26 -0700 Subject: [PATCH 05/13] Minor clean-up. Signed-off-by: Yury-Fridlyand --- .../sql/expression/datetime/DateTimeFunction.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index ca52313478..767409bfe7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -6,11 +6,6 @@ package org.opensearch.sql.expression.datetime; -import static java.time.temporal.ChronoField.DAY_OF_MONTH; -import static java.time.temporal.ChronoField.HOUR_OF_DAY; -import static java.time.temporal.ChronoField.MINUTE_OF_HOUR; -import static java.time.temporal.ChronoField.MONTH_OF_YEAR; -import static java.time.temporal.ChronoField.SECOND_OF_MINUTE; import static java.time.temporal.ChronoField.YEAR; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; @@ -25,10 +20,7 @@ import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; -import java.math.BigDecimal; -import java.math.RoundingMode; import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; @@ -39,11 +31,8 @@ import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; import java.time.format.ResolverStyle; -import java.time.format.SignStyle; import java.time.format.TextStyle; -import java.time.temporal.ChronoField; import java.util.Locale; -import java.util.TimeZone; import java.util.concurrent.TimeUnit; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprDateValue; @@ -574,10 +563,6 @@ private ExprValue exprFromUnixTime(ExprValue time) { } private LocalDateTime exprFromUnixTimeImpl(ExprValue time) { - if (LONG == time.type()) { - return LocalDateTime.ofInstant(Instant.ofEpochSecond(time.longValue()), - ZoneId.of("UTC")); - } return LocalDateTime.ofInstant( Instant.ofEpochSecond((long)Math.floor(time.doubleValue())), ZoneId.of("UTC")) From 65776eddb97591e739d4fb507d3501a4e8bf3780 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Fri, 9 Sep 2022 18:46:43 -0700 Subject: [PATCH 06/13] Add a constant instead of magic numbers. Signed-off-by: Yury-Fridlyand --- .../sql/expression/datetime/DateTimeFunction.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 767409bfe7..5e9611fb5f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -62,6 +62,10 @@ public class DateTimeFunction { // The number of days from year zero to year 1970. private static final Long DAYS_0000_TO_1970 = (146097 * 5L) - (30L * 365L + 7L); + // MySQL doesn't process any datetime/timestamp values which are greater than + // 32536771199.999999, or equivalent '3001-01-18 23:59:59.999999' UTC + private static final Double MYSQL_MAX_TIMESTAMP = 32536771200d; + /** * Register Date and Time Functions. * @@ -556,7 +560,7 @@ private ExprValue exprFromUnixTime(ExprValue time) { // effective maximum is 32536771199.999999, which returns '3001-01-18 23:59:59.999999' UTC. // Regardless of platform or version, a greater value for first argument than the effective // maximum returns 0. - if (32536771200d <= time.doubleValue()) { + if (MYSQL_MAX_TIMESTAMP <= time.doubleValue()) { return ExprNullValue.of(); } return new ExprDatetimeValue(exprFromUnixTimeImpl(time)); @@ -791,7 +795,7 @@ private ExprValue unixTimeStampOf(ExprValue value) { // According to MySQL returns 0 if year < 1970, don't return negative values as java does. return new ExprDoubleValue(0); } - if (res >= 32536771200d) { + if (res >= MYSQL_MAX_TIMESTAMP) { // Return 0 also for dates > '3001-01-19 03:14:07.999999' UTC (32536771199.999999 sec) return new ExprDoubleValue(0); } From dfd2c1ea860a39b6582ee4e2fe07d600cd7138df Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Mon, 12 Sep 2022 18:52:23 -0700 Subject: [PATCH 07/13] Update docs according to PR feedback. Co-authored-by: MaxKsyunz Signed-off-by: Yury-Fridlyand --- docs/user/dql/functions.rst | 48 +++++++++++++++++++--------- docs/user/ppl/functions/datetime.rst | 40 ++++++++++++++--------- 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index f05381cdf5..4057adfac3 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1204,6 +1204,7 @@ Description Usage: Returns a representation of the argument given as a datetime or character string value. Perform reverse conversion for `UNIX_TIMESTAMP`_ function. If second argument is provided, it is used to format the result in the same way as the format string used for the `DATE_FORMAT`_ function. +If timestamp is outside of range 1970-01-01 00:00:00 - 3001-01-18 23:59:59.999999 (0 to 32536771199.999999 epoch time), function returns NULL. Argument type: DOUBLE, STRING @@ -1213,15 +1214,23 @@ DOUBLE -> DATETIME DOUBLE, STRING -> STRING -Example:: +Examples:: + + os> select FROM_UNIXTIME(1220249547) + fetched rows / total rows = 1/1 + +-----------------------------+ + | FROM_UNIXTIME(1220249547) | + |-----------------------------| + | 2008-09-01 06:12:27 | + +-----------------------------+ - os> select FROM_UNIXTIME(1220249547), FROM_UNIXTIME(1220249547, '%T') + os> select FROM_UNIXTIME(1220249547, '%T') fetched rows / total rows = 1/1 - +-----------------------------+-----------------------------------+ - | FROM_UNIXTIME(1220249547) | FROM_UNIXTIME(1220249547, '%T') | - |-----------------------------+-----------------------------------| - | 2008-09-01 06:12:27 | 06:12:27 | - +-----------------------------+-----------------------------------+ + +-----------------------------------+ + | FROM_UNIXTIME(1220249547, '%T') | + |-----------------------------------| + | 06:12:27 | + +-----------------------------------+ HOUR @@ -1591,23 +1600,32 @@ UNIX_TIMESTAMP Description >>>>>>>>>>> -Usage: Converts given argument to Unix time (seconds since Epoch - very beginning of year 1970). If no argument given, it current Unix time. +Usage: Converts given argument to Unix time (seconds since January 1st, 1970 at 00:00:00 UTC). If no argument given, it returns current Unix time. The date argument may be a DATE, DATETIME, or TIMESTAMP string, or a number in YYMMDD, YYMMDDhhmmss, YYYYMMDD, or YYYYMMDDhhmmss format. If the argument includes a time part, it may optionally include a fractional seconds part. +If argument is in invalid format or outside of range 1970-01-01 00:00:00 - 3001-01-18 23:59:59.999999 (0 to 32536771199.999999 epoch time), function returns NULL. You can use `FROM_UNIXTIME`_ to do reverse conversion. Argument type: /DOUBLE/DATE/DATETIME/TIMESTAMP Return type: DOUBLE -Example:: +Examples:: + + os> select UNIX_TIMESTAMP(20771122143845) + fetched rows / total rows = 1/1 + +----------------------------------+ + | UNIX_TIMESTAMP(20771122143845) | + |----------------------------------| + | 3404817525.0 | + +----------------------------------+ - os> select UNIX_TIMESTAMP(20771122143845), UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) + os> select UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) fetched rows / total rows = 1/1 - +----------------------------------+----------------------------------------------------+ - | UNIX_TIMESTAMP(20771122143845) | UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | - |----------------------------------+----------------------------------------------------| - | 3404817525.0 | 848077542.0 | - +----------------------------------+----------------------------------------------------+ + +----------------------------------------------------+ + | UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | + |----------------------------------------------------| + | 848077542.0 | + +----------------------------------------------------+ WEEK diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index fb3a1428f7..c041df5437 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -373,7 +373,7 @@ Description Usage: Returns a representation of the argument given as a datetime or character string value. Perform reverse conversion for `UNIX_TIMESTAMP`_ function. If second argument is provided, it is used to format the result in the same way as the format string used for the `DATE_FORMAT`_ function. - +If timestamp is outside of range 1970-01-01 00:00:00 - 3001-01-18 23:59:59.999999 (0 to 32536771199.999999 epoch time), function returns NULL. Argument type: DOUBLE, STRING Return type map: @@ -382,15 +382,24 @@ DOUBLE -> DATETIME DOUBLE, STRING -> STRING -Example:: +Examples:: + + os> source=people | eval `FROM_UNIXTIME(1220249547)` = FROM_UNIXTIME(1220249547) | fields `FROM_UNIXTIME(1220249547)` + fetched rows / total rows = 1/1 + +-----------------------------+ + | FROM_UNIXTIME(1220249547) | + |-----------------------------| + | 2008-09-01 06:12:27 | + +-----------------------------+ - os> source=people | eval `FROM_UNIXTIME(1220249547)` = FROM_UNIXTIME(1220249547), `FROM_UNIXTIME(1220249547, '%T')` = FROM_UNIXTIME(1220249547, '%T') | fields `FROM_UNIXTIME(1220249547)`, `FROM_UNIXTIME(1220249547, '%T')` + os> source=people | eval `FROM_UNIXTIME(1220249547, '%T')` = FROM_UNIXTIME(1220249547, '%T') | fields `FROM_UNIXTIME(1220249547, '%T')` fetched rows / total rows = 1/1 - +-----------------------------+-----------------------------------+ - | FROM_UNIXTIME(1220249547) | FROM_UNIXTIME(1220249547, '%T') | - |-----------------------------+-----------------------------------| - | 2008-09-01 06:12:27 | 06:12:27 | - +-----------------------------+-----------------------------------+ + +-----------------------------------+ + | FROM_UNIXTIME(1220249547, '%T') | + |-----------------------------------| + | 06:12:27 | + +-----------------------------------+ + HOUR ---- @@ -759,8 +768,9 @@ UNIX_TIMESTAMP Description >>>>>>>>>>> -Usage: Converts given argument to Unix time (seconds since Epoch - very beginning of year 1970). If no argument given, it current Unix time. +Usage: Converts given argument to Unix time (seconds since Epoch - very beginning of year 1970). If no argument given, it returns the current Unix time. The date argument may be a DATE, DATETIME, or TIMESTAMP string, or a number in YYMMDD, YYMMDDhhmmss, YYYYMMDD, or YYYYMMDDhhmmss format. If the argument includes a time part, it may optionally include a fractional seconds part. +If argument is in invalid format or outside of range 1970-01-01 00:00:00 - 3001-01-18 23:59:59.999999 (0 to 32536771199.999999 epoch time), function returns NULL. You can use `FROM_UNIXTIME`_ to do reverse conversion. Argument type: /DOUBLE/DATE/DATETIME/TIMESTAMP @@ -769,13 +779,13 @@ Return type: DOUBLE Example:: - os> source=people | eval `UNIX_TIMESTAMP(20771122143845)` = UNIX_TIMESTAMP(20771122143845), `UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42'))` = UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | fields `UNIX_TIMESTAMP(20771122143845)`, `UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42'))` + os> source=people | eval `UNIX_TIMESTAMP(double)` = UNIX_TIMESTAMP(20771122143845), `UNIX_TIMESTAMP(timestamp)` = UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | fields `UNIX_TIMESTAMP(double)`, `UNIX_TIMESTAMP(timestamp)` fetched rows / total rows = 1/1 - +----------------------------------+----------------------------------------------------+ - | UNIX_TIMESTAMP(20771122143845) | UNIX_TIMESTAMP(TIMESTAMP('1996-11-15 17:05:42')) | - |----------------------------------+----------------------------------------------------| - | 3404817525.0 | 848077542.0 | - +----------------------------------+----------------------------------------------------+ + +--------------------------+-----------------------------+ + | UNIX_TIMESTAMP(double) | UNIX_TIMESTAMP(timestamp) | + |--------------------------+-----------------------------| + | 3404817525.0 | 848077542.0 | + +--------------------------+-----------------------------+ WEEK From 541c63c299629048b16713d5c63515c9b7e3a252 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Mon, 12 Sep 2022 19:05:18 -0700 Subject: [PATCH 08/13] Update tests according to PR feedback. Signed-off-by: Yury-Fridlyand --- .../expression/datetime/FromUnixTimeTest.java | 25 +++++++++++++++++-- .../datetime/UnixTimeStampTest.java | 4 ++- .../datetime/UnixTwoWayConversionTest.java | 3 ++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java index 6434916f12..3177d152fd 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java @@ -25,13 +25,14 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ExpressionTestBase; import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.expression.function.FunctionSignature; -public class FromUnixTimeTest { +public class FromUnixTimeTest extends ExpressionTestBase { Environment env; @@ -174,25 +175,45 @@ public void checkOfDoubleWithFormat(Double value, String format, String expected } @Test - public void checkInvalidValues() { + public void checkInvalidFormat() { assertEquals(new ExprStringValue("q"), fromUnixTime(DSL.literal(0L), DSL.literal("%q")).valueOf(null)); assertEquals(new ExprStringValue(""), fromUnixTime(DSL.literal(0L), DSL.literal("")).valueOf(null)); + } + + @Test + public void checkValueOutsideOfTheRangeWithoutFormat() { assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1L)).valueOf(null)); assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1.5d)).valueOf(null)); assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200L)).valueOf(null)); assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200d)).valueOf(null)); + } + + @Test + public void checkInsideTheRangeWithoutFormat() { assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199L)).valueOf(null)); assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199d)).valueOf(null)); + } + + @Test + public void checkValueOutsideOfTheRangeWithFormat() { assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200L), DSL.literal("%d")).valueOf(null)); assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200d), DSL.literal("%d")).valueOf(null)); + } + + @Test + public void checkInsideTheRangeWithFormat() { assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199L), DSL.literal("%d")).valueOf(null)); assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199d), DSL.literal("%d")).valueOf(null)); + } + + @Test + public void checkNullOrNegativeValues() { assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of())).valueOf(null)); assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1L), DSL.literal("%d")).valueOf(null)); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java index 980fa624db..15f7c67575 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java @@ -263,7 +263,9 @@ private static Stream getInvalidDoubleSamples() { //misc Arguments.of(0d), Arguments.of(-1d), - Arguments.of(42d) + Arguments.of(42d), + //too many digits + Arguments.of(199902201122330d) ); } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java index 0aa88689d5..6922bfba17 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java @@ -26,13 +26,14 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ExpressionTestBase; import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.expression.function.FunctionSignature; -public class UnixTwoWayConversionTest { +public class UnixTwoWayConversionTest extends ExpressionTestBase { Environment env; From faf03721e75f46dc6a0486f8ad280ab4591532cf Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Tue, 13 Sep 2022 10:51:27 -0700 Subject: [PATCH 09/13] Reorder functions in ANTLR syntax file. Signed-off-by: Yury-Fridlyand --- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 7 ++++--- sql/src/main/antlr/OpenSearchSQLParser.g4 | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 0e28f5aa44..bda0752a99 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -373,9 +373,10 @@ trigonometricFunctionName ; dateAndTimeFunctionBase - : ADDDATE | DATE | DATE_ADD | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK | DAYOFYEAR | FROM_DAYS - | HOUR | MICROSECOND | MINUTE | MONTH | MONTHNAME | QUARTER | SECOND | SUBDATE | TIME | TIME_TO_SEC - | TIMESTAMP | TO_DAYS | YEAR | WEEK | DATE_FORMAT | MAKETIME | MAKEDATE | FROM_UNIXTIME | UNIX_TIMESTAMP + : ADDDATE | DATE | DATE_ADD | DATE_FORMAT | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK + | DAYOFYEAR | FROM_DAYS | FROM_UNIXTIME | HOUR | MAKEDATE | MAKETIME | MICROSECOND | MINUTE + | MONTH | MONTHNAME | QUARTER | SECOND | SUBDATE | TIME | TIME_TO_SEC | TIMESTAMP | TO_DAYS + | UNIX_TIMESTAMP | WEEK | YEAR ; /** condition function return boolean value */ diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 4a9e79e40b..0a5bfdd250 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -383,9 +383,10 @@ trigonometricFunctionName ; dateTimeFunctionName - : ADDDATE | DATE | DATE_ADD | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK | DAYOFYEAR | FROM_DAYS - | HOUR | MICROSECOND | MINUTE | MONTH | MONTHNAME | QUARTER | SECOND | SUBDATE | TIME | TIME_TO_SEC - | TIMESTAMP | TO_DAYS | YEAR | WEEK | DATE_FORMAT | MAKETIME | MAKEDATE | FROM_UNIXTIME | UNIX_TIMESTAMP + : ADDDATE | DATE | DATE_ADD | DATE_FORMAT | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK + | DAYOFYEAR | FROM_DAYS | FROM_UNIXTIME | HOUR | MAKEDATE | MAKETIME | MICROSECOND | MINUTE + | MONTH | MONTHNAME | QUARTER | SECOND | SUBDATE | TIME | TIME_TO_SEC | TIMESTAMP | TO_DAYS + | UNIX_TIMESTAMP | WEEK | YEAR ; textFunctionName From 36cb2269019541b14137646c7567152f407008f2 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 14 Sep 2022 09:08:44 -0700 Subject: [PATCH 10/13] Move test methods to common class `DateTimeTestBase`. Signed-off-by: Yury-Fridlyand --- .../expression/datetime/DateTimeTestBase.java | 1 + .../expression/datetime/FromUnixTimeTest.java | 47 +--------------- .../sql/expression/datetime/MakeDateTest.java | 40 +------------ .../sql/expression/datetime/MakeTimeTest.java | 41 +------------- .../datetime/UnixTimeStampTest.java | 50 +---------------- .../datetime/UnixTwoWayConversionTest.java | 56 +------------------ 6 files changed, 6 insertions(+), 229 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java new file mode 100644 index 0000000000..03cfdae00f --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java @@ -0,0 +1 @@ +/* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.expression.datetime; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.util.List; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDatetimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionTestBase; import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.expression.function.FunctionSignature; @ExtendWith(MockitoExtension.class) public class DateTimeTestBase extends ExpressionTestBase { @Mock protected Environment env; @Mock protected Expression nullRef; @Mock protected Expression missingRef; protected FunctionExpression maketime(Expression hour, Expression minute, Expression second) { var repo = new ExpressionConfig().functionRepository(); var func = repo.resolve(new FunctionSignature(new FunctionName("maketime"), List.of(DOUBLE, DOUBLE, DOUBLE))); return (FunctionExpression)func.apply(List.of(hour, minute, second)); } protected LocalTime maketime(Double hour, Double minute, Double second) { return maketime(DSL.literal(hour), DSL.literal(minute), DSL.literal(second)) .valueOf(null).timeValue(); } protected FunctionExpression makedate(Expression year, Expression dayOfYear) { var repo = new ExpressionConfig().functionRepository(); var func = repo.resolve(new FunctionSignature(new FunctionName("makedate"), List.of(DOUBLE, DOUBLE))); return (FunctionExpression)func.apply(List.of(year, dayOfYear)); } protected LocalDate makedate(Double year, Double dayOfYear) { return makedate(DSL.literal(year), DSL.literal(dayOfYear)).valueOf(null).dateValue(); } protected FunctionExpression unixTimeStampExpr() { var repo = new ExpressionConfig().functionRepository(); var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), List.of())); return (FunctionExpression)func.apply(List.of()); } protected Long unixTimeStamp() { return unixTimeStampExpr().valueOf(null).longValue(); } protected FunctionExpression unixTimeStampOf(Expression value) { var repo = new ExpressionConfig().functionRepository(); var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), List.of(value.type()))); return (FunctionExpression)func.apply(List.of(value)); } protected Double unixTimeStampOf(Double value) { return unixTimeStampOf(DSL.literal(value)).valueOf(null).doubleValue(); } protected Double unixTimeStampOf(LocalDate value) { return unixTimeStampOf(DSL.literal(new ExprDateValue(value))).valueOf(null).doubleValue(); } protected Double unixTimeStampOf(LocalDateTime value) { return unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))).valueOf(null).doubleValue(); } protected Double unixTimeStampOf(Instant value) { return unixTimeStampOf(DSL.literal(new ExprTimestampValue(value))).valueOf(null).doubleValue(); } protected FunctionExpression fromUnixTime(Expression value) { var repo = new ExpressionConfig().functionRepository(); var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), List.of(value.type()))); return (FunctionExpression)func.apply(List.of(value)); } protected FunctionExpression fromUnixTime(Expression value, Expression format) { var repo = new ExpressionConfig().functionRepository(); var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), List.of(value.type(), format.type()))); return (FunctionExpression)func.apply(List.of(value, format)); } protected LocalDateTime fromUnixTime(Long value) { return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); } protected LocalDateTime fromUnixTime(Double value) { return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); } protected String fromUnixTime(Long value, String format) { return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); } protected String fromUnixTime(Double value, String format) { return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); } protected ExprValue eval(Expression expression) { return expression.valueOf(env); } } \ No newline at end of file diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java index 3177d152fd..b6d04cbc97 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java @@ -12,7 +12,6 @@ import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; -import java.util.List; import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -22,53 +21,9 @@ import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.ExpressionTestBase; -import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.config.ExpressionConfig; -import org.opensearch.sql.expression.env.Environment; -import org.opensearch.sql.expression.function.FunctionName; -import org.opensearch.sql.expression.function.FunctionSignature; - -public class FromUnixTimeTest extends ExpressionTestBase { - - Environment env; - - private FunctionExpression fromUnixTime(Expression value) { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), - List.of(value.type()))); - return (FunctionExpression)func.apply(List.of(value)); - } - - private FunctionExpression fromUnixTime(Expression value, Expression format) { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), - List.of(value.type(), format.type()))); - return (FunctionExpression)func.apply(List.of(value, format)); - } - - private LocalDateTime fromUnixTime(Long value) { - return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); - } - - private LocalDateTime fromUnixTime(Double value) { - return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); - } - - private String fromUnixTime(Long value, String format) { - return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); - } - private String fromUnixTime(Double value, String format) { - return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); - } - - private ExprValue eval(Expression expression) { - return expression.valueOf(env); - } +public class FromUnixTimeTest extends DateTimeTestBase { private static Stream getLongSamples() { return Stream.of( diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/MakeDateTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/MakeDateTest.java index 497e73ea51..981468f31e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/MakeDateTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/MakeDateTest.java @@ -10,51 +10,17 @@ import static org.mockito.Mockito.when; import static org.opensearch.sql.data.model.ExprValueUtils.missingValue; import static org.opensearch.sql.data.model.ExprValueUtils.nullValue; -import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import java.time.LocalDate; import java.time.Year; -import java.util.List; import java.util.stream.Stream; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.ExpressionTestBase; -import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.config.ExpressionConfig; -import org.opensearch.sql.expression.env.Environment; -import org.opensearch.sql.expression.function.FunctionName; -import org.opensearch.sql.expression.function.FunctionSignature; - -@ExtendWith(MockitoExtension.class) -public class MakeDateTest extends ExpressionTestBase { - - @Mock - Environment env; - - @Mock - Expression nullRef; - - @Mock - Expression missingRef; - - private FunctionExpression makedate(Expression year, Expression dayOfYear) { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("makedate"), - List.of(DOUBLE, DOUBLE))); - return (FunctionExpression)func.apply(List.of(year, dayOfYear)); - } - private LocalDate makedate(Double year, Double dayOfYear) { - return makedate(DSL.literal(year), DSL.literal(dayOfYear)).valueOf(null).dateValue(); - } +public class MakeDateTest extends DateTimeTestBase { @Test public void checkEdgeCases() { @@ -167,8 +133,4 @@ private LocalDate getReferenceValue(double year, double dayOfYear) { } return LocalDate.ofYearDay(yearL, dayL); } - - private ExprValue eval(Expression expression) { - return expression.valueOf(env); - } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/MakeTimeTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/MakeTimeTest.java index 8269f74ccc..7dd78ff845 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/MakeTimeTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/MakeTimeTest.java @@ -11,53 +11,18 @@ import static org.mockito.Mockito.when; import static org.opensearch.sql.data.model.ExprValueUtils.missingValue; import static org.opensearch.sql.data.model.ExprValueUtils.nullValue; -import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import java.time.Duration; import java.time.LocalTime; import java.time.format.DateTimeParseException; -import java.util.List; import java.util.stream.Stream; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.ExpressionTestBase; -import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.config.ExpressionConfig; -import org.opensearch.sql.expression.env.Environment; -import org.opensearch.sql.expression.function.FunctionName; -import org.opensearch.sql.expression.function.FunctionSignature; - -@ExtendWith(MockitoExtension.class) -public class MakeTimeTest extends ExpressionTestBase { - - @Mock - Environment env; - - @Mock - Expression nullRef; - - @Mock - Expression missingRef; - - private FunctionExpression maketime(Expression hour, Expression minute, Expression second) { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("maketime"), - List.of(DOUBLE, DOUBLE, DOUBLE))); - return (FunctionExpression)func.apply(List.of(hour, minute, second)); - } - private LocalTime maketime(Double hour, Double minute, Double second) { - return maketime(DSL.literal(hour), DSL.literal(minute), DSL.literal(second)) - .valueOf(null).timeValue(); - } +public class MakeTimeTest extends DateTimeTestBase { @Test public void checkEdgeCases() { @@ -165,8 +130,4 @@ public void checkRandomValues(double hour, double minute, double second) { assertEquals(0, delta, 1, String.format("hour = %f, minute = %f, second = %f", hour, minute, second)); } - - private ExprValue eval(Expression expression) { - return expression.valueOf(env); - } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java index 15f7c67575..437e195f3e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java @@ -14,7 +14,6 @@ import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.temporal.ChronoField; -import java.util.List; import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -25,56 +24,9 @@ import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprTimestampValue; -import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.ExpressionTestBase; -import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.config.ExpressionConfig; -import org.opensearch.sql.expression.env.Environment; -import org.opensearch.sql.expression.function.FunctionName; -import org.opensearch.sql.expression.function.FunctionSignature; -public class UnixTimeStampTest extends ExpressionTestBase { - - Environment env; - - private FunctionExpression unixTimeStampExpr() { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), List.of())); - return (FunctionExpression)func.apply(List.of()); - } - - private Long unixTimeStamp() { - return unixTimeStampExpr().valueOf(null).longValue(); - } - - private FunctionExpression unixTimeStampOf(Expression value) { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), - List.of(value.type()))); - return (FunctionExpression)func.apply(List.of(value)); - } - - private Double unixTimeStampOf(Double value) { - return unixTimeStampOf(DSL.literal(value)).valueOf(null).doubleValue(); - } - - private Double unixTimeStampOf(LocalDate value) { - return unixTimeStampOf(DSL.literal(new ExprDateValue(value))).valueOf(null).doubleValue(); - } - - private Double unixTimeStampOf(LocalDateTime value) { - return unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))).valueOf(null).doubleValue(); - } - - private Double unixTimeStampOf(Instant value) { - return unixTimeStampOf(DSL.literal(new ExprTimestampValue(value))).valueOf(null).doubleValue(); - } - - private ExprValue eval(Expression expression) { - return expression.valueOf(env); - } +public class UnixTimeStampTest extends DateTimeTestBase { @Test public void checkNoArgs() { diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java index 6922bfba17..109a5e619a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTwoWayConversionTest.java @@ -33,61 +33,7 @@ import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.expression.function.FunctionSignature; -public class UnixTwoWayConversionTest extends ExpressionTestBase { - - Environment env; - - private FunctionExpression unixTimeStampExpr() { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), List.of())); - return (FunctionExpression)func.apply(List.of()); - } - - private Long unixTimeStamp() { - return unixTimeStampExpr().valueOf(null).longValue(); - } - - private FunctionExpression unixTimeStampOf(Expression value) { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), - List.of(value.type()))); - return (FunctionExpression)func.apply(List.of(value)); - } - - private Double unixTimeStampOf(Double value) { - return unixTimeStampOf(DSL.literal(value)).valueOf(null).doubleValue(); - } - - private Double unixTimeStampOf(LocalDate value) { - return unixTimeStampOf(DSL.literal(new ExprDateValue(value))).valueOf(null).doubleValue(); - } - - private Double unixTimeStampOf(LocalDateTime value) { - return unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))).valueOf(null).doubleValue(); - } - - private Double unixTimeStampOf(Instant value) { - return unixTimeStampOf(DSL.literal(new ExprTimestampValue(value))).valueOf(null).doubleValue(); - } - - private FunctionExpression fromUnixTime(Expression value) { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), - List.of(value.type()))); - return (FunctionExpression)func.apply(List.of(value)); - } - - private LocalDateTime fromUnixTime(Long value) { - return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); - } - - private LocalDateTime fromUnixTime(Double value) { - return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); - } - - private ExprValue eval(Expression expression) { - return expression.valueOf(env); - } +public class UnixTwoWayConversionTest extends DateTimeTestBase { @Test public void checkConvertNow() { From ad1523ca3d5221af4c9cfe915357e08d15c044ea Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 14 Sep 2022 09:44:35 -0700 Subject: [PATCH 11/13] Fix EOL. Signed-off-by: Yury-Fridlyand --- .../expression/datetime/DateTimeTestBase.java | 133 +++++++++++++++++- 1 file changed, 132 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java index 03cfdae00f..db9c640197 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java @@ -1 +1,132 @@ -/* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.expression.datetime; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.util.List; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDatetimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionTestBase; import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.expression.function.FunctionSignature; @ExtendWith(MockitoExtension.class) public class DateTimeTestBase extends ExpressionTestBase { @Mock protected Environment env; @Mock protected Expression nullRef; @Mock protected Expression missingRef; protected FunctionExpression maketime(Expression hour, Expression minute, Expression second) { var repo = new ExpressionConfig().functionRepository(); var func = repo.resolve(new FunctionSignature(new FunctionName("maketime"), List.of(DOUBLE, DOUBLE, DOUBLE))); return (FunctionExpression)func.apply(List.of(hour, minute, second)); } protected LocalTime maketime(Double hour, Double minute, Double second) { return maketime(DSL.literal(hour), DSL.literal(minute), DSL.literal(second)) .valueOf(null).timeValue(); } protected FunctionExpression makedate(Expression year, Expression dayOfYear) { var repo = new ExpressionConfig().functionRepository(); var func = repo.resolve(new FunctionSignature(new FunctionName("makedate"), List.of(DOUBLE, DOUBLE))); return (FunctionExpression)func.apply(List.of(year, dayOfYear)); } protected LocalDate makedate(Double year, Double dayOfYear) { return makedate(DSL.literal(year), DSL.literal(dayOfYear)).valueOf(null).dateValue(); } protected FunctionExpression unixTimeStampExpr() { var repo = new ExpressionConfig().functionRepository(); var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), List.of())); return (FunctionExpression)func.apply(List.of()); } protected Long unixTimeStamp() { return unixTimeStampExpr().valueOf(null).longValue(); } protected FunctionExpression unixTimeStampOf(Expression value) { var repo = new ExpressionConfig().functionRepository(); var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), List.of(value.type()))); return (FunctionExpression)func.apply(List.of(value)); } protected Double unixTimeStampOf(Double value) { return unixTimeStampOf(DSL.literal(value)).valueOf(null).doubleValue(); } protected Double unixTimeStampOf(LocalDate value) { return unixTimeStampOf(DSL.literal(new ExprDateValue(value))).valueOf(null).doubleValue(); } protected Double unixTimeStampOf(LocalDateTime value) { return unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))).valueOf(null).doubleValue(); } protected Double unixTimeStampOf(Instant value) { return unixTimeStampOf(DSL.literal(new ExprTimestampValue(value))).valueOf(null).doubleValue(); } protected FunctionExpression fromUnixTime(Expression value) { var repo = new ExpressionConfig().functionRepository(); var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), List.of(value.type()))); return (FunctionExpression)func.apply(List.of(value)); } protected FunctionExpression fromUnixTime(Expression value, Expression format) { var repo = new ExpressionConfig().functionRepository(); var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), List.of(value.type(), format.type()))); return (FunctionExpression)func.apply(List.of(value, format)); } protected LocalDateTime fromUnixTime(Long value) { return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); } protected LocalDateTime fromUnixTime(Double value) { return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); } protected String fromUnixTime(Long value, String format) { return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); } protected String fromUnixTime(Double value, String format) { return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); } protected ExprValue eval(Expression expression) { return expression.valueOf(env); } } \ No newline at end of file +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.datetime; + +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.List; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprDateValue; +import org.opensearch.sql.data.model.ExprDatetimeValue; +import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ExpressionTestBase; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionSignature; + +@ExtendWith(MockitoExtension.class) +public class DateTimeTestBase extends ExpressionTestBase { + + @Mock + protected Environment env; + + @Mock + protected Expression nullRef; + + @Mock + protected Expression missingRef; + + protected FunctionExpression maketime(Expression hour, Expression minute, Expression second) { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("maketime"), + List.of(DOUBLE, DOUBLE, DOUBLE))); + return (FunctionExpression)func.apply(List.of(hour, minute, second)); + } + + protected LocalTime maketime(Double hour, Double minute, Double second) { + return maketime(DSL.literal(hour), DSL.literal(minute), DSL.literal(second)) + .valueOf(null).timeValue(); + } + + protected FunctionExpression makedate(Expression year, Expression dayOfYear) { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("makedate"), + List.of(DOUBLE, DOUBLE))); + return (FunctionExpression)func.apply(List.of(year, dayOfYear)); + } + + protected LocalDate makedate(Double year, Double dayOfYear) { + return makedate(DSL.literal(year), DSL.literal(dayOfYear)).valueOf(null).dateValue(); + } + + protected FunctionExpression unixTimeStampExpr() { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), List.of())); + return (FunctionExpression)func.apply(List.of()); + } + + protected Long unixTimeStamp() { + return unixTimeStampExpr().valueOf(null).longValue(); + } + + protected FunctionExpression unixTimeStampOf(Expression value) { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), + List.of(value.type()))); + return (FunctionExpression)func.apply(List.of(value)); + } + + protected Double unixTimeStampOf(Double value) { + return unixTimeStampOf(DSL.literal(value)).valueOf(null).doubleValue(); + } + + protected Double unixTimeStampOf(LocalDate value) { + return unixTimeStampOf(DSL.literal(new ExprDateValue(value))).valueOf(null).doubleValue(); + } + + protected Double unixTimeStampOf(LocalDateTime value) { + return unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))).valueOf(null).doubleValue(); + } + + protected Double unixTimeStampOf(Instant value) { + return unixTimeStampOf(DSL.literal(new ExprTimestampValue(value))).valueOf(null).doubleValue(); + } + + protected FunctionExpression fromUnixTime(Expression value) { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), + List.of(value.type()))); + return (FunctionExpression)func.apply(List.of(value)); + } + + protected FunctionExpression fromUnixTime(Expression value, Expression format) { + var repo = new ExpressionConfig().functionRepository(); + var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), + List.of(value.type(), format.type()))); + return (FunctionExpression)func.apply(List.of(value, format)); + } + + protected LocalDateTime fromUnixTime(Long value) { + return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); + } + + protected LocalDateTime fromUnixTime(Double value) { + return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); + } + + protected String fromUnixTime(Long value, String format) { + return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); + } + + protected String fromUnixTime(Double value, String format) { + return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); + } + + protected ExprValue eval(Expression expression) { + return expression.valueOf(env); + } +} From eca585b2a4240d4cd906eb734981a42315bcef02 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 14 Sep 2022 19:19:17 -0700 Subject: [PATCH 12/13] Update core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java Signed-off-by: Yury-Fridlyand Co-authored-by: Max Ksyunz --- .../sql/expression/datetime/DateTimeTestBase.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java index db9c640197..d25c6a71a1 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java @@ -39,10 +39,11 @@ public class DateTimeTestBase extends ExpressionTestBase { @Mock protected Expression missingRef; - + @Autowired + protected final BuiltinFunctionRepository functionRepository; + protected FunctionExpression maketime(Expression hour, Expression minute, Expression second) { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("maketime"), + var func = functionRepository.resolve(new FunctionSignature(new FunctionName("maketime"), List.of(DOUBLE, DOUBLE, DOUBLE))); return (FunctionExpression)func.apply(List.of(hour, minute, second)); } From 9d0c52fa55662944ac86084fe78baccb8767683a Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 14 Sep 2022 19:59:22 -0700 Subject: [PATCH 13/13] Fix using `@Autowired`. Signed-off-by: Yury-Fridlyand --- .../expression/datetime/DateTimeTestBase.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java index d25c6a71a1..2516dddfd3 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java @@ -23,10 +23,11 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionTestBase; import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.expression.function.FunctionSignature; +import org.springframework.beans.factory.annotation.Autowired; @ExtendWith(MockitoExtension.class) public class DateTimeTestBase extends ExpressionTestBase { @@ -39,8 +40,9 @@ public class DateTimeTestBase extends ExpressionTestBase { @Mock protected Expression missingRef; + @Autowired - protected final BuiltinFunctionRepository functionRepository; + protected BuiltinFunctionRepository functionRepository; protected FunctionExpression maketime(Expression hour, Expression minute, Expression second) { var func = functionRepository.resolve(new FunctionSignature(new FunctionName("maketime"), @@ -54,8 +56,7 @@ protected LocalTime maketime(Double hour, Double minute, Double second) { } protected FunctionExpression makedate(Expression year, Expression dayOfYear) { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("makedate"), + var func = functionRepository.resolve(new FunctionSignature(new FunctionName("makedate"), List.of(DOUBLE, DOUBLE))); return (FunctionExpression)func.apply(List.of(year, dayOfYear)); } @@ -65,8 +66,8 @@ protected LocalDate makedate(Double year, Double dayOfYear) { } protected FunctionExpression unixTimeStampExpr() { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), List.of())); + var func = functionRepository.resolve( + new FunctionSignature(new FunctionName("unix_timestamp"), List.of())); return (FunctionExpression)func.apply(List.of()); } @@ -75,8 +76,7 @@ protected Long unixTimeStamp() { } protected FunctionExpression unixTimeStampOf(Expression value) { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), + var func = functionRepository.resolve(new FunctionSignature(new FunctionName("unix_timestamp"), List.of(value.type()))); return (FunctionExpression)func.apply(List.of(value)); } @@ -98,15 +98,13 @@ protected Double unixTimeStampOf(Instant value) { } protected FunctionExpression fromUnixTime(Expression value) { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), + var func = functionRepository.resolve(new FunctionSignature(new FunctionName("from_unixtime"), List.of(value.type()))); return (FunctionExpression)func.apply(List.of(value)); } protected FunctionExpression fromUnixTime(Expression value, Expression format) { - var repo = new ExpressionConfig().functionRepository(); - var func = repo.resolve(new FunctionSignature(new FunctionName("from_unixtime"), + var func = functionRepository.resolve(new FunctionSignature(new FunctionName("from_unixtime"), List.of(value.type(), format.type()))); return (FunctionExpression)func.apply(List.of(value, format)); }