Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add datetime functions FROM_UNIXTIME and UNIX_TIMESTAMP #114

Merged
merged 13 commits into from
Sep 15, 2022
Merged

Add datetime functions FROM_UNIXTIME and UNIX_TIMESTAMP #114

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
3dd12e0
Add implementation for `FROM_UNIXTIME` and `UNIX_TIMESTAMP` functions…
Yury-Fridlyand Sep 8, 2022
c954d07
Fix checkstyle.
Yury-Fridlyand Sep 8, 2022
ab766f2
Optimize `FROM_UNIXTIME` signature.
Yury-Fridlyand Sep 8, 2022
6872cf0
Add doctests.
Yury-Fridlyand Sep 8, 2022
73ca29e
Minor clean-up.
Yury-Fridlyand Sep 8, 2022
65776ed
Add a constant instead of magic numbers.
Yury-Fridlyand Sep 10, 2022
dfd2c1e
Update docs according to PR feedback.
Yury-Fridlyand Sep 13, 2022
541c63c
Update tests according to PR feedback.
Yury-Fridlyand Sep 13, 2022
faf0372
Reorder functions in ANTLR syntax file.
Yury-Fridlyand Sep 13, 2022
36cb226
Move test methods to common class `DateTimeTestBase`.
Yury-Fridlyand Sep 14, 2022
ad1523c
Fix EOL.
Yury-Fridlyand Sep 14, 2022
eca585b
Update core/src/test/java/org/opensearch/sql/expression/datetime/Date…
Yury-Fridlyand Sep 15, 2022
9d0c52f
Fix using `@Autowired`.
Yury-Fridlyand Sep 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 160 additions & 0 deletions core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

package org.opensearch.sql.expression.datetime;

import static java.time.temporal.ChronoField.YEAR;
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is there no YEAR field in ExprCoreType?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No.. Why it should be? Aren't you messing ChronoField and ExprCoreType? ExprCoreType is a enum of STRING, DOUBLE, etc

import static org.opensearch.sql.data.type.ExprCoreType.DATE;
import static org.opensearch.sql.data.type.ExprCoreType.DATETIME;
import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE;
Expand All @@ -19,22 +20,32 @@
import static org.opensearch.sql.expression.function.FunctionDSL.impl;
import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling;

import java.text.DecimalFormat;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
import java.time.format.ResolverStyle;
import java.time.format.TextStyle;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
import lombok.experimental.UtilityClass;
import org.opensearch.sql.data.model.ExprDateValue;
import org.opensearch.sql.data.model.ExprDatetimeValue;
import org.opensearch.sql.data.model.ExprDoubleValue;
import org.opensearch.sql.data.model.ExprIntegerValue;
import org.opensearch.sql.data.model.ExprLongValue;
import org.opensearch.sql.data.model.ExprNullValue;
import org.opensearch.sql.data.model.ExprStringValue;
import org.opensearch.sql.data.model.ExprTimeValue;
import org.opensearch.sql.data.model.ExprTimestampValue;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.expression.function.BuiltinFunctionName;
import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
import org.opensearch.sql.expression.function.FunctionName;
Expand All @@ -51,6 +62,10 @@ public class DateTimeFunction {
// The number of days from year zero to year 1970.
private static final Long DAYS_0000_TO_1970 = (146097 * 5L) - (30L * 365L + 7L);

// MySQL doesn't process any datetime/timestamp values which are greater than
// 32536771199.999999, or equivalent '3001-01-18 23:59:59.999999' UTC
private static final Double MYSQL_MAX_TIMESTAMP = 32536771200d;
MaxKsyunz marked this conversation as resolved.
Show resolved Hide resolved

/**
* Register Date and Time Functions.
*
Expand All @@ -67,6 +82,7 @@ public void register(BuiltinFunctionRepository repository) {
repository.register(dayOfWeek());
repository.register(dayOfYear());
repository.register(from_days());
repository.register(from_unixtime());
repository.register(hour());
repository.register(makedate());
repository.register(maketime());
Expand All @@ -82,6 +98,7 @@ public void register(BuiltinFunctionRepository repository) {
repository.register(timestamp());
repository.register(date_format());
repository.register(to_days());
repository.register(unix_timestamp());
repository.register(week());
repository.register(year());
}
Expand Down Expand Up @@ -230,6 +247,13 @@ private FunctionResolver from_days() {
impl(nullMissingHandling(DateTimeFunction::exprFromDays), DATE, LONG));
}

private FunctionResolver from_unixtime() {
return define(BuiltinFunctionName.FROM_UNIXTIME.getName(),
impl(nullMissingHandling(DateTimeFunction::exprFromUnixTime), DATETIME, DOUBLE),
impl(nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat),
STRING, DOUBLE, STRING));
}

/**
* HOUR(STRING/TIME/DATETIME/TIMESTAMP). return the hour value for time.
*/
Expand Down Expand Up @@ -378,6 +402,16 @@ private FunctionResolver to_days() {
impl(nullMissingHandling(DateTimeFunction::exprToDays), LONG, DATETIME));
}

private FunctionResolver unix_timestamp() {
return define(BuiltinFunctionName.UNIX_TIMESTAMP.getName(),
impl(DateTimeFunction::unixTimeStamp, LONG),
impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DATE),
impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DATETIME),
impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, TIMESTAMP),
impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DOUBLE)
);
}

/**
* WEEK(DATE[,mode]). return the week number for date.
*/
Expand Down Expand Up @@ -518,6 +552,35 @@ private ExprValue exprFromDays(ExprValue exprValue) {
return new ExprDateValue(LocalDate.ofEpochDay(exprValue.longValue() - DAYS_0000_TO_1970));
}

private ExprValue exprFromUnixTime(ExprValue time) {
if (0 > time.doubleValue()) {
return ExprNullValue.of();
}
// According to MySQL documentation:
// effective maximum is 32536771199.999999, which returns '3001-01-18 23:59:59.999999' UTC.
// Regardless of platform or version, a greater value for first argument than the effective
// maximum returns 0.
if (MYSQL_MAX_TIMESTAMP <= time.doubleValue()) {
return ExprNullValue.of();
}
return new ExprDatetimeValue(exprFromUnixTimeImpl(time));
}

private LocalDateTime exprFromUnixTimeImpl(ExprValue time) {
return LocalDateTime.ofInstant(
Instant.ofEpochSecond((long)Math.floor(time.doubleValue())),
ZoneId.of("UTC"))
acarbonetto marked this conversation as resolved.
Show resolved Hide resolved
.withNano((int)((time.doubleValue() % 1) * 1E9));
acarbonetto marked this conversation as resolved.
Show resolved Hide resolved
}

private ExprValue exprFromUnixTimeFormat(ExprValue time, ExprValue format) {
var value = exprFromUnixTime(time);
if (value.equals(ExprNullValue.of())) {
return ExprNullValue.of();
}
return DateTimeFormatterUtil.getFormattedDate(value, format);
}

/**
* Hour implementation for ExprValue.
*
Expand Down Expand Up @@ -719,6 +782,103 @@ private ExprValue exprWeek(ExprValue date, ExprValue mode) {
CalendarLookup.getWeekNumber(mode.integerValue(), date.dateValue()));
}

private ExprValue unixTimeStamp() {
return new ExprLongValue(Instant.now().getEpochSecond());
}

private ExprValue unixTimeStampOf(ExprValue value) {
var res = unixTimeStampOfImpl(value);
if (res == null) {
return ExprNullValue.of();
}
if (res < 0) {
// According to MySQL returns 0 if year < 1970, don't return negative values as java does.
return new ExprDoubleValue(0);
}
if (res >= MYSQL_MAX_TIMESTAMP) {
// Return 0 also for dates > '3001-01-19 03:14:07.999999' UTC (32536771199.999999 sec)
return new ExprDoubleValue(0);
}
return new ExprDoubleValue(res);
}

private Double unixTimeStampOfImpl(ExprValue value) {
// Also, according to MySQL documentation:
// The date argument may be a DATE, DATETIME, or TIMESTAMP ...
switch ((ExprCoreType)value.type()) {
case DATE: return value.dateValue().toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d;
case DATETIME: return value.datetimeValue().toEpochSecond(ZoneOffset.UTC)
+ value.datetimeValue().getNano() / 1E9;
case TIMESTAMP: return value.timestampValue().getEpochSecond()
+ value.timestampValue().getNano() / 1E9;
default:
// ... or a number in YYMMDD, YYMMDDhhmmss, YYYYMMDD, or YYYYMMDDhhmmss format.
// If the argument includes a time part, it may optionally include a fractional
// seconds part.

var dateFormatShortYear = new DateTimeFormatterBuilder()
.appendValueReduced(YEAR, 2, 2, 1970)
.appendPattern("MMdd")
.toFormatter()
.withResolverStyle(ResolverStyle.STRICT);

var dateFormatLongYear = new DateTimeFormatterBuilder()
.appendValue(YEAR, 4)
.appendPattern("MMdd")
.toFormatter()
.withResolverStyle(ResolverStyle.STRICT);

var dateTimeFormatShortYear = new DateTimeFormatterBuilder()
.appendValueReduced(YEAR, 2, 2, 1970)
.appendPattern("MMddHHmmss")
.toFormatter()
.withResolverStyle(ResolverStyle.STRICT);

var dateTimeFormatLongYear = new DateTimeFormatterBuilder()
.appendValue(YEAR,4)
.appendPattern("MMddHHmmss")
.toFormatter()
.withResolverStyle(ResolverStyle.STRICT);

var format = new DecimalFormat("0.#");
format.setMinimumFractionDigits(0);
format.setMaximumFractionDigits(6);
String input = format.format(value.doubleValue());
double fraction = 0;
if (input.contains(".")) {
// Keeping fraction second part and adding it to the result, don't parse it
// Because `toEpochSecond` returns only `long`
// input = 12345.6789 becomes input = 12345 and fraction = 0.6789
fraction = value.doubleValue() - Math.round(Math.ceil(value.doubleValue()));
input = input.substring(0, input.indexOf('.'));
}
try {
acarbonetto marked this conversation as resolved.
Show resolved Hide resolved
var res = LocalDateTime.parse(input, dateTimeFormatShortYear);
return res.toEpochSecond(ZoneOffset.UTC) + fraction;
} catch (DateTimeParseException ignored) {
// nothing to do, try another format
}
try {
var res = LocalDateTime.parse(input, dateTimeFormatLongYear);
return res.toEpochSecond(ZoneOffset.UTC) + fraction;
} catch (DateTimeParseException ignored) {
// nothing to do, try another format
}
try {
var res = LocalDate.parse(input, dateFormatShortYear);
return res.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d;
} catch (DateTimeParseException ignored) {
// nothing to do, try another format
}
try {
var res = LocalDate.parse(input, dateFormatLongYear);
return res.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d;
} catch (DateTimeParseException ignored) {
return null;
}
}
}

/**
* Week for date implementation for ExprValue.
* When mode is not specified default value mode 0 is used for default_week_format.
Expand Down
2 changes: 2 additions & 0 deletions core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ public enum BuiltinFunctionName {
DAYOFWEEK(FunctionName.of("dayofweek")),
DAYOFYEAR(FunctionName.of("dayofyear")),
FROM_DAYS(FunctionName.of("from_days")),
FROM_UNIXTIME(FunctionName.of("from_unixtime")),
HOUR(FunctionName.of("hour")),
MAKEDATE(FunctionName.of("makedate")),
MAKETIME(FunctionName.of("maketime")),
Expand All @@ -82,6 +83,7 @@ public enum BuiltinFunctionName {
TIMESTAMP(FunctionName.of("timestamp")),
DATE_FORMAT(FunctionName.of("date_format")),
TO_DAYS(FunctionName.of("to_days")),
UNIX_TIMESTAMP(FunctionName.of("unix_timestamp")),
WEEK(FunctionName.of("week")),
YEAR(FunctionName.of("year")),

Expand Down
12 changes: 0 additions & 12 deletions core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
import static org.opensearch.sql.data.model.ExprValueUtils.stringValue;
import static org.opensearch.sql.data.type.ExprCoreType.DATE;
import static org.opensearch.sql.data.type.ExprCoreType.DATETIME;
import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE;
import static org.opensearch.sql.data.type.ExprCoreType.INTEGER;
import static org.opensearch.sql.data.type.ExprCoreType.INTERVAL;
import static org.opensearch.sql.data.type.ExprCoreType.LONG;
Expand All @@ -25,15 +24,7 @@
import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP;

import com.google.common.collect.ImmutableList;
import java.time.Duration;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.Year;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.stream.IntStream;
import lombok.AllArgsConstructor;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand All @@ -52,10 +43,7 @@
import org.opensearch.sql.expression.Expression;
import org.opensearch.sql.expression.ExpressionTestBase;
import org.opensearch.sql.expression.FunctionExpression;
import org.opensearch.sql.expression.config.ExpressionConfig;
import org.opensearch.sql.expression.env.Environment;
import org.opensearch.sql.expression.function.FunctionName;
import org.opensearch.sql.expression.function.FunctionSignature;

@ExtendWith(MockitoExtension.class)
class DateTimeFunctionTest extends ExpressionTestBase {
Expand Down
Loading