Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add typeof function. #867

Merged
merged 6 commits into from
Oct 20, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions core/src/main/java/org/opensearch/sql/expression/DSL.java
Original file line number Diff line number Diff line change
Expand Up @@ -662,6 +662,11 @@ public FunctionExpression castDatetime(Expression value) {
.compile(BuiltinFunctionName.CAST_TO_DATETIME.getName(), Arrays.asList(value));
}

public FunctionExpression typeof(Expression value) {
return (FunctionExpression) repository
.compile(BuiltinFunctionName.TYPEOF.getName(), Arrays.asList(value));
}

public FunctionExpression match(Expression... args) {
return compile(BuiltinFunctionName.MATCH, args);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.opensearch.sql.expression.operator.arthmetic.ArithmeticFunction;
import org.opensearch.sql.expression.operator.arthmetic.MathematicalFunction;
import org.opensearch.sql.expression.operator.convert.TypeCastOperator;
import org.opensearch.sql.expression.operator.convert.TypeOfOperator;
import org.opensearch.sql.expression.operator.predicate.BinaryPredicateOperator;
import org.opensearch.sql.expression.operator.predicate.UnaryPredicateOperator;
import org.opensearch.sql.expression.text.TextFunction;
Expand Down Expand Up @@ -45,6 +46,7 @@ public BuiltinFunctionRepository functionRepository() {
WindowFunctions.register(builtinFunctionRepository);
TextFunction.register(builtinFunctionRepository);
TypeCastOperator.register(builtinFunctionRepository);
TypeOfOperator.register(builtinFunctionRepository);
OpenSearchFunctions.register(builtinFunctionRepository);
return builtinFunctionRepository;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ public enum BuiltinFunctionName {
CAST_TO_TIME(FunctionName.of("cast_to_time")),
CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")),
CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")),
TYPEOF(FunctionName.of("typeof")),

/**
* Relevance Function.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.expression.operator.convert;

import static org.opensearch.sql.data.type.ExprCoreType.ARRAY;
import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN;
import static org.opensearch.sql.data.type.ExprCoreType.BYTE;
import static org.opensearch.sql.data.type.ExprCoreType.DATE;
import static org.opensearch.sql.data.type.ExprCoreType.DATETIME;
import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE;
import static org.opensearch.sql.data.type.ExprCoreType.FLOAT;
import static org.opensearch.sql.data.type.ExprCoreType.INTEGER;
import static org.opensearch.sql.data.type.ExprCoreType.INTERVAL;
import static org.opensearch.sql.data.type.ExprCoreType.LONG;
import static org.opensearch.sql.data.type.ExprCoreType.SHORT;
import static org.opensearch.sql.data.type.ExprCoreType.STRING;
import static org.opensearch.sql.data.type.ExprCoreType.STRUCT;
import static org.opensearch.sql.data.type.ExprCoreType.TIME;
import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP;
import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED;
import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN;
import static org.opensearch.sql.expression.function.FunctionDSL.impl;

import lombok.experimental.UtilityClass;
import org.opensearch.sql.data.model.ExprStringValue;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.expression.function.BuiltinFunctionName;
import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
import org.opensearch.sql.expression.function.DefaultFunctionResolver;
import org.opensearch.sql.expression.function.FunctionDSL;

@UtilityClass
public class TypeOfOperator {
dai-chen marked this conversation as resolved.
Show resolved Hide resolved
/**
* Register TypeOf Operator.
*/
public static void register(BuiltinFunctionRepository repository) {
repository.register(typeof());
}

// Auxiliary function useful for debugging
private static DefaultFunctionResolver typeof() {
return FunctionDSL.define(BuiltinFunctionName.TYPEOF.getName(),
impl(TypeOfOperator::exprTypeOf, STRING, ARRAY),
impl(TypeOfOperator::exprTypeOf, STRING, BOOLEAN),
impl(TypeOfOperator::exprTypeOf, STRING, BYTE),
impl(TypeOfOperator::exprTypeOf, STRING, DATE),
impl(TypeOfOperator::exprTypeOf, STRING, DATETIME),
impl(TypeOfOperator::exprTypeOf, STRING, DOUBLE),
impl(TypeOfOperator::exprTypeOf, STRING, FLOAT),
impl(TypeOfOperator::exprTypeOf, STRING, INTEGER),
impl(TypeOfOperator::exprTypeOf, STRING, INTERVAL),
impl(TypeOfOperator::exprTypeOf, STRING, LONG),
impl(TypeOfOperator::exprTypeOf, STRING, SHORT),
impl(TypeOfOperator::exprTypeOf, STRING, STRING),
impl(TypeOfOperator::exprTypeOf, STRING, STRUCT),
impl(TypeOfOperator::exprTypeOf, STRING, TIME),
impl(TypeOfOperator::exprTypeOf, STRING, TIMESTAMP),
impl(TypeOfOperator::exprTypeOf, STRING, UNDEFINED),
impl(TypeOfOperator::exprTypeOf, STRING, UNKNOWN)
);
}

private static ExprValue exprTypeOf(ExprValue input) {
return new ExprStringValue(input.type().typeName());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.expression.operator.convert;

import static org.junit.jupiter.api.Assertions.assertEquals;

import java.time.Duration;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.util.LinkedHashMap;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.opensearch.sql.data.model.AbstractExprValue;
import org.opensearch.sql.data.model.ExprBooleanValue;
import org.opensearch.sql.data.model.ExprByteValue;
import org.opensearch.sql.data.model.ExprCollectionValue;
import org.opensearch.sql.data.model.ExprDateValue;
import org.opensearch.sql.data.model.ExprDatetimeValue;
import org.opensearch.sql.data.model.ExprDoubleValue;
import org.opensearch.sql.data.model.ExprFloatValue;
import org.opensearch.sql.data.model.ExprIntegerValue;
import org.opensearch.sql.data.model.ExprIntervalValue;
import org.opensearch.sql.data.model.ExprLongValue;
import org.opensearch.sql.data.model.ExprMissingValue;
import org.opensearch.sql.data.model.ExprNullValue;
import org.opensearch.sql.data.model.ExprShortValue;
import org.opensearch.sql.data.model.ExprStringValue;
import org.opensearch.sql.data.model.ExprTimeValue;
import org.opensearch.sql.data.model.ExprTimestampValue;
import org.opensearch.sql.data.model.ExprTupleValue;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.data.type.ExprType;
import org.opensearch.sql.expression.DSL;
import org.opensearch.sql.expression.config.ExpressionConfig;

public class TypeOfOperatorTest {
private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository());

@Test
void typeof() {
dai-chen marked this conversation as resolved.
Show resolved Hide resolved
assertEquals("ARRAY", typeofGetValue(new ExprCollectionValue(List.of())));
assertEquals("BOOLEAN", typeofGetValue(ExprBooleanValue.of(false)));
assertEquals("BYTE", typeofGetValue(new ExprByteValue(0)));
assertEquals("DATE", typeofGetValue(new ExprDateValue(LocalDate.now())));
assertEquals("DATETIME", typeofGetValue(new ExprDatetimeValue(LocalDateTime.now())));
assertEquals("DOUBLE", typeofGetValue(new ExprDoubleValue(0)));
assertEquals("FLOAT", typeofGetValue(new ExprFloatValue(0)));
assertEquals("INTEGER", typeofGetValue(new ExprIntegerValue(0)));
assertEquals("INTERVAL", typeofGetValue(new ExprIntervalValue(Duration.ofDays(0))));
assertEquals("LONG", typeofGetValue(new ExprLongValue(0)));
assertEquals("SHORT", typeofGetValue(new ExprShortValue(0)));
assertEquals("STRING", typeofGetValue(new ExprStringValue("")));
assertEquals("STRUCT", typeofGetValue(new ExprTupleValue(new LinkedHashMap<>())));
assertEquals("TIME", typeofGetValue(new ExprTimeValue(LocalTime.now())));
assertEquals("TIMESTAMP", typeofGetValue(new ExprTimestampValue(Instant.now())));
assertEquals("UNDEFINED", typeofGetValue(ExprNullValue.of()));
assertEquals("UNDEFINED", typeofGetValue(ExprMissingValue.of()));
assertEquals("UNKNOWN", typeofGetValue(new AbstractExprValue() {
@Override
public int compare(ExprValue other) {
return 0;
}

@Override
public boolean equal(ExprValue other) {
return false;
}

@Override
public Object value() {
return null;
}

@Override
public ExprType type() {
return ExprCoreType.UNKNOWN;
}
}));
}

private String typeofGetValue(ExprValue input) {
return dsl.typeof(DSL.literal(input)).valueOf(null).stringValue();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,22 @@ public void castStatementInWhereClauseDatetimeCastTest() {
assertEquals(response.getJSONArray("schema").get(0).toString(), schema_result);
}

@Test
public void typeof() {
dai-chen marked this conversation as resolved.
Show resolved Hide resolved
JSONObject response = executeJdbcRequest("SELECT typeof('pewpew'), typeof(NULL), typeof(1.0),"
+ "typeof(12345), typeof(1234567891011), typeof(INTERVAL 2 DAY);");
verifyDataRows(response,
rows("STRING", "UNDEFINED", "DOUBLE", "INTEGER", "LONG", "INTERVAL"));

response = executeJdbcRequest("SELECT"
+ " typeof(CAST('1961-04-12 09:07:00' AS TIMESTAMP)),"
+ " typeof(CAST('09:07:00' AS TIME)),"
+ " typeof(CAST('1961-04-12' AS DATE)),"
+ " typeof(DATETIME('1961-04-12 09:07:00'))");
verifyDataRows(response,
rows("TIMESTAMP", "TIME", "DATE", "DATETIME"));
}

@Test
public void concat_ws_field_and_string() throws Exception {
//here is a bug,csv field with spa
Expand Down
22 changes: 22 additions & 0 deletions integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NUMERIC;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC;
import static org.opensearch.sql.util.MatcherUtils.rows;
import static org.opensearch.sql.util.MatcherUtils.schema;
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
import static org.opensearch.sql.util.MatcherUtils.verifySchema;

import java.io.IOException;
Expand Down Expand Up @@ -72,4 +74,24 @@ public void test_long_integer_data_type() throws IOException {
schema("long2", "long"));
}

@Test
public void typeof() throws IOException {
JSONObject response = executeQuery(String.format("source=%s | eval "
+ "`str` = typeof('pewpew'), `null` = typeof(1/0), `double` = typeof(1.0),"
+ "`int` = typeof(12345), `long` = typeof(1234567891011), `interval` = typeof(INTERVAL 2 DAY)"
+ " | fields `str`, `null`, `double`, `int`, `long`, `interval`",
TEST_INDEX_DATATYPE_NUMERIC));
verifyDataRows(response,
rows("STRING", "UNDEFINED", "DOUBLE", "INTEGER", "LONG", "INTERVAL"));

response = executeQuery(String.format("source=%s | eval "
+ "`timestamp` = typeof(CAST('1961-04-12 09:07:00' AS TIMESTAMP)),"
+ "`time` = typeof(CAST('09:07:00' AS TIME)),"
+ "`date` = typeof(CAST('1961-04-12' AS DATE)),"
+ "`datetime` = typeof(DATETIME('1961-04-12 09:07:00'))"
+ " | fields `timestamp`, `time`, `date`, `datetime`",
TEST_INDEX_DATATYPE_NUMERIC));
verifyDataRows(response,
rows("TIMESTAMP", "TIME", "DATE", "DATETIME"));
}
}
1 change: 1 addition & 0 deletions ppl/src/main/antlr/OpenSearchPPLLexer.g4
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,7 @@ ISNOTNULL: 'ISNOTNULL';
IFNULL: 'IFNULL';
NULLIF: 'NULLIF';
IF: 'IF';
TYPEOF: 'TYPEOF';

// RELEVANCE FUNCTIONS AND PARAMETERS
MATCH: 'MATCH';
Expand Down
1 change: 1 addition & 0 deletions ppl/src/main/antlr/OpenSearchPPLParser.g4
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,7 @@ constantFunctionName
conditionFunctionBase
: LIKE
| IF | ISNULL | ISNOTNULL | IFNULL | NULLIF
| TYPEOF
;

textFunctionBase
Expand Down
1 change: 1 addition & 0 deletions sql/src/main/antlr/OpenSearchSQLLexer.g4
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,7 @@ STATS: 'STATS';
TERM: 'TERM';
TERMS: 'TERMS';
TOPHITS: 'TOPHITS';
TYPEOF: 'TYPEOF';
WEEK_OF_YEAR: 'WEEK_OF_YEAR';
WILDCARDQUERY: 'WILDCARDQUERY';
WILDCARD_QUERY: 'WILDCARD_QUERY';
Expand Down
2 changes: 1 addition & 1 deletion sql/src/main/antlr/OpenSearchSQLParser.g4
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,7 @@ textFunctionName
;

flowControlFunctionName
: IF | IFNULL | NULLIF | ISNULL
: IF | IFNULL | NULLIF | ISNULL | TYPEOF
dai-chen marked this conversation as resolved.
Show resolved Hide resolved
;

singleFieldRelevanceFunctionName
Expand Down