forked from opensearch-project/sql
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adding Highlight support in PPL. Adding wildcard support for highligh…
…t in SQL and PPL. Signed-off-by: forestmvey <[email protected]>
- Loading branch information
1 parent
c2973a3
commit 4fa96cf
Showing
30 changed files
with
632 additions
and
190 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
45 changes: 45 additions & 0 deletions
45
core/src/main/java/org/opensearch/sql/ast/tree/Highlight.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.ast.tree; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import java.util.List; | ||
| import lombok.EqualsAndHashCode; | ||
| import lombok.Getter; | ||
| import lombok.RequiredArgsConstructor; | ||
| import lombok.Setter; | ||
| import lombok.ToString; | ||
| import org.opensearch.sql.ast.AbstractNodeVisitor; | ||
| import org.opensearch.sql.ast.expression.UnresolvedExpression; | ||
|
|
||
| /** | ||
| * AST node represent Highlight operation. | ||
| */ | ||
| @Getter | ||
| @Setter | ||
| @ToString | ||
| @EqualsAndHashCode(callSuper = false) | ||
| @RequiredArgsConstructor | ||
| public class Highlight extends UnresolvedPlan { | ||
| private final UnresolvedExpression expression; | ||
| private UnresolvedPlan child; | ||
|
|
||
| @Override | ||
| public Highlight attach(UnresolvedPlan child) { | ||
| this.child = child; | ||
| return this; | ||
| } | ||
|
|
||
| @Override | ||
| public List<UnresolvedPlan> getChild() { | ||
| return ImmutableList.of(this.child); | ||
| } | ||
|
|
||
| @Override | ||
| public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) { | ||
| return nodeVisitor.visitHighlight(this, context); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
57 changes: 0 additions & 57 deletions
57
core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
121 changes: 121 additions & 0 deletions
121
core/src/main/java/org/opensearch/sql/planner/physical/HighlightOperator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,121 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.planner.physical; | ||
|
|
||
| import static org.opensearch.sql.data.type.ExprCoreType.STRING; | ||
| import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; | ||
| import static org.opensearch.sql.expression.env.Environment.extendEnv; | ||
|
|
||
| import com.google.common.collect.ImmutableMap; | ||
| import java.util.LinkedHashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.stream.Collectors; | ||
| import lombok.EqualsAndHashCode; | ||
| import lombok.Getter; | ||
| import org.apache.commons.lang3.tuple.ImmutablePair; | ||
| import org.apache.commons.lang3.tuple.Pair; | ||
| import org.opensearch.sql.common.utils.StringUtils; | ||
| import org.opensearch.sql.data.model.ExprTupleValue; | ||
| import org.opensearch.sql.data.model.ExprValue; | ||
| import org.opensearch.sql.data.model.ExprValueUtils; | ||
| import org.opensearch.sql.expression.DSL; | ||
| import org.opensearch.sql.expression.Expression; | ||
| import org.opensearch.sql.expression.ReferenceExpression; | ||
| import org.opensearch.sql.expression.env.Environment; | ||
|
|
||
| /** | ||
| * HighlightOperator evaluates the {@link HighlightOperator#highlight} to put result | ||
| * into the output. Highlight fields in input are matched to the appropriate output. | ||
| * Direct mapping between input and output, as well as partial mapping is made | ||
| * dependent on highlight expression. | ||
| * | ||
| */ | ||
| @EqualsAndHashCode | ||
| public class HighlightOperator extends PhysicalPlan { | ||
| @Getter | ||
| private final PhysicalPlan input; | ||
| @Getter | ||
| private final Expression highlight; | ||
|
|
||
| public HighlightOperator(PhysicalPlan input, Expression highlight) { | ||
| this.input = input; | ||
| this.highlight = highlight; | ||
| } | ||
|
|
||
| @Override | ||
| public <R, C> R accept(PhysicalPlanNodeVisitor<R, C> visitor, C context) { | ||
| return visitor.visitHighlight(this, context); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean hasNext() { | ||
| return input.hasNext(); | ||
| } | ||
|
|
||
| @Override | ||
| public ExprValue next() { | ||
| ExprValue inputValue = input.next(); | ||
| Pair<String, ExprValue> evalMap = mapHighlight(inputValue.bindingTuples()); | ||
|
|
||
| if (STRUCT == inputValue.type()) { | ||
| ImmutableMap.Builder<String, ExprValue> resultBuilder = new ImmutableMap.Builder<>(); | ||
| Map<String, ExprValue> tupleValue = ExprValueUtils.getTupleValue(inputValue); | ||
| for (Map.Entry<String, ExprValue> valueEntry : tupleValue.entrySet()) { | ||
| resultBuilder.put(valueEntry); | ||
| } | ||
| resultBuilder.put(evalMap); | ||
| return ExprTupleValue.fromExprValueMap(resultBuilder.build()); | ||
| } else { | ||
| return inputValue; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Evaluate the expression in the {@link HighlightOperator#highlight} with {@link Environment}. | ||
| * @param env {@link Environment} | ||
| * @return The mapping of reference and {@link ExprValue} for expression. | ||
| */ | ||
| private Pair<String, ExprValue> mapHighlight(Environment<Expression, ExprValue> env) { | ||
| String osHighlightKey = "_highlight"; | ||
| if (!highlight.toString().contains("*")) { | ||
| osHighlightKey += "." + StringUtils.unquoteText(highlight.toString()); | ||
| } | ||
|
|
||
| ReferenceExpression osOutputVar = DSL.ref(osHighlightKey, STRING); | ||
| ExprValue value = osOutputVar.valueOf(env); | ||
|
|
||
| // In the event of multiple returned highlights and wildcard being | ||
| // used in conjunction with other highlight calls, we need to ensure | ||
| // only wildcard regex matching is mapped to wildcard call. | ||
| if (StringUtils.unquoteText(highlight.toString()).matches("(.+\\*)|(\\*.+)") | ||
| && value.type() == STRUCT) { | ||
| value = new ExprTupleValue( | ||
| new LinkedHashMap<String, ExprValue>(value.tupleValue() | ||
| .entrySet() | ||
| .stream() | ||
| .filter(s -> s.getKey().matches( | ||
| StringUtils.unquoteText( | ||
| highlight.toString().replace("*", "(.*)")))) | ||
| .collect(Collectors.toMap( | ||
| e -> e.getKey(), | ||
| e -> e.getValue())))); | ||
| } | ||
|
|
||
| String sqlHighlightKey = "highlight(" + StringUtils.unquoteText(highlight.toString()) + ")"; | ||
| ReferenceExpression sqlOutputVar = DSL.ref(sqlHighlightKey, STRING); | ||
|
|
||
| // Add mapping for sql output and opensearch returned highlight fields | ||
| extendEnv(env, sqlOutputVar, value); | ||
|
|
||
| return new ImmutablePair<>(sqlOutputVar.toString(), value); | ||
| } | ||
|
|
||
| @Override | ||
| public List<PhysicalPlan> getChild() { | ||
| return List.of(this.input); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.