Skip to content

Commit

Permalink
fix(search): fix filters for hasX and numValues fields (#9729)
Browse files Browse the repository at this point in the history
  • Loading branch information
@RyanHolstien
RyanHolstien authored Jan 26, 2024
1 parent fc27ab2 commit 051f570
Show file tree
Hide file tree
Showing 11 changed files with 175 additions and 71 deletions.
12 changes: 12 additions & 0 deletions entity-registry/src/main/java/com/linkedin/metadata/models/ConfigEntitySpec.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@
import com.linkedin.data.schema.RecordDataSchema;
import com.linkedin.data.schema.TyperefDataSchema;
import com.linkedin.metadata.models.annotation.EntityAnnotation;
import com.linkedin.metadata.models.annotation.SearchableAnnotation;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
Expand All @@ -19,6 +21,7 @@ public class ConfigEntitySpec implements EntitySpec {
private final Map<String, AspectSpec> _aspectSpecs;

private List<SearchableFieldSpec> _searchableFieldSpecs;
private Map<String, Set<SearchableAnnotation.FieldType>> searchableFieldTypeMap;

public ConfigEntitySpec(
@Nonnull final String entityName,
Expand Down Expand Up @@ -89,4 +92,13 @@ public List<SearchableFieldSpec> getSearchableFieldSpecs() {

return _searchableFieldSpecs;
}

@Override
public Map<String, Set<SearchableAnnotation.FieldType>> getSearchableFieldTypes() {
if (searchableFieldTypeMap == null) {
searchableFieldTypeMap = EntitySpec.super.getSearchableFieldTypes();
}

return searchableFieldTypeMap;
}
}
12 changes: 12 additions & 0 deletions entity-registry/src/main/java/com/linkedin/metadata/models/DefaultEntitySpec.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@
import com.linkedin.data.schema.RecordDataSchema;
import com.linkedin.data.schema.TyperefDataSchema;
import com.linkedin.metadata.models.annotation.EntityAnnotation;
import com.linkedin.metadata.models.annotation.SearchableAnnotation;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
Expand All @@ -24,6 +26,7 @@ public class DefaultEntitySpec implements EntitySpec {
private final TyperefDataSchema _aspectTyperefSchema;

private List<SearchableFieldSpec> _searchableFieldSpecs;
private Map<String, Set<SearchableAnnotation.FieldType>> searchableFieldTypeMap;

public DefaultEntitySpec(
@Nonnull final Collection<AspectSpec> aspectSpecs,
Expand Down Expand Up @@ -102,4 +105,13 @@ public List<SearchableFieldSpec> getSearchableFieldSpecs() {

return _searchableFieldSpecs;
}

@Override
public Map<String, Set<SearchableAnnotation.FieldType>> getSearchableFieldTypes() {
if (searchableFieldTypeMap == null) {
searchableFieldTypeMap = EntitySpec.super.getSearchableFieldTypes();
}

return searchableFieldTypeMap;
}
}
45 changes: 35 additions & 10 deletions entity-registry/src/main/java/com/linkedin/metadata/models/EntitySpec.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
import com.linkedin.data.schema.RecordDataSchema;
import com.linkedin.data.schema.TyperefDataSchema;
import com.linkedin.metadata.models.annotation.EntityAnnotation;
import com.linkedin.metadata.models.annotation.SearchableAnnotation;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -39,16 +41,39 @@ default List<SearchableFieldSpec> getSearchableFieldSpecs() {
.collect(Collectors.toList());
}

default Map<String, Set<SearchableFieldSpec>> getSearchableFieldSpecMap() {
return getSearchableFieldSpecs().stream()
.collect(
Collectors.toMap(
searchableFieldSpec -> searchableFieldSpec.getSearchableAnnotation().getFieldName(),
searchableFieldSpec -> new HashSet<>(Collections.singleton(searchableFieldSpec)),
(set1, set2) -> {
set1.addAll(set2);
return set1;
}));
default Map<String, Set<SearchableAnnotation.FieldType>> getSearchableFieldTypes() {
// Get additional fields and mint SearchableFieldSpecs for them
Map<String, Set<SearchableAnnotation.FieldType>> fieldSpecMap = new HashMap<>();
for (SearchableFieldSpec fieldSpec : getSearchableFieldSpecs()) {
SearchableAnnotation searchableAnnotation = fieldSpec.getSearchableAnnotation();
if (searchableAnnotation.getNumValuesFieldName().isPresent()) {
String fieldName = searchableAnnotation.getNumValuesFieldName().get();
Set<SearchableAnnotation.FieldType> fieldSet = new HashSet<>();
fieldSet.add(SearchableAnnotation.FieldType.COUNT);
fieldSpecMap.put(fieldName, fieldSet);
}
if (searchableAnnotation.getHasValuesFieldName().isPresent()) {
String fieldName = searchableAnnotation.getHasValuesFieldName().get();
Set<SearchableAnnotation.FieldType> fieldSet = new HashSet<>();
fieldSet.add(SearchableAnnotation.FieldType.BOOLEAN);
fieldSpecMap.put(fieldName, fieldSet);
}
}
fieldSpecMap.putAll(
getSearchableFieldSpecs().stream()
.collect(
Collectors.toMap(
searchableFieldSpec ->
searchableFieldSpec.getSearchableAnnotation().getFieldName(),
searchableFieldSpec ->
new HashSet<>(
Collections.singleton(
searchableFieldSpec.getSearchableAnnotation().getFieldType())),
(set1, set2) -> {
set1.addAll(set2);
return set1;
})));
return fieldSpecMap;
}

default List<SearchScoreFieldSpec> getSearchScoreFieldSpecs() {
Expand Down
8 changes: 4 additions & 4 deletions metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import com.linkedin.metadata.config.search.SearchConfiguration;
import com.linkedin.metadata.config.search.custom.CustomSearchConfiguration;
import com.linkedin.metadata.models.EntitySpec;
import com.linkedin.metadata.models.SearchableFieldSpec;
import com.linkedin.metadata.models.annotation.SearchableAnnotation;
import com.linkedin.metadata.models.registry.EntityRegistry;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.search.elasticsearch.query.request.SearchRequestHandler;
Expand Down Expand Up @@ -557,7 +557,7 @@ private QueryBuilder buildQueryStringV2(
queryBuilder.filter(QueryBuilders.rangeQuery(BROWSE_PATH_V2_DEPTH).gt(browseDepthVal));

queryBuilder.filter(
SearchRequestHandler.getFilterQuery(filter, entitySpec.getSearchableFieldSpecMap()));
SearchRequestHandler.getFilterQuery(filter, entitySpec.getSearchableFieldTypes()));

return queryBuilder;
}
Expand All @@ -583,9 +583,9 @@ private QueryBuilder buildQueryStringBrowseAcrossEntities(

queryBuilder.filter(QueryBuilders.rangeQuery(BROWSE_PATH_V2_DEPTH).gt(browseDepthVal));

Map<String, Set<SearchableFieldSpec>> searchableFields =
Map<String, Set<SearchableAnnotation.FieldType>> searchableFields =
entitySpecs.stream()
.flatMap(entitySpec -> entitySpec.getSearchableFieldSpecMap().entrySet().stream())
.flatMap(entitySpec -> entitySpec.getSearchableFieldTypes().entrySet().stream())
.collect(
Collectors.toMap(
Map.Entry::getKey,
Expand Down
3 changes: 1 addition & 2 deletions metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,7 @@ public long docCount(@Nonnull String entityName) {
EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName);
CountRequest countRequest =
new CountRequest(indexConvention.getIndexName(entitySpec))
.query(
SearchRequestHandler.getFilterQuery(null, entitySpec.getSearchableFieldSpecMap()));
.query(SearchRequestHandler.getFilterQuery(null, entitySpec.getSearchableFieldTypes()));
try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "docCount").time()) {
return client.count(countRequest, RequestOptions.DEFAULT).getCount();
} catch (IOException e) {
Expand Down
10 changes: 6 additions & 4 deletions .../com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
public class AutocompleteRequestHandler {

private final List<String> _defaultAutocompleteFields;
private final Map<String, Set<SearchableFieldSpec>> searchableFields;
private final Map<String, Set<SearchableAnnotation.FieldType>> searchableFieldTypes;

private static final Map<EntitySpec, AutocompleteRequestHandler>
AUTOCOMPLETE_QUERY_BUILDER_BY_ENTITY_NAME = new ConcurrentHashMap<>();
Expand All @@ -56,14 +56,16 @@ public AutocompleteRequestHandler(@Nonnull EntitySpec entitySpec) {
.map(SearchableAnnotation::getFieldName),
Stream.of("urn"))
.collect(Collectors.toList());
searchableFields =
searchableFieldTypes =
fieldSpecs.stream()
.collect(
Collectors.toMap(
searchableFieldSpec ->
searchableFieldSpec.getSearchableAnnotation().getFieldName(),
searchableFieldSpec ->
new HashSet<>(Collections.singleton(searchableFieldSpec)),
new HashSet<>(
Collections.singleton(
searchableFieldSpec.getSearchableAnnotation().getFieldType())),
(set1, set2) -> {
set1.addAll(set2);
return set1;
Expand All @@ -81,7 +83,7 @@ public SearchRequest getSearchRequest(
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(limit);
searchSourceBuilder.query(getQuery(input, field));
searchSourceBuilder.postFilter(ESUtils.buildFilterQuery(filter, false, searchableFields));
searchSourceBuilder.postFilter(ESUtils.buildFilterQuery(filter, false, searchableFieldTypes));
searchSourceBuilder.highlighter(getHighlights(field));
searchRequest.source(searchSourceBuilder);
return searchRequest;
Expand Down
13 changes: 7 additions & 6 deletions ...n/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public class SearchRequestHandler {
private final SearchConfiguration _configs;
private final SearchQueryBuilder _searchQueryBuilder;
private final AggregationQueryBuilder _aggregationQueryBuilder;
private final Map<String, Set<SearchableFieldSpec>> searchableFields;
private final Map<String, Set<SearchableAnnotation.FieldType>> searchableFieldTypes;

private SearchRequestHandler(
@Nonnull EntitySpec entitySpec,
Expand All @@ -122,9 +122,9 @@ private SearchRequestHandler(
_searchQueryBuilder = new SearchQueryBuilder(configs, customSearchConfiguration);
_aggregationQueryBuilder = new AggregationQueryBuilder(configs, annotations);
_configs = configs;
searchableFields =
searchableFieldTypes =
_entitySpecs.stream()
.flatMap(entitySpec -> entitySpec.getSearchableFieldSpecMap().entrySet().stream())
.flatMap(entitySpec -> entitySpec.getSearchableFieldTypes().entrySet().stream())
.collect(
Collectors.toMap(
Map.Entry::getKey,
Expand Down Expand Up @@ -182,12 +182,13 @@ private BinaryOperator<String> mapMerger() {
}

public BoolQueryBuilder getFilterQuery(@Nullable Filter filter) {
return getFilterQuery(filter, searchableFields);
return getFilterQuery(filter, searchableFieldTypes);
}

public static BoolQueryBuilder getFilterQuery(
@Nullable Filter filter, Map<String, Set<SearchableFieldSpec>> searchableFields) {
BoolQueryBuilder filterQuery = ESUtils.buildFilterQuery(filter, false, searchableFields);
@Nullable Filter filter,
Map<String, Set<SearchableAnnotation.FieldType>> searchableFieldTypes) {
BoolQueryBuilder filterQuery = ESUtils.buildFilterQuery(filter, false, searchableFieldTypes);

return filterSoftDeletedByDefault(filter, filterQuery);
}
Expand Down
Loading

0 comments on commit 051f570

Please sign in to comment.