From e5e98d9c712174aa2ed713069556a727d71cfbc4 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Tue, 17 Sep 2024 07:36:57 -0500 Subject: [PATCH] refactor(search): refactor field type detection * Add fallback to mappings detected field type in addition to PDL annotations * Removed manual list of non-annotation driven field types (runId, _entityName, etc) --- .../graphql/resolvers/ResolverUtils.java | 50 ++----- .../assertion/AssertionRunEventResolver.java | 3 +- .../auth/ListAccessTokensResolver.java | 5 +- .../resolvers/chart/BrowseV2Resolver.java | 4 +- .../ListDataProductAssetsResolver.java | 6 +- .../domain/DomainEntitiesResolver.java | 4 +- .../CreateDynamicFormAssignmentResolver.java | 4 +- .../source/ListIngestionSourcesResolver.java | 5 +- .../load/TimeSeriesAspectResolver.java | 9 +- .../resolvers/mutate/util/FormUtils.java | 9 +- .../ownership/ListOwnershipTypesResolver.java | 5 +- .../policy/ListPoliciesResolver.java | 6 +- .../resolvers/query/ListQueriesResolver.java | 8 +- .../ListRecommendationsResolver.java | 5 +- .../AggregateAcrossEntitiesResolver.java | 4 +- .../search/AutoCompleteResolver.java | 6 +- .../resolvers/search/AutocompleteUtils.java | 5 +- .../search/ScrollAcrossEntitiesResolver.java | 4 +- .../search/ScrollAcrossLineageResolver.java | 5 +- .../search/SearchAcrossEntitiesResolver.java | 5 +- .../search/SearchAcrossLineageResolver.java | 5 +- .../resolvers/search/SearchResolver.java | 5 +- .../view/ListGlobalViewsResolver.java | 8 +- .../resolvers/view/ListMyViewsResolver.java | 2 +- .../graphql/resolvers/view/ViewUtils.java | 6 +- .../graphql/resolvers/ResolverUtilsTest.java | 17 +-- .../auth/ListAccessTokensResolverTest.java | 2 +- .../browse/BrowseV2ResolverTest.java | 4 +- .../query/ListQueriesResolverTest.java | 2 +- .../AggregateAcrossEntitiesResolverTest.java | 2 +- .../SearchAcrossEntitiesResolverTest.java | 2 +- .../view/CreateViewResolverTest.java | 4 +- .../view/ListGlobalViewsResolverTest.java | 2 +- .../view/ListMyViewsResolverTest.java | 6 +- .../view/UpdateViewResolverTest.java | 8 +- .../graphql/resolvers/view/ViewUtilsTest.java | 6 +- .../metadata/search/LineageSearchService.java | 7 +- .../indexbuilder/EntityIndexBuilders.java | 10 +- .../indexbuilder/MappingsBuilder.java | 21 +-- .../elasticsearch/query/ESBrowseDAO.java | 7 +- .../elasticsearch/query/ESSearchDAO.java | 16 ++- .../query/request/SearchRequestHandler.java | 127 ++++++++++++++++-- .../search/LineageServiceTestBase.java | 14 +- .../indexbuilder/MappingsBuilderTest.java | 18 ++- .../request/SearchRequestHandlerTest.java | 37 ++++- .../test/search/SearchTestUtils.java | 4 +- .../gms/servlet/ConfigSearchExport.java | 6 +- 47 files changed, 277 insertions(+), 223 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java index d34fd9f32d322e..5f873b4bebab32 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java @@ -6,7 +6,6 @@ import com.datahub.authentication.Authentication; import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.StringArray; import com.linkedin.datahub.graphql.QueryContext; @@ -14,14 +13,12 @@ import com.linkedin.datahub.graphql.generated.AndFilterInput; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.resolvers.search.SearchUtils; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; import com.linkedin.metadata.query.filter.Criterion; import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; -import com.linkedin.metadata.search.utils.ESUtils; import com.linkedin.metadata.service.ViewService; import com.linkedin.view.DataHubViewInfo; import graphql.schema.DataFetchingEnvironment; @@ -39,8 +36,6 @@ public class ResolverUtils { - private static final Set KEYWORD_EXCLUDED_FILTERS = - ImmutableSet.of("runId", "_entityType"); private static final ObjectMapper MAPPER = new ObjectMapper(); static { @@ -111,11 +106,10 @@ public static Map buildFacetFilters( return facetFilters; } - public static List criterionListFromAndFilter( - List andFilters, @Nullable AspectRetriever aspectRetriever) { + public static List criterionListFromAndFilter(List andFilters) { return andFilters != null && !andFilters.isEmpty() ? andFilters.stream() - .map(filter -> criterionFromFilter(filter, aspectRetriever)) + .map(filter -> criterionFromFilter(filter)) .collect(Collectors.toList()) : Collections.emptyList(); } @@ -124,14 +118,13 @@ public static List criterionListFromAndFilter( // conjunctive criterion // arrays, rather than just one for the AND case. public static ConjunctiveCriterionArray buildConjunctiveCriterionArrayWithOr( - @Nonnull List orFilters, @Nullable AspectRetriever aspectRetriever) { + @Nonnull List orFilters) { return new ConjunctiveCriterionArray( orFilters.stream() .map( orFilter -> { CriterionArray andCriterionForOr = - new CriterionArray( - criterionListFromAndFilter(orFilter.getAnd(), aspectRetriever)); + new CriterionArray(criterionListFromAndFilter(orFilter.getAnd())); return new ConjunctiveCriterion().setAnd(andCriterionForOr); }) .collect(Collectors.toList())); @@ -139,9 +132,7 @@ public static ConjunctiveCriterionArray buildConjunctiveCriterionArrayWithOr( @Nullable public static Filter buildFilter( - @Nullable List andFilters, - @Nullable List orFilters, - @Nullable AspectRetriever aspectRetriever) { + @Nullable List andFilters, @Nullable List orFilters) { if ((andFilters == null || andFilters.isEmpty()) && (orFilters == null || orFilters.isEmpty())) { return null; @@ -150,34 +141,21 @@ public static Filter buildFilter( // Or filters are the new default. We will check them first. // If we have OR filters, we need to build a series of CriterionArrays if (orFilters != null && !orFilters.isEmpty()) { - return new Filter().setOr(buildConjunctiveCriterionArrayWithOr(orFilters, aspectRetriever)); + return new Filter().setOr(buildConjunctiveCriterionArrayWithOr(orFilters)); } // If or filters are not set, someone may be using the legacy and filters - final List andCriterions = criterionListFromAndFilter(andFilters, aspectRetriever); + final List andCriterions = criterionListFromAndFilter(andFilters); return new Filter() .setOr( new ConjunctiveCriterionArray( new ConjunctiveCriterion().setAnd(new CriterionArray(andCriterions)))); } - public static Criterion criterionFromFilter( - final FacetFilterInput filter, @Nullable AspectRetriever aspectRetriever) { - return criterionFromFilter(filter, false, aspectRetriever); - } - // Translates a FacetFilterInput (graphql input class) into Criterion (our internal model) - public static Criterion criterionFromFilter( - final FacetFilterInput filter, - final Boolean skipKeywordSuffix, - @Nullable AspectRetriever aspectRetriever) { + public static Criterion criterionFromFilter(final FacetFilterInput filter) { Criterion result = new Criterion(); - - if (skipKeywordSuffix) { - result.setField(filter.getField()); - } else { - result.setField(getFilterField(filter.getField(), skipKeywordSuffix, aspectRetriever)); - } + result.setField(filter.getField()); // `value` is deprecated in place of `values`- this is to support old query patterns. If values // is provided, @@ -210,16 +188,6 @@ public static Criterion criterionFromFilter( return result; } - private static String getFilterField( - final String originalField, - final boolean skipKeywordSuffix, - @Nullable AspectRetriever aspectRetriever) { - if (KEYWORD_EXCLUDED_FILTERS.contains(originalField)) { - return originalField; - } - return ESUtils.toKeywordField(originalField, skipKeywordSuffix, aspectRetriever); - } - public static Filter viewFilter( OperationContext opContext, ViewService viewService, String viewUrn) { if (viewUrn == null) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java index 0e9d2cea611416..f7e86f1f2f3452 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java @@ -12,6 +12,7 @@ import com.linkedin.datahub.graphql.generated.AssertionRunStatus; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.FilterInput; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.dataset.mappers.AssertionRunEventMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; @@ -147,7 +148,7 @@ public static Filter buildFilter( .setAnd( new CriterionArray( facetFilters.stream() - .map(filter -> criterionFromFilter(filter, true, aspectRetriever)) + .map(ResolverUtils::criterionFromFilter) .collect(Collectors.toList()))))); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java index e0ecebbbc7bc2e..29d44b526692f5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java @@ -71,10 +71,7 @@ public CompletableFuture get(DataFetchingEnvironment envi .withSearchFlags(flags -> flags.setFulltext(true)), Constants.ACCESS_TOKEN_ENTITY_NAME, "", - buildFilter( - filters, - Collections.emptyList(), - context.getOperationContext().getAspectRetriever()), + buildFilter(filters, Collections.emptyList()), sortCriteria, start, count); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java index b54ca398aef980..18ee5f595ce582 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java @@ -74,9 +74,7 @@ public CompletableFuture get(DataFetchingEnvironment environmen ? BROWSE_PATH_V2_DELIMITER + String.join(BROWSE_PATH_V2_DELIMITER, input.getPath()) : ""; - final Filter inputFilter = - ResolverUtils.buildFilter( - null, input.getOrFilters(), context.getOperationContext().getAspectRetriever()); + final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); BrowseResultV2 browseResults = _entityClient.browseV2( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java index 5c43fafb678c5c..e59f7b3116acdb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java @@ -151,11 +151,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) filters.removeIf(f -> f.getField().equals(OUTPUT_PORTS_FILTER_FIELD)); } // add urns from the aspect to our filters - final Filter baseFilter = - ResolverUtils.buildFilter( - filters, - input.getOrFilters(), - context.getOperationContext().getAspectRetriever()); + final Filter baseFilter = ResolverUtils.buildFilter(filters, input.getOrFilters()); final Filter finalFilter = buildFilterWithUrns( context.getDataHubAppConfig(), new HashSet<>(urnsToFilterOn), baseFilter); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java index 6a880503802cb4..c6265380fb2fd0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java @@ -79,9 +79,7 @@ public CompletableFuture get(final DataFetchingEnvironment enviro .getFilters() .forEach( filter -> { - criteria.add( - criterionFromFilter( - filter, true, context.getOperationContext().getAspectRetriever())); + criteria.add(criterionFromFilter(filter)); }); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java index 3cf4d9175d45bf..b9d74f8af660e8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java @@ -33,9 +33,7 @@ public CompletableFuture get(final DataFetchingEnvironment environment) final CreateDynamicFormAssignmentInput input = bindArgument(environment.getArgument("input"), CreateDynamicFormAssignmentInput.class); final Urn formUrn = UrnUtils.getUrn(input.getFormUrn()); - final DynamicFormAssignment formAssignment = - FormUtils.mapDynamicFormAssignment( - input, context.getOperationContext().getAspectRetriever()); + final DynamicFormAssignment formAssignment = FormUtils.mapDynamicFormAssignment(input); return GraphQLConcurrencyUtils.supplyAsync( () -> { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java index 1a2806224e4a92..8ead47aa65ceb0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java @@ -68,10 +68,7 @@ public CompletableFuture get( .withSearchFlags(flags -> flags.setFulltext(true)), Constants.INGESTION_SOURCE_ENTITY_NAME, query, - buildFilter( - filters, - Collections.emptyList(), - context.getOperationContext().getAspectRetriever()), + buildFilter(filters, Collections.emptyList()), null, start, count); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java index 4d4b898618bf9c..0f4f0b2645e425 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java @@ -9,9 +9,9 @@ import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.FilterInput; import com.linkedin.datahub.graphql.generated.TimeSeriesAspect; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.EnvelopedAspect; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; @@ -120,7 +120,7 @@ public CompletableFuture> get(DataFetchingEnvironment env maybeStartTimeMillis, maybeEndTimeMillis, maybeLimit, - buildFilters(maybeFilters, context.getOperationContext().getAspectRetriever()), + buildFilters(maybeFilters), maybeSort); // Step 2: Bind profiles into GraphQL strong types. @@ -135,8 +135,7 @@ public CompletableFuture> get(DataFetchingEnvironment env "get"); } - private Filter buildFilters( - @Nullable FilterInput maybeFilters, @Nullable AspectRetriever aspectRetriever) { + private Filter buildFilters(@Nullable FilterInput maybeFilters) { if (maybeFilters == null) { return null; } @@ -147,7 +146,7 @@ private Filter buildFilters( .setAnd( new CriterionArray( maybeFilters.getAnd().stream() - .map(filter -> criterionFromFilter(filter, true, aspectRetriever)) + .map(ResolverUtils::criterionFromFilter) .collect(Collectors.toList()))))); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java index d118c04d19393d..cac0cca2682e84 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java @@ -18,7 +18,6 @@ import com.linkedin.form.FormPromptType; import com.linkedin.form.FormType; import com.linkedin.form.StructuredPropertyParams; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -31,7 +30,6 @@ import java.util.UUID; import java.util.stream.Collectors; import javax.annotation.Nonnull; -import javax.annotation.Nullable; public class FormUtils { @@ -61,16 +59,13 @@ public static PrimitivePropertyValueArray getStructuredPropertyValuesFromInput( /** Map a GraphQL CreateDynamicFormAssignmentInput to the GMS DynamicFormAssignment aspect */ @Nonnull public static DynamicFormAssignment mapDynamicFormAssignment( - @Nonnull final CreateDynamicFormAssignmentInput input, - @Nullable AspectRetriever aspectRetriever) { + @Nonnull final CreateDynamicFormAssignmentInput input) { Objects.requireNonNull(input, "input must not be null"); final DynamicFormAssignment result = new DynamicFormAssignment(); final Filter filter = new Filter() - .setOr( - ResolverUtils.buildConjunctiveCriterionArrayWithOr( - input.getOrFilters(), aspectRetriever)); + .setOr(ResolverUtils.buildConjunctiveCriterionArrayWithOr(input.getOrFilters())); result.setFilter(filter); return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java index da0d5dd07a94f0..05fc540e39de4c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java @@ -63,10 +63,7 @@ public CompletableFuture get(DataFetchingEnvironment e context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)), Constants.OWNERSHIP_TYPE_ENTITY_NAME, query, - buildFilter( - filters, - Collections.emptyList(), - context.getOperationContext().getAspectRetriever()), + buildFilter(filters, Collections.emptyList()), Collections.singletonList(DEFAULT_SORT_CRITERION), start, count); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java index ce11451aa1913f..4120401e0150f9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java @@ -59,11 +59,7 @@ public CompletableFuture get(final DataFetchingEnvironment e log.debug( "User {} listing policies with filters {}", context.getActorUrn(), filters.toString()); - final Filter filter = - ResolverUtils.buildFilter( - facetFilters, - Collections.emptyList(), - context.getOperationContext().getAspectRetriever()); + final Filter filter = ResolverUtils.buildFilter(facetFilters, Collections.emptyList()); return _policyFetcher .fetchPolicies(context.getOperationContext(), start, query, count, filter) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java index aa411f019a4c08..3c84884bedbae8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolver.java @@ -15,7 +15,6 @@ import com.linkedin.datahub.graphql.generated.ListQueriesResult; import com.linkedin.datahub.graphql.generated.QueryEntity; import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; @@ -74,7 +73,7 @@ public CompletableFuture get(final DataFetchingEnvironment en flags -> flags.setFulltext(true).setSkipHighlighting(true)), QUERY_ENTITY_NAME, query, - buildFilters(input, context.getOperationContext().getAspectRetriever()), + buildFilters(input), sortCriteria, start, count); @@ -111,8 +110,7 @@ private List mapUnresolvedQueries(final List queryUrns) { } @Nullable - private Filter buildFilters( - @Nonnull final ListQueriesInput input, @Nullable AspectRetriever aspectRetriever) { + private Filter buildFilters(@Nonnull final ListQueriesInput input) { final AndFilterInput criteria = new AndFilterInput(); List andConditions = new ArrayList<>(); @@ -139,6 +137,6 @@ private Filter buildFilters( } criteria.setAnd(andConditions); - return buildFilter(Collections.emptyList(), ImmutableList.of(criteria), aspectRetriever); + return buildFilter(Collections.emptyList(), ImmutableList.of(criteria)); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java index 01818778643905..28334b2c0af9a4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java @@ -17,6 +17,7 @@ import com.linkedin.datahub.graphql.generated.RecommendationRenderType; import com.linkedin.datahub.graphql.generated.RecommendationRequestContext; import com.linkedin.datahub.graphql.generated.SearchParams; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.query.filter.CriterionArray; @@ -105,9 +106,7 @@ private com.linkedin.metadata.recommendation.RecommendationRequestContext mapReq searchRequestContext.setFilters( new CriterionArray( requestContext.getSearchRequestContext().getFilters().stream() - .map( - facetField -> - criterionFromFilter(facetField, opContext.getAspectRetriever())) + .map(ResolverUtils::criterionFromFilter) .collect(Collectors.toList()))); } mappedRequestContext.setSearchRequestContext(searchRequestContext); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java index 19bccaf2650866..29b71d95ad9749 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java @@ -64,9 +64,7 @@ public CompletableFuture get(DataFetchingEnvironment environme UrnUtils.getUrn(input.getViewUrn())) : null; - final Filter inputFilter = - ResolverUtils.buildFilter( - null, input.getOrFilters(), context.getOperationContext().getAspectRetriever()); + final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); final SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java index 79792940ef27f7..5e4f56dbfff980 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteResolver.java @@ -51,11 +51,7 @@ public CompletableFuture get(DataFetchingEnvironment enviro throw new ValidationException("'query' parameter can not be null or empty"); } - final Filter filter = - ResolverUtils.buildFilter( - input.getFilters(), - input.getOrFilters(), - context.getOperationContext().getRetrieverContext().orElseThrow().getAspectRetriever()); + final Filter filter = ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); final int limit = input.getLimit() != null ? input.getLimit() : DEFAULT_LIMIT; return GraphQLConcurrencyUtils.supplyAsync( () -> { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java index 5b5888b89b241b..c28b18acf195b8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutocompleteUtils.java @@ -43,10 +43,7 @@ public static CompletableFuture batchGetAutocomplet GraphQLConcurrencyUtils.supplyAsync( () -> { final Filter filter = - ResolverUtils.buildFilter( - input.getFilters(), - input.getOrFilters(), - context.getOperationContext().getAspectRetriever()); + ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); final Filter finalFilter = view != null ? SearchUtils.combineFilters( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java index 8b8b93353bc6e8..77eef1b9a25c69 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java @@ -72,9 +72,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) UrnUtils.getUrn(input.getViewUrn())) : null; - final Filter baseFilter = - ResolverUtils.buildFilter( - null, input.getOrFilters(), context.getOperationContext().getAspectRetriever()); + final Filter baseFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); final SearchFlags searchFlags; com.linkedin.datahub.graphql.generated.SearchFlags inputFlags = input.getSearchFlags(); if (inputFlags != null) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java index 1b719b6f786205..2c058eb60a7ee3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java @@ -130,10 +130,7 @@ public CompletableFuture get(DataFetchingEnvironment entityNames, sanitizedQuery, maxHops, - ResolverUtils.buildFilter( - facetFilters, - input.getOrFilters(), - context.getOperationContext().getAspectRetriever()), + ResolverUtils.buildFilter(facetFilters, input.getOrFilters()), null, scrollId, keepAlive, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java index 0dbed92b7d58e7..d103704146d399 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java @@ -61,10 +61,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) : null; final Filter baseFilter = - ResolverUtils.buildFilter( - input.getFilters(), - input.getOrFilters(), - context.getOperationContext().getAspectRetriever()); + ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags()); List sortCriteria; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java index dc3a1fc17e4ec9..c90be924ac69f5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java @@ -139,10 +139,7 @@ public CompletableFuture get(DataFetchingEnvironment count); final Filter filter = - ResolverUtils.buildFilter( - input.getFilters(), - input.getOrFilters(), - context.getOperationContext().getAspectRetriever()); + ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); final SearchFlags searchFlags; com.linkedin.datahub.graphql.generated.SearchFlags inputFlags = input.getSearchFlags(); if (inputFlags != null) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java index 7a48e305dbfe49..45751fc6eb8cb2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java @@ -86,10 +86,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) context.getOperationContext().withSearchFlags(flags -> searchFlags), entityName, sanitizedQuery, - ResolverUtils.buildFilter( - input.getFilters(), - input.getOrFilters(), - context.getOperationContext().getAspectRetriever()), + ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()), Collections.emptyList(), start, count)); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java index 265f4d5f5d56e2..bda950ef3eb581 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java @@ -16,7 +16,6 @@ import com.linkedin.datahub.graphql.generated.ListViewsResult; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; @@ -31,7 +30,6 @@ import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; /** Resolver used for listing global DataHub Views. */ @@ -73,7 +71,7 @@ public CompletableFuture get(final DataFetchingEnvironment envi context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)), Constants.DATAHUB_VIEW_ENTITY_NAME, query, - buildFilters(context.getOperationContext().getAspectRetriever()), + buildFilters(), Collections.singletonList(DEFAULT_SORT_CRITERION), start, count); @@ -109,7 +107,7 @@ private List mapUnresolvedViews(final List entityUrns) { return results; } - private Filter buildFilters(@Nullable AspectRetriever aspectRetriever) { + private Filter buildFilters() { final AndFilterInput globalCriteria = new AndFilterInput(); List andConditions = new ArrayList<>(); andConditions.add( @@ -120,6 +118,6 @@ private Filter buildFilters(@Nullable AspectRetriever aspectRetriever) { false, FilterOperator.EQUAL)); globalCriteria.setAnd(andConditions); - return buildFilter(Collections.emptyList(), ImmutableList.of(globalCriteria), aspectRetriever); + return buildFilter(Collections.emptyList(), ImmutableList.of(globalCriteria)); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java index abfdeb2d608693..53a7c92bbe8ec3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java @@ -132,6 +132,6 @@ private Filter buildFilters( filterCriteria.setAnd(andConditions); // Currently, there is no way to fetch the views belonging to another user. - return buildFilter(Collections.emptyList(), ImmutableList.of(filterCriteria), aspectRetriever); + return buildFilter(Collections.emptyList(), ImmutableList.of(filterCriteria)); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java index 70a5ced4bfbf10..aa41a1e4876517 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java @@ -141,7 +141,7 @@ private static Filter buildAndFilter( .setAnd( new CriterionArray( input.stream() - .map(f -> ResolverUtils.criterionFromFilter(f, aspectRetriever)) + .map(ResolverUtils::criterionFromFilter) .collect(Collectors.toList())))))); return result; } @@ -157,9 +157,7 @@ private static Filter buildOrFilter( new ConjunctiveCriterion() .setAnd( new CriterionArray( - ImmutableList.of( - ResolverUtils.criterionFromFilter( - filter, aspectRetriever))))) + ImmutableList.of(ResolverUtils.criterionFromFilter(filter))))) .collect(Collectors.toList()))); return result; } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java index b01aacaaab65ca..2950b50bec8f18 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java @@ -13,7 +13,6 @@ import com.linkedin.datahub.graphql.TestUtils; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.FilterOperator; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.config.DataHubAppConfiguration; import com.linkedin.metadata.config.MetadataChangeProposalConfig; import com.linkedin.metadata.query.filter.Condition; @@ -45,8 +44,7 @@ public void testCriterionFromFilter() throws Exception { null, ImmutableList.of("urn:li:tag:abc", "urn:li:tag:def"), false, - FilterOperator.EQUAL), - mock(AspectRetriever.class)); + FilterOperator.EQUAL)); assertEquals( valuesCriterion, new Criterion() @@ -54,13 +52,12 @@ public void testCriterionFromFilter() throws Exception { .setValues(new StringArray(ImmutableList.of("urn:li:tag:abc", "urn:li:tag:def"))) .setNegated(false) .setCondition(Condition.EQUAL) - .setField("tags.keyword")); + .setField("tags")); // this is the legacy pathway Criterion valueCriterion = criterionFromFilter( - new FacetFilterInput("tags", "urn:li:tag:abc", null, true, FilterOperator.EQUAL), - mock(AspectRetriever.class)); + new FacetFilterInput("tags", "urn:li:tag:abc", null, true, FilterOperator.EQUAL)); assertEquals( valueCriterion, new Criterion() @@ -68,14 +65,12 @@ public void testCriterionFromFilter() throws Exception { .setValues(new StringArray(ImmutableList.of("urn:li:tag:abc"))) .setNegated(true) .setCondition(Condition.EQUAL) - .setField("tags.keyword")); + .setField("tags")); // check that both being null doesn't cause a NPE. this should never happen except via API // interaction Criterion doubleNullCriterion = - criterionFromFilter( - new FacetFilterInput("tags", null, null, true, FilterOperator.EQUAL), - mock(AspectRetriever.class)); + criterionFromFilter(new FacetFilterInput("tags", null, null, true, FilterOperator.EQUAL)); assertEquals( doubleNullCriterion, new Criterion() @@ -83,7 +78,7 @@ public void testCriterionFromFilter() throws Exception { .setValues(new StringArray(ImmutableList.of())) .setNegated(true) .setCondition(Condition.EQUAL) - .setField("tags.keyword")); + .setField("tags")); } @Test diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java index 020f74475ea607..30962aaef54125 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java @@ -46,7 +46,7 @@ public void testGetSuccess() throws Exception { any(), Mockito.eq(Constants.ACCESS_TOKEN_ENTITY_NAME), Mockito.eq(""), - Mockito.eq(buildFilter(filters, Collections.emptyList(), null)), + Mockito.eq(buildFilter(filters, Collections.emptyList())), Mockito.any(List.class), Mockito.eq(input.getStart()), Mockito.eq(input.getCount()))) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java index 9cf7e62e65e253..4897d0819b59fb 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java @@ -2,7 +2,6 @@ import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; import com.google.common.collect.ImmutableList; import com.linkedin.common.AuditStamp; @@ -18,7 +17,6 @@ import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.resolvers.chart.BrowseV2Resolver; import com.linkedin.entity.client.EntityClient; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.browse.BrowseResultGroupV2; import com.linkedin.metadata.browse.BrowseResultGroupV2Array; import com.linkedin.metadata.browse.BrowseResultMetadata; @@ -102,7 +100,7 @@ public static void testBrowseV2SuccessWithQueryAndFilter() throws Exception { facetFilterInput.setValues(ImmutableList.of("urn:li:corpuser:test")); andFilterInput.setAnd(ImmutableList.of(facetFilterInput)); orFilters.add(andFilterInput); - Filter filter = ResolverUtils.buildFilter(null, orFilters, mock(AspectRetriever.class)); + Filter filter = ResolverUtils.buildFilter(null, orFilters); EntityClient mockClient = initMockEntityClient( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java index ee728b17e8c621..414c62b693b698 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/query/ListQueriesResolverTest.java @@ -170,6 +170,6 @@ private Filter buildFilter(@Nullable QuerySource source, @Nullable String entity FilterOperator.EQUAL)); } criteria.setAnd(andConditions); - return ResolverUtils.buildFilter(Collections.emptyList(), ImmutableList.of(criteria), null); + return ResolverUtils.buildFilter(Collections.emptyList(), ImmutableList.of(criteria)); } } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java index d32eb9fcf120ca..129866bb0fa07a 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java @@ -107,7 +107,7 @@ public static void testApplyViewBaseFilter() throws Exception { FormService mockFormService = Mockito.mock(FormService.class); ViewService mockService = initMockViewService(TEST_VIEW_URN, info); - Filter baseFilter = createFilter("baseField.keyword", "baseTest"); + Filter baseFilter = createFilter("baseField", "baseTest"); EntityClient mockClient = initMockEntityClient( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java index 30d6f2dc6f2836..86508f1fd2742b 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java @@ -164,7 +164,7 @@ public static void testApplyViewBaseFilter() throws Exception { new CriterionArray( ImmutableList.of( new Criterion() - .setField("baseField.keyword") + .setField("baseField") .setValue("baseTest") .setCondition(Condition.EQUAL) .setNegated(false) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolverTest.java index 502188d4977a53..c009cf37c53971 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolverTest.java @@ -110,7 +110,7 @@ public void testGetSuccess() throws Exception { ImmutableList.of( new Criterion() .setCondition(Condition.EQUAL) - .setField("test1.keyword") + .setField("test1") .setValue( "value1") // Unfortunate --- For // backwards compat. @@ -121,7 +121,7 @@ public void testGetSuccess() throws Exception { .setNegated(false), new Criterion() .setCondition(Condition.IN) - .setField("test2.keyword") + .setField("test2") .setValue( "value1") // Unfortunate --- For // backwards compat. diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolverTest.java index a3b9e25e992259..b5c0531db792ba 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolverTest.java @@ -56,7 +56,7 @@ public void testGetSuccessInput() throws Exception { new CriterionArray( ImmutableList.of( new Criterion() - .setField("type.keyword") + .setField("type") .setValue(DataHubViewType.GLOBAL.toString()) .setValues( new StringArray( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolverTest.java index 99b0e76976748e..85d24f9251eaa3 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolverTest.java @@ -58,7 +58,7 @@ public void testGetSuccessInput1() throws Exception { new CriterionArray( ImmutableList.of( new Criterion() - .setField("createdBy.keyword") + .setField("createdBy") .setValue(TEST_USER.toString()) .setValues( new StringArray( @@ -67,7 +67,7 @@ public void testGetSuccessInput1() throws Exception { .setCondition(Condition.EQUAL) .setNegated(false), new Criterion() - .setField("type.keyword") + .setField("type") .setValue(DataHubViewType.GLOBAL.toString()) .setValues( new StringArray( @@ -124,7 +124,7 @@ public void testGetSuccessInput2() throws Exception { new CriterionArray( ImmutableList.of( new Criterion() - .setField("createdBy.keyword") + .setField("createdBy") .setValue(TEST_USER.toString()) .setValues( new StringArray( diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolverTest.java index fe32bec01fd7ed..86a502b40b9364 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolverTest.java @@ -111,7 +111,7 @@ public void testGetSuccessGlobalViewIsCreator() throws Exception { ImmutableList.of( new Criterion() .setCondition(Condition.EQUAL) - .setField("test1.keyword") + .setField("test1") .setValue( "value1") // Unfortunate --- For // backwards compat. @@ -122,7 +122,7 @@ public void testGetSuccessGlobalViewIsCreator() throws Exception { .setNegated(false), new Criterion() .setCondition(Condition.IN) - .setField("test2.keyword") + .setField("test2") .setValue( "value1") // Unfortunate --- For // backwards compat. @@ -175,7 +175,7 @@ public void testGetSuccessGlobalViewManageGlobalViews() throws Exception { ImmutableList.of( new Criterion() .setCondition(Condition.EQUAL) - .setField("test1.keyword") + .setField("test1") .setValue( "value1") // Unfortunate --- For // backwards compat. @@ -186,7 +186,7 @@ public void testGetSuccessGlobalViewManageGlobalViews() throws Exception { .setNegated(false), new Criterion() .setCondition(Condition.IN) - .setField("test2.keyword") + .setField("test2") .setValue( "value1") // Unfortunate --- For // backwards compat. diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java index 701ddd84c173e7..d142be1321a5c3 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java @@ -154,8 +154,7 @@ public void testMapDefinition() throws Exception { new StringArray( ImmutableList.of("value1", "value2"))) .setValue("value1") // Disgraceful - .setField( - "test1.keyword") // Consider whether we + .setField("test1") // Consider whether we // should NOT go through // the keyword mapping. .setCondition(Condition.IN), @@ -165,8 +164,7 @@ public void testMapDefinition() throws Exception { new StringArray( ImmutableList.of("value3", "value4"))) .setValue("value3") // Disgraceful - .setField( - "test2.keyword") // Consider whether we + .setField("test2") // Consider whether we // should NOT go through // the keyword mapping. .setCondition(Condition.CONTAIN)))))))); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java index bae1d6ce92ece7..435731a3f9d041 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java @@ -84,7 +84,6 @@ public class LineageSearchService { private final ExecutorService cacheRefillExecutor = Executors.newFixedThreadPool(1); private static final String DEGREE_FILTER = "degree"; - private static final String DEGREE_FILTER_INPUT = "degree.keyword"; private static final AggregationMetadata DEGREE_FILTER_GROUP = new AggregationMetadata() .setName(DEGREE_FILTER) @@ -252,7 +251,7 @@ public LineageSearchResult searchAcrossLineage( try { Filter reducedFilters = SearchUtils.removeCriteria( - inputFilters, criterion -> criterion.getField().equals(DEGREE_FILTER_INPUT)); + inputFilters, criterion -> criterion.getField().equals(DEGREE_FILTER)); if (canDoLightning(lineageRelationships, finalInput, reducedFilters, sortCriteria)) { codePath = "lightning"; @@ -657,7 +656,7 @@ private List filterRelationships( if (conjunctiveCriterion.hasAnd()) { List degreeFilter = conjunctiveCriterion.getAnd().stream() - .filter(criterion -> criterion.getField().equals(DEGREE_FILTER_INPUT)) + .filter(criterion -> criterion.getField().equals(DEGREE_FILTER)) .flatMap(c -> c.getValues().stream()) .collect(Collectors.toList()); if (!degreeFilter.isEmpty()) { @@ -801,7 +800,7 @@ public LineageScrollResult scrollAcrossLineage( Filter reducedFilters = SearchUtils.removeCriteria( - inputFilters, criterion -> criterion.getField().equals(DEGREE_FILTER_INPUT)); + inputFilters, criterion -> criterion.getField().equals(DEGREE_FILTER)); return getScrollResultInBatches( opContext, lineageRelationships, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java index 6f1d7f74f15427..79815886355f6c 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java @@ -43,12 +43,13 @@ public void reindexAll(Collection> prope public List buildReindexConfigs( Collection> properties) { Map settings = settingsBuilder.getSettings(); - MappingsBuilder.setEntityRegistry(entityRegistry); + return entityRegistry.getEntitySpecs().values().stream() .map( entitySpec -> { try { - Map mappings = MappingsBuilder.getMappings(entitySpec, properties); + Map mappings = + MappingsBuilder.getMappings(entityRegistry, entitySpec, properties); return indexBuilder.buildReindexState( indexConvention.getIndexName(entitySpec), mappings, settings); } catch (IOException e) { @@ -68,13 +69,14 @@ public List buildReindexConfigs( public List buildReindexConfigsWithNewStructProp( Urn urn, StructuredPropertyDefinition property) { Map settings = settingsBuilder.getSettings(); - MappingsBuilder.setEntityRegistry(entityRegistry); + return entityRegistry.getEntitySpecs().values().stream() .map( entitySpec -> { try { Map mappings = - MappingsBuilder.getMappings(entitySpec, List.of(Pair.of(urn, property))); + MappingsBuilder.getMappings( + entityRegistry, entitySpec, List.of(Pair.of(urn, property))); return indexBuilder.buildReindexState( indexConvention.getIndexName(entitySpec), mappings, settings, true); } catch (IOException e) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java index 5dc28a8fd598da..47bf80bde6cf7b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java @@ -65,12 +65,11 @@ public static Map getPartialNgramConfigWithOverrides( WORD_GRAMS_LENGTH_4); // Alias field mappings constants - public static final String ALIAS = "alias"; + public static final String ALIAS_FIELD_TYPE = "alias"; public static final String PATH = "path"; public static final String PROPERTIES = "properties"; public static final String DYNAMIC_TEMPLATES = "dynamic_templates"; - private static EntityRegistry entityRegistry; private MappingsBuilder() {} @@ -82,9 +81,10 @@ private MappingsBuilder() {} * @return mappings */ public static Map getMappings( + @Nonnull EntityRegistry entityRegistry, @Nonnull final EntitySpec entitySpec, Collection> structuredProperties) { - Map mappings = getMappings(entitySpec); + Map mappings = getMappings(entityRegistry, entitySpec); String entityName = entitySpec.getEntityAnnotation().getName(); Map structuredPropertiesForEntity = @@ -124,7 +124,8 @@ public static Map getMappings( return mappings; } - public static Map getMappings(@Nonnull final EntitySpec entitySpec) { + public static Map getMappings( + @Nonnull EntityRegistry entityRegistry, @Nonnull final EntitySpec entitySpec) { Map mappings = new HashMap<>(); entitySpec @@ -141,6 +142,7 @@ public static Map getMappings(@Nonnull final EntitySpec entitySp searchableRefFieldSpec -> mappings.putAll( getMappingForSearchableRefField( + entityRegistry, searchableRefFieldSpec, searchableRefFieldSpec.getSearchableRefAnnotation().getDepth()))); // Fixed fields @@ -332,7 +334,9 @@ private static Map getMappingsForSearchScoreField( } private static Map getMappingForSearchableRefField( - @Nonnull final SearchableRefFieldSpec searchableRefFieldSpec, @Nonnull final int depth) { + @Nonnull EntityRegistry entityRegistry, + @Nonnull final SearchableRefFieldSpec searchableRefFieldSpec, + @Nonnull final int depth) { Map mappings = new HashMap<>(); Map mappingForField = new HashMap<>(); Map mappingForProperty = new HashMap<>(); @@ -354,6 +358,7 @@ private static Map getMappingForSearchableRefField( entitySearchableRefFieldSpec -> mappingForField.putAll( getMappingForSearchableRefField( + entityRegistry, entitySearchableRefFieldSpec, Math.min( depth - 1, @@ -375,14 +380,10 @@ private static Map getMappingsForFieldNameAliases( fieldNameAliases.forEach( alias -> { Map aliasMappings = new HashMap<>(); - aliasMappings.put(TYPE, ALIAS); + aliasMappings.put(TYPE, ALIAS_FIELD_TYPE); aliasMappings.put(PATH, searchableFieldSpec.getSearchableAnnotation().getFieldName()); mappings.put(alias, aliasMappings); }); return mappings; } - - public static void setEntityRegistry(@Nonnull final EntityRegistry entityRegistryInput) { - entityRegistry = entityRegistryInput; - } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java index 9d4980f6f37a29..f1c42a1d277da0 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java @@ -610,7 +610,11 @@ private QueryBuilder buildQueryStringV2( EntitySpec entitySpec = opContext.getEntityRegistry().getEntitySpec(entityName); QueryBuilder query = SearchRequestHandler.getBuilder( - entitySpec, searchConfiguration, customSearchConfiguration, queryFilterRewriteChain) + opContext.getEntityRegistry(), + entitySpec, + searchConfiguration, + customSearchConfiguration, + queryFilterRewriteChain) .getQuery( finalOpContext, input, @@ -647,6 +651,7 @@ private QueryBuilder buildQueryStringBrowseAcrossEntities( QueryBuilder query = SearchRequestHandler.getBuilder( + finalOpContext.getEntityRegistry(), entitySpecs, searchConfiguration, customSearchConfiguration, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index d6329ba75d4282..cec73de7041263 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -120,6 +120,7 @@ private SearchResult executeAndExtract( return transformIndexIntoEntityName( opContext.getSearchContext().getIndexConvention(), SearchRequestHandler.getBuilder( + opContext.getEntityRegistry(), entitySpec, searchConfiguration, customSearchConfiguration, @@ -220,6 +221,7 @@ private ScrollResult executeAndExtract( return transformIndexIntoEntityName( opContext.getSearchContext().getIndexConvention(), SearchRequestHandler.getBuilder( + opContext.getEntityRegistry(), entitySpecs, searchConfiguration, customSearchConfiguration, @@ -267,6 +269,7 @@ public SearchResult search( // Step 1: construct the query final SearchRequest searchRequest = SearchRequestHandler.getBuilder( + opContext.getEntityRegistry(), entitySpecs, searchConfiguration, customSearchConfiguration, @@ -304,7 +307,11 @@ public SearchResult filter( Filter transformedFilters = transformFilterForEntities(filters, indexConvention); final SearchRequest searchRequest = SearchRequestHandler.getBuilder( - entitySpec, searchConfiguration, customSearchConfiguration, queryFilterRewriteChain) + opContext.getEntityRegistry(), + entitySpec, + searchConfiguration, + customSearchConfiguration, + queryFilterRewriteChain) .getFilterRequest(opContext, transformedFilters, sortCriteria, from, size); searchRequest.indices(indexConvention.getIndexName(entitySpec)); @@ -384,6 +391,7 @@ public Map aggregateByValue( IndexConvention indexConvention = opContext.getSearchContext().getIndexConvention(); final SearchRequest searchRequest = SearchRequestHandler.getBuilder( + opContext.getEntityRegistry(), entitySpecs, searchConfiguration, customSearchConfiguration, @@ -502,7 +510,11 @@ private SearchRequest getScrollRequest( } return SearchRequestHandler.getBuilder( - entitySpecs, searchConfiguration, customSearchConfiguration, queryFilterRewriteChain) + opContext.getEntityRegistry(), + entitySpecs, + searchConfiguration, + customSearchConfiguration, + queryFilterRewriteChain) .getSearchRequest( opContext, finalInput, postFilters, sortCriteria, sort, pitId, keepAlive, size, facets); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index 91cfeaf43a4cb4..c935e6f54742c3 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -1,6 +1,12 @@ package com.linkedin.metadata.search.elasticsearch.query.request; +import static com.linkedin.metadata.search.elasticsearch.indexbuilder.MappingsBuilder.ALIAS_FIELD_TYPE; +import static com.linkedin.metadata.search.elasticsearch.indexbuilder.MappingsBuilder.PATH; +import static com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder.TYPE; +import static com.linkedin.metadata.search.utils.ESUtils.DATE_FIELD_TYPE; +import static com.linkedin.metadata.search.utils.ESUtils.KEYWORD_FIELD_TYPE; import static com.linkedin.metadata.search.utils.ESUtils.NAME_SUGGESTION; +import static com.linkedin.metadata.search.utils.ESUtils.OBJECT_FIELD_TYPE; import static com.linkedin.metadata.search.utils.ESUtils.applyDefaultSearchFilters; import com.google.common.annotations.VisibleForTesting; @@ -13,6 +19,7 @@ import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.SearchableFieldSpec; import com.linkedin.metadata.models.annotation.SearchableAnnotation; +import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; @@ -27,6 +34,7 @@ import com.linkedin.metadata.search.SearchResultMetadata; import com.linkedin.metadata.search.SearchSuggestion; import com.linkedin.metadata.search.SearchSuggestionArray; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.MappingsBuilder; import com.linkedin.metadata.search.elasticsearch.query.filter.QueryFilterRewriteChain; import com.linkedin.metadata.search.features.Features; import com.linkedin.metadata.search.utils.ESAccessControlUtil; @@ -38,6 +46,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -82,14 +91,21 @@ public class SearchRequestHandler { private final QueryFilterRewriteChain queryFilterRewriteChain; private SearchRequestHandler( + @Nonnull EntityRegistry entityRegistry, @Nonnull EntitySpec entitySpec, @Nonnull SearchConfiguration configs, @Nullable CustomSearchConfiguration customSearchConfiguration, @Nonnull QueryFilterRewriteChain queryFilterRewriteChain) { - this(ImmutableList.of(entitySpec), configs, customSearchConfiguration, queryFilterRewriteChain); + this( + entityRegistry, + ImmutableList.of(entitySpec), + configs, + customSearchConfiguration, + queryFilterRewriteChain); } private SearchRequestHandler( + @Nonnull EntityRegistry entityRegistry, @Nonnull List entitySpecs, @Nonnull SearchConfiguration configs, @Nullable CustomSearchConfiguration customSearchConfiguration, @@ -106,21 +122,12 @@ private SearchRequestHandler( searchQueryBuilder = new SearchQueryBuilder(configs, customSearchConfiguration); aggregationQueryBuilder = new AggregationQueryBuilder(configs, entitySearchAnnotations); this.configs = configs; - searchableFieldTypes = - this.entitySpecs.stream() - .flatMap(entitySpec -> entitySpec.getSearchableFieldTypes().entrySet().stream()) - .collect( - Collectors.toMap( - Map.Entry::getKey, - Map.Entry::getValue, - (set1, set2) -> { - set1.addAll(set2); - return set1; - })); + this.searchableFieldTypes = buildSearchableFieldTypes(entityRegistry, entitySpecs); this.queryFilterRewriteChain = queryFilterRewriteChain; } public static SearchRequestHandler getBuilder( + @Nonnull EntityRegistry entityRegistry, @Nonnull EntitySpec entitySpec, @Nonnull SearchConfiguration configs, @Nullable CustomSearchConfiguration customSearchConfiguration, @@ -129,10 +136,15 @@ public static SearchRequestHandler getBuilder( ImmutableList.of(entitySpec), k -> new SearchRequestHandler( - entitySpec, configs, customSearchConfiguration, queryFilterRewriteChain)); + entityRegistry, + entitySpec, + configs, + customSearchConfiguration, + queryFilterRewriteChain)); } public static SearchRequestHandler getBuilder( + @Nonnull EntityRegistry entityRegistry, @Nonnull List entitySpecs, @Nonnull SearchConfiguration configs, @Nullable CustomSearchConfiguration customSearchConfiguration, @@ -141,7 +153,11 @@ public static SearchRequestHandler getBuilder( ImmutableList.copyOf(entitySpecs), k -> new SearchRequestHandler( - entitySpecs, configs, customSearchConfiguration, queryFilterRewriteChain)); + entityRegistry, + entitySpecs, + configs, + customSearchConfiguration, + queryFilterRewriteChain)); } private Map> getSearchableAnnotations() { @@ -589,4 +605,87 @@ private HighlightBuilder getValidatedHighlighter(Collection fieldsToHigh .forEach(highlightBuilder::field); return highlightBuilder; } + + /** + * Calculate the field types based on annotations if available, with fallback to ES mappings + * + * @param entitySpecs entitySepcts + * @return Field name to annotation field types + */ + private static Map> buildSearchableFieldTypes( + @Nonnull EntityRegistry entityRegistry, @Nonnull List entitySpecs) { + return entitySpecs.stream() + .flatMap( + entitySpec -> { + Map> annotationFieldTypes = + entitySpec.getSearchableFieldTypes(); + + // fallback to mappings + Map> rawMappingTypes = + ((Map) + MappingsBuilder.getMappings(entityRegistry, entitySpec) + .getOrDefault("properties", Map.of())) + .entrySet().stream() + .filter( + entry -> + !annotationFieldTypes.containsKey(entry.getKey()) + && ((Map) entry.getValue()).containsKey(TYPE)) + .collect( + Collectors.toMap( + Map.Entry::getKey, e -> (Map) e.getValue())); + + Map> mappingFieldTypes = + rawMappingTypes.entrySet().stream() + .map( + entry -> Map.entry(entry.getKey(), entry.getValue().get(TYPE).toString())) + .map( + entry -> + Map.entry( + entry.getKey(), + fallbackMappingToAnnotation(entry.getValue()).stream() + .collect(Collectors.toSet()))) + .filter(entry -> !entry.getValue().isEmpty()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + // aliases - pull from annotations + Map> aliasFieldTypes = + rawMappingTypes.entrySet().stream() + .filter( + entry -> ALIAS_FIELD_TYPE.equals(entry.getValue().get(TYPE).toString())) + .map( + entry -> + Map.entry( + entry.getKey(), + annotationFieldTypes.getOrDefault( + entry.getValue().get(PATH).toString(), + Collections.emptySet()))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + return Stream.concat( + annotationFieldTypes.entrySet().stream(), + Stream.concat( + mappingFieldTypes.entrySet().stream(), aliasFieldTypes.entrySet().stream())); + }) + .collect( + Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (set1, set2) -> { + set1.addAll(set2); + return set1; + })); + } + + private static Set fallbackMappingToAnnotation( + @Nonnull String mappingType) { + switch (mappingType) { + case KEYWORD_FIELD_TYPE: + return Set.of(SearchableAnnotation.FieldType.KEYWORD); + case DATE_FIELD_TYPE: + return Set.of(SearchableAnnotation.FieldType.DATETIME); + case OBJECT_FIELD_TYPE: + return Set.of(SearchableAnnotation.FieldType.OBJECT); + } + return Collections.emptySet(); + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java index 3cb7e8bd3fb1b1..88382f14b35e2a 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java @@ -297,12 +297,12 @@ public void testSearchService() throws Exception { assertEquals(searchResult.getEntities().get(0).getEntity(), urn); assertEquals(searchResult.getEntities().get(0).getDegree().intValue(), 1); - searchResult = searchAcrossLineage(QueryUtils.newFilter("degree.keyword", "1"), TEST1); + searchResult = searchAcrossLineage(QueryUtils.newFilter("degree", "1"), TEST1); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn); assertEquals(searchResult.getEntities().get(0).getDegree().intValue(), 1); - searchResult = searchAcrossLineage(QueryUtils.newFilter("degree.keyword", "2"), TEST1); + searchResult = searchAcrossLineage(QueryUtils.newFilter("degree", "2"), TEST1); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); clearCache(false); @@ -573,13 +573,13 @@ public void testScrollAcrossLineage() throws Exception { assertEquals(scrollResult.getEntities().get(0).getDegree().intValue(), 1); assertNull(scrollResult.getScrollId()); - scrollResult = scrollAcrossLineage(QueryUtils.newFilter("degree.keyword", "1"), TEST1); + scrollResult = scrollAcrossLineage(QueryUtils.newFilter("degree", "1"), TEST1); assertEquals(scrollResult.getNumEntities().intValue(), 1); assertEquals(scrollResult.getEntities().get(0).getEntity(), urn); assertEquals(scrollResult.getEntities().get(0).getDegree().intValue(), 1); assertNull(scrollResult.getScrollId()); - scrollResult = scrollAcrossLineage(QueryUtils.newFilter("degree.keyword", "2"), TEST1); + scrollResult = scrollAcrossLineage(QueryUtils.newFilter("degree", "2"), TEST1); assertEquals(scrollResult.getNumEntities().intValue(), 0); assertEquals(scrollResult.getEntities().size(), 0); assertNull(scrollResult.getScrollId()); @@ -679,14 +679,14 @@ public void testLightningSearchService() throws Exception { verify(lineageSearchService, times(1)) .getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); - searchResult = searchAcrossLineage(QueryUtils.newFilter("degree.keyword", "1"), testStar); + searchResult = searchAcrossLineage(QueryUtils.newFilter("degree", "1"), testStar); assertEquals(searchResult.getNumEntities().intValue(), 1); assertEquals(searchResult.getEntities().get(0).getEntity(), urn); assertEquals(searchResult.getEntities().get(0).getDegree().intValue(), 1); verify(lineageSearchService, times(2)) .getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); - searchResult = searchAcrossLineage(QueryUtils.newFilter("degree.keyword", "2"), testStar); + searchResult = searchAcrossLineage(QueryUtils.newFilter("degree", "2"), testStar); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); verify(lineageSearchService, times(3)) @@ -931,7 +931,7 @@ public void testLightningSearchService() throws Exception { CriterionArray critArr = new CriterionArray(ImmutableList.of(platform1Crit)); conCritArr.add(new ConjunctiveCriterion().setAnd(critArr)); Criterion degreeCrit = - new Criterion().setField("degree.keyword").setValue("2").setCondition(Condition.EQUAL); + new Criterion().setField("degree").setValue("2").setCondition(Condition.EQUAL); conCritArr.add( new ConjunctiveCriterion().setAnd(new CriterionArray(ImmutableList.of(degreeCrit)))); Filter filter = new Filter().setOr(conCritArr); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java index ea9658e9c585eb..15a4712bb93f55 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.search.indexbuilder; import static com.linkedin.metadata.Constants.*; +import static org.mockito.Mockito.mock; import static org.testng.Assert.*; import com.datahub.test.TestRefEntity; @@ -28,7 +29,8 @@ public class MappingsBuilderTest { @Test public void testMappingsBuilder() { - Map result = MappingsBuilder.getMappings(TestEntitySpecBuilder.getSpec()); + Map result = + MappingsBuilder.getMappings(mock(EntityRegistry.class), TestEntitySpecBuilder.getSpec()); assertEquals(result.size(), 1); Map properties = (Map) result.get("properties"); assertEquals(properties.size(), 22); @@ -176,7 +178,7 @@ public void testMappingsBuilder() { public void testGetMappingsWithStructuredProperty() throws URISyntaxException { // Baseline comparison: Mappings with no structured props Map resultWithoutStructuredProps = - MappingsBuilder.getMappings(TestEntitySpecBuilder.getSpec()); + MappingsBuilder.getMappings(mock(EntityRegistry.class), TestEntitySpecBuilder.getSpec()); // Test that a structured property that does not apply to the entity does not alter the mappings StructuredPropertyDefinition structPropNotForThisEntity = @@ -188,6 +190,7 @@ public void testGetMappingsWithStructuredProperty() throws URISyntaxException { .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); Map resultWithOnlyUnrelatedStructuredProp = MappingsBuilder.getMappings( + mock(EntityRegistry.class), TestEntitySpecBuilder.getSpec(), List.of( Pair.of( @@ -209,6 +212,7 @@ public void testGetMappingsWithStructuredProperty() throws URISyntaxException { .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); Map resultWithOnlyRelatedStructuredProp = MappingsBuilder.getMappings( + mock(EntityRegistry.class), TestEntitySpecBuilder.getSpec(), List.of( Pair.of( @@ -243,6 +247,7 @@ public void testGetMappingsWithStructuredProperty() throws URISyntaxException { // Test that only structured properties that apply are included Map resultWithBothStructuredProps = MappingsBuilder.getMappings( + mock(EntityRegistry.class), TestEntitySpecBuilder.getSpec(), List.of( Pair.of( @@ -258,7 +263,7 @@ public void testGetMappingsWithStructuredProperty() throws URISyntaxException { public void testGetMappingsWithStructuredPropertyV1() throws URISyntaxException { // Baseline comparison: Mappings with no structured props Map resultWithoutStructuredProps = - MappingsBuilder.getMappings(TestEntitySpecBuilder.getSpec()); + MappingsBuilder.getMappings(mock(EntityRegistry.class), TestEntitySpecBuilder.getSpec()); // Test that a structured property that does not apply to the entity does not alter the mappings StructuredPropertyDefinition structPropNotForThisEntity = @@ -270,6 +275,7 @@ public void testGetMappingsWithStructuredPropertyV1() throws URISyntaxException .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); Map resultWithOnlyUnrelatedStructuredProp = MappingsBuilder.getMappings( + mock(EntityRegistry.class), TestEntitySpecBuilder.getSpec(), List.of( Pair.of( @@ -291,6 +297,7 @@ public void testGetMappingsWithStructuredPropertyV1() throws URISyntaxException .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); Map resultWithOnlyRelatedStructuredProp = MappingsBuilder.getMappings( + mock(EntityRegistry.class), TestEntitySpecBuilder.getSpec(), List.of( Pair.of( @@ -325,6 +332,7 @@ public void testGetMappingsWithStructuredPropertyV1() throws URISyntaxException // Test that only structured properties that apply are included Map resultWithBothStructuredProps = MappingsBuilder.getMappings( + mock(EntityRegistry.class), TestEntitySpecBuilder.getSpec(), List.of( Pair.of( @@ -449,9 +457,9 @@ public void testGetMappingsForStructuredPropertyV1() throws URISyntaxException { @Test public void testRefMappingsBuilder() { EntityRegistry entityRegistry = getTestEntityRegistry(); - MappingsBuilder.setEntityRegistry(entityRegistry); + EntitySpec entitySpec = new EntitySpecBuilder().buildEntitySpec(new TestRefEntity().schema()); - Map result = MappingsBuilder.getMappings(entitySpec); + Map result = MappingsBuilder.getMappings(entityRegistry, entitySpec); assertEquals(result.size(), 1); Map properties = (Map) result.get("properties"); assertEquals(properties.size(), 7); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java index 7da0a14f212799..70695639c48c2e 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchRequestHandlerTest.java @@ -97,7 +97,11 @@ public void testDatasetFieldsAndHighlights() { EntitySpec entitySpec = operationContext.getEntityRegistry().getEntitySpec("dataset"); SearchRequestHandler datasetHandler = SearchRequestHandler.getBuilder( - entitySpec, testQueryConfig, null, QueryFilterRewriteChain.EMPTY); + operationContext.getEntityRegistry(), + entitySpec, + testQueryConfig, + null, + QueryFilterRewriteChain.EMPTY); /* Ensure efficient query performance, we do not expect upstream/downstream/fineGrained lineage @@ -118,6 +122,7 @@ public void testCustomHighlights() { EntitySpec entitySpec = operationContext.getEntityRegistry().getEntitySpec("dataset"); SearchRequestHandler requestHandler = SearchRequestHandler.getBuilder( + operationContext.getEntityRegistry(), TestEntitySpecBuilder.getSpec(), testQueryConfig, null, @@ -147,7 +152,11 @@ public void testCustomHighlights() { public void testSearchRequestHandlerHighlightingTurnedOff() { SearchRequestHandler requestHandler = SearchRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), testQueryConfig, null, QueryFilterRewriteChain.EMPTY); + operationContext.getEntityRegistry(), + TestEntitySpecBuilder.getSpec(), + testQueryConfig, + null, + QueryFilterRewriteChain.EMPTY); SearchRequest searchRequest = requestHandler.getSearchRequest( operationContext.withSearchFlags( @@ -188,7 +197,11 @@ public void testSearchRequestHandlerHighlightingTurnedOff() { public void testSearchRequestHandler() { SearchRequestHandler requestHandler = SearchRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), testQueryConfig, null, QueryFilterRewriteChain.EMPTY); + operationContext.getEntityRegistry(), + TestEntitySpecBuilder.getSpec(), + testQueryConfig, + null, + QueryFilterRewriteChain.EMPTY); SearchRequest searchRequest = requestHandler.getSearchRequest( operationContext.withSearchFlags( @@ -252,7 +265,11 @@ public void testSearchRequestHandler() { public void testAggregationsInSearch() { SearchRequestHandler requestHandler = SearchRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), testQueryConfig, null, QueryFilterRewriteChain.EMPTY); + operationContext.getEntityRegistry(), + TestEntitySpecBuilder.getSpec(), + testQueryConfig, + null, + QueryFilterRewriteChain.EMPTY); final String nestedAggString = String.format("_entityType%stextFieldOverride", AGGREGATION_SEPARATOR_CHAR); SearchRequest searchRequest = @@ -321,7 +338,11 @@ public void testFilteredSearch() { final SearchRequestHandler requestHandler = SearchRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), testQueryConfig, null, QueryFilterRewriteChain.EMPTY); + operationContext.getEntityRegistry(), + TestEntitySpecBuilder.getSpec(), + testQueryConfig, + null, + QueryFilterRewriteChain.EMPTY); final BoolQueryBuilder testQuery = constructFilterQuery(requestHandler, false); @@ -702,7 +723,11 @@ private BoolQueryBuilder getQuery(final Criterion filterCriterion) { final SearchRequestHandler requestHandler = SearchRequestHandler.getBuilder( - TestEntitySpecBuilder.getSpec(), testQueryConfig, null, QueryFilterRewriteChain.EMPTY); + operationContext.getEntityRegistry(), + TestEntitySpecBuilder.getSpec(), + testQueryConfig, + null, + QueryFilterRewriteChain.EMPTY); return (BoolQueryBuilder) requestHandler diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java index 78e274063b6c73..f10c6e59bae7c1 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java +++ b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java @@ -2,7 +2,6 @@ import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.AUTO_COMPLETE_ENTITY_TYPES; import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.SEARCHABLE_ENTITY_TYPES; -import static org.mockito.Mockito.mock; import com.datahub.authentication.Authentication; import com.datahub.plugins.auth.authorization.Authorizer; @@ -14,7 +13,6 @@ import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.SearchableEntityType; import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.config.DataHubAppConfiguration; import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.linkedin.metadata.graph.LineageDirection; @@ -188,7 +186,7 @@ public static LineageSearchResult lineage( .collect(Collectors.toList()), "*", hops, - ResolverUtils.buildFilter(filters, List.of(), mock(AspectRetriever.class)), + ResolverUtils.buildFilter(filters, List.of()), null, 0, 100); diff --git a/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java b/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java index f56cbc36e4a66c..f80581bf6fbdde 100644 --- a/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java +++ b/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java @@ -92,7 +92,11 @@ private void writeSearchCsv(WebApplicationContext ctx, PrintWriter pw) { EntitySpec entitySpec = entitySpecOpt.get(); SearchRequest searchRequest = SearchRequestHandler.getBuilder( - entitySpec, searchConfiguration, null, queryFilterRewriteChain) + entityRegistry, + entitySpec, + searchConfiguration, + null, + queryFilterRewriteChain) .getSearchRequest( getOperationContext(ctx) .withSearchFlags(