From b873f6bbe39193dd18a8096765b483716205ab6f Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Tue, 2 Apr 2024 14:11:18 -0500 Subject: [PATCH 1/2] feat(lineage): give via and paths in entity lineage response --- .../load/EntityLineageResultResolver.java | 21 +++++++++++++++++++ .../graphql/types/mappers/MapperUtils.java | 9 ++++++++ .../UrnSearchAcrossLineageResultsMapper.java | 10 +-------- .../src/main/resources/entity.graphql | 9 ++++++++ .../src/main/resources/search.graphql | 5 +++++ .../graph/elastic/ESGraphQueryDAO.java | 8 ++++--- .../metadata/search/LineageSearchService.java | 1 + .../metadata/graph/LineageRelationship.pdl | 5 +++++ .../metadata/search/LineageSearchEntity.pdl | 5 +++++ ...com.linkedin.entity.entities.snapshot.json | 5 +++++ ...nkedin.lineage.relationships.snapshot.json | 5 +++++ 11 files changed, 71 insertions(+), 12 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java index 8de18ec01e6dc2..ee3eeb62008b49 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java @@ -1,8 +1,11 @@ package com.linkedin.datahub.graphql.resolvers.load; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static com.linkedin.datahub.graphql.types.mappers.MapperUtils.*; import com.datahub.authorization.AuthorizationConfiguration; +import com.linkedin.common.UrnArray; +import com.linkedin.common.UrnArrayArray; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.SetMode; @@ -21,7 +24,9 @@ import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import io.datahubproject.metadata.services.RestrictedService; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @@ -156,6 +161,22 @@ private LineageRelationship mapEntityRelationship( result.setUpdatedActor(UrnToEntityMapper.map(context, updatedActor)); } result.setIsManual(lineageRelationship.hasIsManual() && lineageRelationship.isIsManual()); + if (lineageRelationship.getPaths() != null) { + UrnArrayArray paths = lineageRelationship.getPaths(); + List viaNodes = new ArrayList<>(); + result.setPaths( + paths.stream().map(path -> mapPath(context, path)).collect(Collectors.toList())); + for (UrnArray path : paths) { + int pathLength = path.size(); + if (pathLength > 2) { + for (int i = 1; i < pathLength - 1; i++) { + Urn via = path.get(i); + viaNodes.add(via.toString()); + } + } + } + result.setVia(viaNodes); + } return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java index 3cae0155a86db5..6bda333256a4c9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java @@ -3,9 +3,11 @@ import static com.linkedin.datahub.graphql.util.SearchInsightsUtil.*; import static com.linkedin.metadata.utils.SearchUtil.*; +import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.AggregationMetadata; +import com.linkedin.datahub.graphql.generated.EntityPath; import com.linkedin.datahub.graphql.generated.FacetMetadata; import com.linkedin.datahub.graphql.generated.MatchedField; import com.linkedin.datahub.graphql.generated.SearchResult; @@ -104,4 +106,11 @@ public static SearchSuggestion mapSearchSuggestion( return new SearchSuggestion( suggestion.getText(), suggestion.getScore(), Math.toIntExact(suggestion.getFrequency())); } + + public static EntityPath mapPath(@Nullable final QueryContext context, UrnArray path) { + EntityPath entityPath = new EntityPath(); + entityPath.setPath( + path.stream().map(p -> UrnToEntityMapper.map(context, p)).collect(Collectors.toList())); + return entityPath; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java index b85303909c0801..ca363deb90c4de 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java @@ -3,11 +3,9 @@ import static com.linkedin.datahub.graphql.types.mappers.MapperUtils.*; import static com.linkedin.datahub.graphql.util.SearchInsightsUtil.*; -import com.linkedin.common.UrnArray; import com.linkedin.data.template.RecordTemplate; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Entity; -import com.linkedin.datahub.graphql.generated.EntityPath; import com.linkedin.datahub.graphql.generated.FreshnessStats; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResult; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResults; @@ -72,13 +70,7 @@ private SearchAcrossLineageResult mapResult( .setDegree(searchEntity.getDegree()) .setDegrees(new ArrayList<>(searchEntity.getDegrees())) .setExplored(Boolean.TRUE.equals(searchEntity.isExplored())) + .setIgnoredAsHop(Boolean.TRUE.equals(searchEntity.isIgnoredAsHop())) .build(); } - - private EntityPath mapPath(@Nullable final QueryContext context, UrnArray path) { - EntityPath entityPath = new EntityPath(); - entityPath.setPath( - path.stream().map(p -> UrnToEntityMapper.map(context, p)).collect(Collectors.toList())); - return entityPath; - } } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index b750d206261018..c82ffe2e0b7c3a 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -1331,6 +1331,15 @@ type LineageRelationship { """ isManual: Boolean + """ + The paths traversed for this relationship + """ + paths: [EntityPath] + + """ + Any in-between via nodes for this relationship + """ + via: [String] } """ diff --git a/datahub-graphql-core/src/main/resources/search.graphql b/datahub-graphql-core/src/main/resources/search.graphql index 13c1ff2e8a7648..499ac3a0860d40 100644 --- a/datahub-graphql-core/src/main/resources/search.graphql +++ b/datahub-graphql-core/src/main/resources/search.graphql @@ -747,6 +747,11 @@ type SearchAcrossLineageResult { """ explored: Boolean! + """ + Whether this relationship was ignored as a hop + """ + ignoredAsHop: Boolean! + } """ diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java index ea8d8fea54633c..bdcbf020ecf781 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java @@ -389,9 +389,11 @@ private Stream processOneHopLineage( || platformMatches( lineageRelationship.getEntity(), ignoreAsHops.get(entityType))))) - .map(LineageRelationship::getEntity) - .forEach(additionalCurrentLevel::add); - ; + .forEach( + lineageRelationship -> { + additionalCurrentLevel.add(lineageRelationship.getEntity()); + lineageRelationship.setIgnoredAsHop(true); + }); if (!additionalCurrentLevel.isEmpty()) { Stream ignoreAsHopUrns = processOneHopLineage( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java index bb316f6f2b41c3..94f56fec2acc93 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java @@ -739,6 +739,7 @@ private LineageSearchEntity buildLineageSearchEntity( entity.setDegrees(lineageRelationship.getDegrees()); } entity.setExplored(Boolean.TRUE.equals(lineageRelationship.isExplored())); + entity.setIgnoredAsHop(Boolean.TRUE.equals(lineageRelationship.isIgnoredAsHop())); } return entity; } diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/LineageRelationship.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/LineageRelationship.pdl index a169157955e67b..552dd7323b5517 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/LineageRelationship.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/graph/LineageRelationship.pdl @@ -72,4 +72,9 @@ record LineageRelationship { * Marks this relationship as explored during the graph walk */ explored: optional boolean + + /** + * Whether this relationship was ignored as a hop while performing the graph walk + */ + ignoredAsHop: optional boolean } diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/search/LineageSearchEntity.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/search/LineageSearchEntity.pdl index fdfc8b2d53291c..3fd8a48c6bf5ee 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/search/LineageSearchEntity.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/search/LineageSearchEntity.pdl @@ -34,4 +34,9 @@ record LineageSearchEntity includes SearchEntity { * Marks an entity as having been explored for as a part of the graph walk */ explored: optional boolean + + /** + * Whether this relationship was ignored as a hop while performing the graph walk + */ + ignoredAsHop: optional boolean } \ No newline at end of file diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 4915f06ffe5d2a..43845a5fbbf6a3 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -6210,6 +6210,11 @@ "type" : "boolean", "doc" : "Marks an entity as having been explored for as a part of the graph walk", "optional" : true + }, { + "name" : "ignoredAsHop", + "type" : "boolean", + "doc" : "Whether this relationship was ignored as a hop while performing the graph walk", + "optional" : true } ] } }, diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json index 00b3c925d0e731..3886faffadedbb 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.lineage.relationships.snapshot.json @@ -182,6 +182,11 @@ "type" : "boolean", "doc" : "Marks this relationship as explored during the graph walk", "optional" : true + }, { + "name" : "ignoredAsHop", + "type" : "boolean", + "doc" : "Whether this relationship was ignored as a hop while performing the graph walk", + "optional" : true } ] } }, From 82c1f7d2ed24e00ef49413081899daf8a1229dc4 Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Tue, 2 Apr 2024 17:30:36 -0500 Subject: [PATCH 2/2] remove via --- .../load/EntityLineageResultResolver.java | 14 -------------- .../src/main/resources/entity.graphql | 5 ----- 2 files changed, 19 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java index ee3eeb62008b49..e28ec3dbb870fa 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java @@ -4,7 +4,6 @@ import static com.linkedin.datahub.graphql.types.mappers.MapperUtils.*; import com.datahub.authorization.AuthorizationConfiguration; -import com.linkedin.common.UrnArray; import com.linkedin.common.UrnArrayArray; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; @@ -24,9 +23,7 @@ import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import io.datahubproject.metadata.services.RestrictedService; -import java.util.ArrayList; import java.util.HashSet; -import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @@ -163,19 +160,8 @@ private LineageRelationship mapEntityRelationship( result.setIsManual(lineageRelationship.hasIsManual() && lineageRelationship.isIsManual()); if (lineageRelationship.getPaths() != null) { UrnArrayArray paths = lineageRelationship.getPaths(); - List viaNodes = new ArrayList<>(); result.setPaths( paths.stream().map(path -> mapPath(context, path)).collect(Collectors.toList())); - for (UrnArray path : paths) { - int pathLength = path.size(); - if (pathLength > 2) { - for (int i = 1; i < pathLength - 1; i++) { - Urn via = path.get(i); - viaNodes.add(via.toString()); - } - } - } - result.setVia(viaNodes); } return result; diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index c82ffe2e0b7c3a..106148c425791c 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -1335,11 +1335,6 @@ type LineageRelationship { The paths traversed for this relationship """ paths: [EntityPath] - - """ - Any in-between via nodes for this relationship - """ - via: [String] } """