From 05093e5a24be0138deacb343f16094a4594c090f Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Mon, 7 Jun 2021 11:14:07 -0700 Subject: [PATCH 1/8] versioned aspects --- .../datahub/graphql/GmsClientFactory.java | 11 +++++++ .../datahub/graphql/GmsGraphQLEngine.java | 15 +++++++-- .../resolvers/load/AspectResolver.java | 32 +++++++++++++++++++ .../graphql/types/aspect/AspectType.java | 30 +++++++++++++++++ .../src/main/resources/gms.graphql | 4 +++ datahub-web-react/src/graphql/dataset.graphql | 32 +++++++++++++++++++ .../metadata/entity/ebean/EbeanAspectDao.java | 15 +++++++++ .../entity/ebean/EbeanEntityService.java | 3 ++ 8 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsClientFactory.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsClientFactory.java index 29ffdd606cfdc..4e03de54ead17 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsClientFactory.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsClientFactory.java @@ -81,4 +81,15 @@ public static EntityClient getEntitiesClient() { } return _entities; } + + public static EntityClient getAspectType() { + if (_entities == null) { + synchronized (GmsClientFactory.class) { + if (_entities == null) { + _entities = new EntityClient(REST_CLIENT); + } + } + } + return _entities; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index a0bb7a1492d12..172a0e4a96117 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -12,6 +12,7 @@ import com.linkedin.datahub.graphql.generated.RelatedDataset; import com.linkedin.datahub.graphql.generated.SearchResult; import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata; +import com.linkedin.datahub.graphql.resolvers.load.AspectResolver; import com.linkedin.datahub.graphql.resolvers.load.EntityTypeResolver; import com.linkedin.datahub.graphql.resolvers.load.LoadableTypeBatchResolver; import com.linkedin.datahub.graphql.resolvers.mutate.MutableTypeResolver; @@ -21,6 +22,7 @@ import com.linkedin.datahub.graphql.types.EntityType; import com.linkedin.datahub.graphql.types.LoadableType; import com.linkedin.datahub.graphql.types.SearchableEntityType; +import com.linkedin.datahub.graphql.types.aspect.AspectType; import com.linkedin.datahub.graphql.types.chart.ChartType; import com.linkedin.datahub.graphql.types.corpuser.CorpUserType; import com.linkedin.datahub.graphql.types.corpgroup.CorpGroupType; @@ -97,8 +99,10 @@ public class GmsGraphQLEngine { GmsClientFactory.getRelationshipsClient() ); public static final GlossaryTermType GLOSSARY_TERM_TYPE = new GlossaryTermType(GmsClientFactory.getEntitiesClient()); + public static final AspectType ASPECT_TYPE = new AspectType(); - /** + + /** * Configures the graph objects that can be fetched primary key. */ public static final List> ENTITY_TYPES = ImmutableList.of( @@ -127,7 +131,11 @@ public class GmsGraphQLEngine { /** * Configures all graph objects */ - public static final List> LOADABLE_TYPES = Stream.concat(ENTITY_TYPES.stream(), RELATIONSHIP_TYPES.stream()).collect(Collectors.toList()); + public static final List> LOADABLE_TYPES = Stream.concat( + ENTITY_TYPES.stream(), + RELATIONSHIP_TYPES.stream()).collect(Collectors.toList(), + Stream.of(ASPECT_TYPE), + ); /** * Configures the graph objects for owner @@ -293,6 +301,9 @@ private static void configureDatasetResolvers(final RuntimeWiring.Builder builde UPSTREAM_LINEAGE_TYPE, (env) -> ((Entity) env.getSource()).getUrn())) ) + .dataFetcher("schema", new AuthenticatedResolver<>( + new AspectResolver<>()) + ) ) .type("Owner", typeWiring -> typeWiring .dataFetcher("owner", new AuthenticatedResolver<>( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java new file mode 100644 index 0000000000000..59a0ec7bffe69 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java @@ -0,0 +1,32 @@ +package com.linkedin.datahub.graphql.resolvers.load; + +import com.linkedin.datahub.graphql.types.LoadableType; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import org.dataloader.DataLoader; + + +/** + * Generic GraphQL resolver responsible for + * + * 1. Retrieving a single input urn. + * 2. Resolving a single {@link LoadableType}. + * + * Note that this resolver expects that {@link DataLoader}s were registered + * for the provided {@link LoadableType} under the name provided by {@link LoadableType#name()} + * + * @param the generated GraphQL POJO corresponding to the resolved type. + */ +public class AspectResolver implements DataFetcher> { + + public AspectResolver() { + } + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) { + final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader("aspect"); + return loader.load(urn); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java new file mode 100644 index 0000000000000..40db95de10d2b --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java @@ -0,0 +1,30 @@ +package com.linkedin.datahub.graphql.types.aspect; + +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Aspects; +import com.linkedin.datahub.graphql.types.LoadableType; +import java.util.List; +import javax.annotation.Nonnull; + + +public class AspectType implements LoadableType { + + /** + * Returns generated GraphQL class associated with the type + */ + @Override + public Class objectClass() { + return Aspects.class; + } + + /** + * Retrieves an list of entities given a list of urn strings. The list returned is expected to + * be of same length of the list of urns, where nulls are provided in place of an entity object if an entity cannot be found. + * @param urns to retrieve + * @param context the {@link QueryContext} corresponding to the request. + */ + @Override + public List batchLoad(@Nonnull List urns, @Nonnull QueryContext context) throws Exception { + return null; + } +} diff --git a/datahub-graphql-core/src/main/resources/gms.graphql b/datahub-graphql-core/src/main/resources/gms.graphql index 38a044e3279e1..45c32c01551c7 100644 --- a/datahub-graphql-core/src/main/resources/gms.graphql +++ b/datahub-graphql-core/src/main/resources/gms.graphql @@ -296,6 +296,8 @@ type Dataset implements EntityWithRelationships & Entity { The structured glossary terms associated with the dataset """ glossaryTerms: GlossaryTerms + + versionedSchema(version: Int, aspect: String): Schema } type GlossaryTerm implements Entity { @@ -581,6 +583,8 @@ type KeyValueSchema { valueSchema: String! } +union Aspects = Schema | GlobalTags + type SchemaField { """ Flattened name of the field computed from jsonPath field diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index 7d23bf4204ebe..0d73102425816 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -79,6 +79,38 @@ query getDataset($urn: String!) { } primaryKeys } + pastSchema: schema(version: -1) { + datasetUrn + name + platformUrn + version + hash + platformSchema { + ... on TableSchema { + schema + } + ... on KeyValueSchema { + keySchema + valueSchema + } + } + fields { + fieldPath + jsonPath + nullable + description + type + nativeDataType + recursive + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + } + primaryKeys + } editableSchemaMetadata { editableSchemaFieldInfo { fieldPath diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java index 8e546024764f2..16f03a0eed4e3 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java @@ -178,6 +178,21 @@ protected EbeanAspectV2 getLatestAspect(@Nonnull final String urn, @Nonnull fina return _server.find(EbeanAspectV2.class, key); } + @Nullable + public long getMaxVersion(@Nonnull final String urn, @Nonnull final String aspectName) { + validateConnection(); + List result = _server.find(EbeanAspectV2.class) + .where() + .eq("urn", "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)").eq("aspect", "browsePaths") + .orderBy() + .desc("version") + .findList(); + if (result.size() == 0) { + return -1; + } + return result.get(0).getKey().getVersion(); + } + @Nullable public EbeanAspectV2 getAspect(@Nonnull final String urn, @Nonnull final String aspectName, final long version) { validateConnection(); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index 7b405f69a48ab..01b21c55775f4 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -86,6 +86,9 @@ public Map> getLatestAspects(@Nonnull final Set u @Override @Nullable public RecordTemplate getAspect(@Nonnull final Urn urn, @Nonnull final String aspectName, @Nonnull long version) { + if (version < 0) { + version = _entityDao.getMaxVersion(urn.toString(), aspectName) - version + 1; + } final EbeanAspectV2.PrimaryKey primaryKey = new EbeanAspectV2.PrimaryKey(urn.toString(), aspectName, version); final Optional maybeAspect = Optional.ofNullable(_entityDao.getAspect(primaryKey)); return maybeAspect From b0719873c13231bad3e34a982de0fd62f899335f Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Wed, 9 Jun 2021 10:59:55 -0700 Subject: [PATCH 2/8] versioned aspects cont --- .../datahub/graphql/AspectLoadKey.java | 17 +++++ .../datahub/graphql/GmsGraphQLEngine.java | 29 +++++++-- .../resolvers/load/AspectResolver.java | 33 ++++++++-- .../type/AspectInterfaceTypeResolver.java | 20 ++++++ .../datahub/graphql/types/LoadableType.java | 5 +- .../graphql/types/aspect/AspectType.java | 21 ++---- .../graphql/types/chart/ChartType.java | 11 ++-- .../types/corpgroup/CorpGroupType.java | 6 +- .../graphql/types/corpuser/CorpUserType.java | 6 +- .../types/dashboard/DashboardType.java | 13 ++-- .../graphql/types/dataflow/DataFlowType.java | 12 ++-- .../graphql/types/datajob/DataJobType.java | 12 ++-- .../types/dataplatform/DataPlatformType.java | 4 +- .../graphql/types/dataset/DatasetType.java | 44 +++++++++++-- .../types/dataset/mappers/DatasetMapper.java | 2 +- .../mappers/DatasetSnapshotMapper.java | 2 + .../types/glossary/GlossaryTermType.java | 10 ++- .../DataFlowDataJobsRelationshipsType.java | 6 +- .../types/lineage/DownstreamLineageType.java | 5 +- .../types/lineage/UpstreamLineageType.java | 5 +- .../graphql/types/mlmodel/MLModelType.java | 8 ++- .../datahub/graphql/types/tag/TagType.java | 9 ++- .../src/main/resources/gms.graphql | 65 +++++++++++++++++-- .../extractor/SnapshotToAspectMap.java | 53 +++++++++++++++ 24 files changed, 318 insertions(+), 80 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/AspectInterfaceTypeResolver.java create mode 100644 metadata-io/src/main/java/com/linkedin/metadata/extractor/SnapshotToAspectMap.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java new file mode 100644 index 0000000000000..3be600868549a --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java @@ -0,0 +1,17 @@ +package com.linkedin.datahub.graphql; + +import lombok.Data; + + +@Data +public class AspectLoadKey { + private String aspectName; + private String urn; + private Long version; + + public AspectLoadKey(String _urn, String _aspectName, Long _version) { + urn = _urn; + version = _version; + aspectName = _aspectName; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 172a0e4a96117..0aaedf0c96582 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql; import com.google.common.collect.ImmutableList; +import com.linkedin.datahub.graphql.generated.Aspect; import com.linkedin.datahub.graphql.generated.Chart; import com.linkedin.datahub.graphql.generated.ChartInfo; import com.linkedin.datahub.graphql.generated.DashboardInfo; @@ -16,6 +17,7 @@ import com.linkedin.datahub.graphql.resolvers.load.EntityTypeResolver; import com.linkedin.datahub.graphql.resolvers.load.LoadableTypeBatchResolver; import com.linkedin.datahub.graphql.resolvers.mutate.MutableTypeResolver; +import com.linkedin.datahub.graphql.resolvers.type.AspectInterfaceTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.HyperParameterValueTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.ResultsTypeResolver; import com.linkedin.datahub.graphql.types.BrowsableEntityType; @@ -52,9 +54,12 @@ import com.linkedin.datahub.graphql.types.lineage.DataFlowDataJobsRelationshipsType; import com.linkedin.datahub.graphql.types.glossary.GlossaryTermType; +import graphql.execution.DataFetcherResult; +import graphql.schema.DataFetcher; import graphql.schema.idl.RuntimeWiring; import org.apache.commons.io.IOUtils; import org.dataloader.BatchLoaderContextProvider; +import org.dataloader.BatchLoaderEnvironment; import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; @@ -133,9 +138,8 @@ public class GmsGraphQLEngine { */ public static final List> LOADABLE_TYPES = Stream.concat( ENTITY_TYPES.stream(), - RELATIONSHIP_TYPES.stream()).collect(Collectors.toList(), - Stream.of(ASPECT_TYPE), - ); + RELATIONSHIP_TYPES.stream() + ).collect(Collectors.toList()); /** * Configures the graph objects for owner @@ -204,6 +208,7 @@ public static GraphQLEngine.Builder builder() { return GraphQLEngine.builder() .addSchema(schema()) .addDataLoaders(loaderSuppliers(LOADABLE_TYPES)) + .addDataLoader("Aspect", (context) -> createAspectLoader(context)) .configureRuntimeWiring(GmsGraphQLEngine::configureRuntimeWiring); } @@ -302,7 +307,7 @@ private static void configureDatasetResolvers(final RuntimeWiring.Builder builde (env) -> ((Entity) env.getSource()).getUrn())) ) .dataFetcher("schema", new AuthenticatedResolver<>( - new AspectResolver<>()) + new AspectResolver()) ) ) .type("Owner", typeWiring -> typeWiring @@ -470,6 +475,7 @@ private static void configureTypeResolvers(final RuntimeWiring.Builder builder) .type("HyperParameterValueType", typeWiring -> typeWiring .typeResolver(new HyperParameterValueTypeResolver()) ) + .type("Aspect", typeWiring -> typeWiring.typeResolver(new AspectInterfaceTypeResolver())) .type("ResultsType", typeWiring -> typeWiring .typeResolver(new ResultsTypeResolver())); } @@ -530,7 +536,7 @@ private static void configureDataJobResolvers(final RuntimeWiring.Builder builde } - private static DataLoader createDataLoader(final LoadableType graphType, final QueryContext queryContext) { + private static DataLoader> createDataLoader(final LoadableType graphType, final QueryContext queryContext) { BatchLoaderContextProvider contextProvider = () -> queryContext; DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setBatchLoaderContextProvider(contextProvider); return DataLoader.newDataLoader((keys, context) -> CompletableFuture.supplyAsync(() -> { @@ -542,6 +548,19 @@ private static DataLoader createDataLoader(final LoadableType }), loaderOptions); } + private static DataLoader> createAspectLoader(final QueryContext queryContext) { + BatchLoaderContextProvider contextProvider = () -> queryContext; + DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setBatchLoaderContextProvider(contextProvider); + AspectType aspectType = new AspectType(); + return DataLoader.newDataLoader((keys, context) -> CompletableFuture.supplyAsync(() -> { + try { + return aspectType.batchLoad(keys, context.getContext()); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to retrieve entities of type Aspect", e)); + } + }), loaderOptions); + } + private GmsGraphQLEngine() { } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java index 59a0ec7bffe69..c893d866aeb11 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java @@ -1,10 +1,15 @@ package com.linkedin.datahub.graphql.resolvers.load; +import com.linkedin.data.Data; +import com.linkedin.data.DataMap; +import com.linkedin.datahub.graphql.AspectLoadKey; +import com.linkedin.datahub.graphql.generated.Aspect; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.types.LoadableType; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; +import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.function.Function; import org.dataloader.DataLoader; @@ -17,16 +22,32 @@ * Note that this resolver expects that {@link DataLoader}s were registered * for the provided {@link LoadableType} under the name provided by {@link LoadableType#name()} * - * @param the generated GraphQL POJO corresponding to the resolved type. */ -public class AspectResolver implements DataFetcher> { +public class AspectResolver implements DataFetcher> { public AspectResolver() { } @Override - public CompletableFuture get(DataFetchingEnvironment environment) { - final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader("aspect"); - return loader.load(urn); + public CompletableFuture get(DataFetchingEnvironment environment) { + final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader("aspect"); + String fieldName = environment.getField().getName(); + Long version = environment.getArgument("version"); + String urn = ((Entity) environment.getSource()).getUrn(); + + Object localContext = environment.getLocalContext(); + // if we have context & the version is 0, we should try to retrieve it from the fetched entity + if (localContext == null && version == 0 || version == null) { + if (localContext instanceof Map) { + DataMap aspect = ((Map) localContext).getOrDefault(fieldName, null); + if (aspect != null) { + return CompletableFuture.completedFuture(AspectMapper.map(aspect)); + } + } + } + + // if the aspect is not in the cache, we need to fetch it + return loader.load(new AspectLoadKey(urn, fieldName, version)); + // return loader.load(urn); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/AspectInterfaceTypeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/AspectInterfaceTypeResolver.java new file mode 100644 index 0000000000000..f7894c8ed474d --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/AspectInterfaceTypeResolver.java @@ -0,0 +1,20 @@ +package com.linkedin.datahub.graphql.resolvers.type; + +import graphql.TypeResolutionEnvironment; +import graphql.schema.GraphQLObjectType; +import graphql.schema.TypeResolver; + +/** + * Responsible for resolving the {@link com.linkedin.datahub.graphql.generated.Entity} interface type. + */ +public class AspectInterfaceTypeResolver implements TypeResolver { + + public AspectInterfaceTypeResolver() { } + + @Override + public GraphQLObjectType getType(TypeResolutionEnvironment env) { + String fieldName = env.getField().getName(); + String typeName = fieldName.substring(0, 1).toUpperCase() + fieldName.substring(1); + return env.getSchema().getObjectType(typeName); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java index c9d9315668389..6ee0faedf6642 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/LoadableType.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableList; import com.linkedin.datahub.graphql.QueryContext; +import graphql.execution.DataFetcherResult; import javax.annotation.Nonnull; import java.util.List; @@ -31,7 +32,7 @@ default String name() { * @param urn to retrieve * @param context the {@link QueryContext} corresponding to the request. */ - default T load(@Nonnull final String urn, @Nonnull final QueryContext context) throws Exception { + default DataFetcherResult load(@Nonnull final String urn, @Nonnull final QueryContext context) throws Exception { return batchLoad(ImmutableList.of(urn), context).get(0); }; @@ -42,6 +43,6 @@ default T load(@Nonnull final String urn, @Nonnull final QueryContext context) t * @param urns to retrieve * @param context the {@link QueryContext} corresponding to the request. */ - List batchLoad(@Nonnull final List urns, @Nonnull final QueryContext context) throws Exception; + List> batchLoad(@Nonnull final List urns, @Nonnull final QueryContext context) throws Exception; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java index 40db95de10d2b..0a93ebe8b2614 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java @@ -1,30 +1,21 @@ package com.linkedin.datahub.graphql.types.aspect; +import com.linkedin.datahub.graphql.AspectLoadKey; import com.linkedin.datahub.graphql.QueryContext; -import com.linkedin.datahub.graphql.generated.Aspects; -import com.linkedin.datahub.graphql.types.LoadableType; +import com.linkedin.datahub.graphql.generated.Aspect; +import graphql.execution.DataFetcherResult; import java.util.List; import javax.annotation.Nonnull; -public class AspectType implements LoadableType { - - /** - * Returns generated GraphQL class associated with the type - */ - @Override - public Class objectClass() { - return Aspects.class; - } - +public class AspectType { /** * Retrieves an list of entities given a list of urn strings. The list returned is expected to * be of same length of the list of urns, where nulls are provided in place of an entity object if an entity cannot be found. - * @param urns to retrieve + * @param keys to retrieve * @param context the {@link QueryContext} corresponding to the request. */ - @Override - public List batchLoad(@Nonnull List urns, @Nonnull QueryContext context) throws Exception { + public List> batchLoad(@Nonnull List keys, @Nonnull QueryContext context) throws Exception { return null; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java index ec154623be967..f46206855e51a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java @@ -32,6 +32,7 @@ import com.linkedin.metadata.snapshot.Snapshot; import com.linkedin.r2.RemoteInvocationException; +import graphql.execution.DataFetcherResult; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URISyntaxException; @@ -68,7 +69,7 @@ public Class objectClass() { } @Override - public List batchLoad(@Nonnull List urns, @Nonnull QueryContext context) throws Exception { + public List> batchLoad(@Nonnull List urns, @Nonnull QueryContext context) throws Exception { final List chartUrns = urns.stream() .map(this::getChartUrn) .collect(Collectors.toList()); @@ -84,7 +85,8 @@ public List batchLoad(@Nonnull List urns, @Nonnull QueryContext c gmsResults.add(chartMap.getOrDefault(urn, null)); } return gmsResults.stream() - .map(gmsChart -> gmsChart == null ? null : ChartSnapshotMapper.map(gmsChart.getValue().getChartSnapshot())) + .map(gmsChart -> gmsChart == null ? null + : DataFetcherResult.newResult().data(ChartSnapshotMapper.map(gmsChart.getValue().getChartSnapshot())).build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load Charts", e); @@ -129,7 +131,8 @@ public BrowseResults browse(@Nonnull List path, start, count); final List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List charts = batchLoad(urns, context); + final List charts = batchLoad(urns, context).stream().map(chartResult -> chartResult.getData()).collect( + Collectors.toList()); final BrowseResults browseResults = new BrowseResults(); browseResults.setStart(result.getFrom()); browseResults.setCount(result.getPageSize()); @@ -168,6 +171,6 @@ public Chart update(@Nonnull ChartUpdateInput input, @Nonnull QueryContext conte throw new RuntimeException(String.format("Failed to write entity with urn %s", input.getUrn()), e); } - return load(input.getUrn(), context); + return load(input.getUrn(), context).getData(); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java index 4fbae386b0105..79e88be9ccc65 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/CorpGroupType.java @@ -17,6 +17,7 @@ import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.SearchResult; +import graphql.execution.DataFetcherResult; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URISyntaxException; @@ -46,7 +47,7 @@ public EntityType type() { } @Override - public List batchLoad(final List urns, final QueryContext context) { + public List> batchLoad(final List urns, final QueryContext context) { try { final List corpGroupUrns = urns .stream() @@ -61,7 +62,8 @@ public List batchLoad(final List urns, final QueryContext con results.add(corpGroupMap.getOrDefault(urn, null)); } return results.stream() - .map(gmsCorpGroup -> gmsCorpGroup == null ? null : CorpGroupSnapshotMapper.map(gmsCorpGroup.getValue().getCorpGroupSnapshot())) + .map(gmsCorpGroup -> gmsCorpGroup == null ? null + : DataFetcherResult.newResult().data(CorpGroupSnapshotMapper.map(gmsCorpGroup.getValue().getCorpGroupSnapshot())).build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load CorpGroup", e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java index 567769360013a..c8032760d6000 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/CorpUserType.java @@ -18,6 +18,7 @@ import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.SearchResult; +import graphql.execution.DataFetcherResult; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URISyntaxException; @@ -47,7 +48,7 @@ public EntityType type() { } @Override - public List batchLoad(final List urns, final QueryContext context) { + public List> batchLoad(final List urns, final QueryContext context) { try { final List corpUserUrns = urns .stream() @@ -62,7 +63,8 @@ public List batchLoad(final List urns, final QueryContext cont results.add(corpUserMap.getOrDefault(urn, null)); } return results.stream() - .map(gmsCorpUser -> gmsCorpUser == null ? null : CorpUserSnapshotMapper.map(gmsCorpUser.getValue().getCorpUserSnapshot())) + .map(gmsCorpUser -> gmsCorpUser == null ? null + : DataFetcherResult.newResult().data(CorpUserSnapshotMapper.map(gmsCorpUser.getValue().getCorpUserSnapshot())).build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load Datasets", e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java index c69cfdfd3a2ef..beb74f61079c0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java @@ -34,6 +34,7 @@ import com.linkedin.metadata.snapshot.Snapshot; import com.linkedin.r2.RemoteInvocationException; +import graphql.execution.DataFetcherResult; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URISyntaxException; @@ -70,7 +71,7 @@ public Class objectClass() { } @Override - public List batchLoad(@Nonnull List urns, @Nonnull QueryContext context) throws Exception { + public List> batchLoad(@Nonnull List urns, @Nonnull QueryContext context) throws Exception { final List dashboardUrns = urns.stream() .map(this::getDashboardUrn) .collect(Collectors.toList()); @@ -86,8 +87,9 @@ public List batchLoad(@Nonnull List urns, @Nonnull QueryConte gmsResults.add(dashboardMap.getOrDefault(urn, null)); } return gmsResults.stream() - .map(gmsDashboard -> gmsDashboard == null ? null : DashboardSnapshotMapper.map( - gmsDashboard.getValue().getDashboardSnapshot())) + .map(gmsDashboard -> gmsDashboard == null ? null + : DataFetcherResult.newResult().data(DashboardSnapshotMapper.map( + gmsDashboard.getValue().getDashboardSnapshot())).build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load Dashboards", e); @@ -130,7 +132,8 @@ public BrowseResults browse(@Nonnull List path, start, count); final List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List dashboards = batchLoad(urns, context); + final List dashboards = batchLoad(urns, context).stream().map(dashboardDataFetcherResult -> dashboardDataFetcherResult.getData()).collect( + Collectors.toList()); final BrowseResults browseResults = new BrowseResults(); browseResults.setStart(result.getFrom()); browseResults.setCount(result.getPageSize()); @@ -169,6 +172,6 @@ public Dashboard update(@Nonnull DashboardUpdateInput input, @Nonnull QueryConte throw new RuntimeException(String.format("Failed to write entity with urn %s", input.getUrn()), e); } - return load(input.getUrn(), context); + return load(input.getUrn(), context).getData(); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java index 92e480cf3ccf6..06a47bbb31ad3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java @@ -35,6 +35,7 @@ import com.linkedin.metadata.snapshot.DataFlowSnapshot; import com.linkedin.metadata.snapshot.Snapshot; import com.linkedin.r2.RemoteInvocationException; +import graphql.execution.DataFetcherResult; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; @@ -73,7 +74,7 @@ public Class inputClass() { } @Override - public List batchLoad(final List urns, final QueryContext context) throws Exception { + public List> batchLoad(final List urns, final QueryContext context) throws Exception { final List dataFlowUrns = urns.stream() .map(this::getDataFlowUrn) .collect(Collectors.toList()); @@ -88,8 +89,8 @@ public List batchLoad(final List urns, final QueryContext cont .map(flowUrn -> dataFlowMap.getOrDefault(flowUrn, null)).collect(Collectors.toList()); return gmsResults.stream() - .map(gmsDataFlow -> gmsDataFlow == null ? null : DataFlowSnapshotMapper.map( - gmsDataFlow.getValue().getDataFlowSnapshot())) + .map(gmsDataFlow -> gmsDataFlow == null ? null : DataFetcherResult.newResult().data(DataFlowSnapshotMapper.map( + gmsDataFlow.getValue().getDataFlowSnapshot())).build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load DataFlows", e); @@ -138,7 +139,8 @@ public BrowseResults browse(@Nonnull List path, @Nullable List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List dataFlows = batchLoad(urns, context); + final List dataFlows = batchLoad(urns, context).stream().map(dataFlow -> dataFlow.getData()).collect( + Collectors.toList()); final BrowseResults browseResults = new BrowseResults(); browseResults.setStart(result.getFrom()); browseResults.setCount(result.getPageSize()); @@ -187,6 +189,6 @@ public DataFlow update(@Nonnull DataFlowUpdateInput input, @Nonnull QueryContext throw new RuntimeException(String.format("Failed to write entity with urn %s", input.getUrn()), e); } - return load(input.getUrn(), context); + return load(input.getUrn(), context).getData(); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java index 5c3d9822fdd25..208b5eb8ef040 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java @@ -34,6 +34,7 @@ import com.linkedin.metadata.query.SearchResult; import com.linkedin.metadata.snapshot.DataJobSnapshot; import com.linkedin.metadata.snapshot.Snapshot; +import graphql.execution.DataFetcherResult; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; @@ -74,7 +75,7 @@ public Class inputClass() { } @Override - public List batchLoad(final List urns, final QueryContext context) throws Exception { + public List> batchLoad(final List urns, final QueryContext context) throws Exception { final List dataJobUrns = urns.stream() .map(this::getDataJobUrn) .collect(Collectors.toList()); @@ -89,7 +90,9 @@ public List batchLoad(final List urns, final QueryContext conte .map(jobUrn -> dataJobMap.getOrDefault(jobUrn, null)).collect(Collectors.toList()); return gmsResults.stream() - .map(gmsDataJob -> gmsDataJob == null ? null : DataJobSnapshotMapper.map(gmsDataJob.getValue().getDataJobSnapshot())) + .map(gmsDataJob -> gmsDataJob == null ? null + : DataFetcherResult.newResult().data( + DataJobSnapshotMapper.map(gmsDataJob.getValue().getDataJobSnapshot())).build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load DataJobs", e); @@ -139,7 +142,8 @@ public BrowseResults browse(@Nonnull List path, @Nullable List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List dataJobs = batchLoad(urns, context); + final List dataJobs = batchLoad(urns, context).stream().map(dataFetcherResult -> dataFetcherResult.getData()).collect( + Collectors.toList()); final BrowseResults browseResults = new BrowseResults(); browseResults.setStart(result.getFrom()); browseResults.setCount(result.getPageSize()); @@ -192,6 +196,6 @@ public DataJob update(@Nonnull DataJobUpdateInput input, @Nonnull QueryContext c throw new RuntimeException(String.format("Failed to write entity with urn %s", input.getUrn()), e); } - return load(input.getUrn(), context); + return load(input.getUrn(), context).getData(); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java index 09ae409285641..03c8be3c30d09 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java @@ -7,6 +7,7 @@ import com.linkedin.datahub.graphql.types.dataplatform.mappers.DataPlatformMapper; import com.linkedin.dataplatform.client.DataPlatforms; +import graphql.execution.DataFetcherResult; import java.net.URISyntaxException; import java.util.List; import java.util.Map; @@ -27,7 +28,7 @@ public Class objectClass() { } @Override - public List batchLoad(final List urns, final QueryContext context) { + public List> batchLoad(final List urns, final QueryContext context) { try { if (_urnToPlatform == null) { _urnToPlatform = _dataPlatformsClient.getAllPlatforms().stream() @@ -36,6 +37,7 @@ public List batchLoad(final List urns, final QueryContext } return urns.stream() .map(key -> _urnToPlatform.containsKey(key) ? _urnToPlatform.get(key) : getUnknownDataPlatform(key)) + .map(dataPlatform -> DataFetcherResult.newResult().data(dataPlatform).build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load DataPlatforms", e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index a5820239c2626..743cf1e02ce56 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -29,12 +29,14 @@ import com.linkedin.dataset.client.Datasets; import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.Entity; +import com.linkedin.metadata.extractor.SnapshotToAspectMap; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; import com.linkedin.metadata.snapshot.Snapshot; import com.linkedin.r2.RemoteInvocationException; +import graphql.execution.DataFetcherResult; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.ArrayList; @@ -71,8 +73,34 @@ public EntityType type() { return EntityType.DATASET; } + public List legacyBatchLoad(final List urns, final QueryContext context) { + + final List datasetUrns = urns.stream() + .map(DatasetUtils::getDatasetUrn) + .collect(Collectors.toList()); + + try { + final Map datasetMap = _datasetsClient.batchGet(datasetUrns + .stream() + .filter(Objects::nonNull) + .collect(Collectors.toSet())); + + final List gmsResults = new ArrayList<>(); + for (DatasetUrn urn : datasetUrns) { + gmsResults.add(datasetMap.getOrDefault(urn, null)); + } + return gmsResults.stream() + .map(gmsDataset -> + gmsDataset == null ? null : DatasetSnapshotMapper.map(gmsDataset.getValue().getDatasetSnapshot())) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to batch load Datasets", e); + } + } + + @Override - public List batchLoad(final List urns, final QueryContext context) { + public List> batchLoad(final List urns, final QueryContext context) { final List datasetUrns = urns.stream() .map(DatasetUtils::getDatasetUrn) @@ -89,9 +117,13 @@ public List batchLoad(final List urns, final QueryContext conte gmsResults.add(datasetMap.getOrDefault(urn, null)); } return gmsResults.stream() - .map(gmsDataset -> gmsDataset == null ? null : DatasetSnapshotMapper.map( - gmsDataset.getValue().getDatasetSnapshot())) - .collect(Collectors.toList()); + .map(gmsDataset -> + gmsDataset == null ? null : DataFetcherResult.newResult() + .data(DatasetSnapshotMapper.map(gmsDataset.getValue().getDatasetSnapshot())) + .localContext(SnapshotToAspectMap.extractAspectMap(gmsDataset.getValue().getDatasetSnapshot())) + .build() + ) + .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load Datasets", e); } @@ -134,7 +166,7 @@ public BrowseResults browse(@Nonnull List path, start, count); final List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List datasets = batchLoad(urns, context); + final List datasets = legacyBatchLoad(urns, context); final BrowseResults browseResults = new BrowseResults(); browseResults.setStart(result.getFrom()); browseResults.setCount(result.getPageSize()); @@ -187,6 +219,6 @@ public Dataset update(@Nonnull DatasetUpdateInput input, @Nonnull QueryContext c throw new RuntimeException(String.format("Failed to write entity with urn %s", input.getUrn()), e); } - return load(input.getUrn(), context); + return load(input.getUrn(), context).getData(); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index 66dcc5b839b9f..e4d873729c040 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -46,7 +46,7 @@ public Dataset apply(@Nonnull final com.linkedin.dataset.Dataset dataset) { result.setExternalUrl(dataset.getExternalUrl().toString()); } if (dataset.hasSchemaMetadata()) { - result.setSchema(SchemaMetadataMapper.map(dataset.getSchemaMetadata())); + // result.setSchema(SchemaMetadataMapper.map(dataset.getSchemaMetadata())); } if (dataset.hasEditableSchemaMetadata()) { result.setEditableSchemaMetadata(EditableSchemaMetadataMapper.map(dataset.getEditableSchemaMetadata())); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetSnapshotMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetSnapshotMapper.java index de4d977d3a9bc..65d5eaf218a4d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetSnapshotMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetSnapshotMapper.java @@ -70,9 +70,11 @@ public Dataset apply(@Nonnull final DatasetSnapshot dataset) { } else if (aspect instanceof Ownership) { result.setOwnership(OwnershipMapper.map((Ownership) aspect)); } else if (aspect instanceof SchemaMetadata) { + /* result.setSchema( SchemaMetadataMapper.map((SchemaMetadata) aspect) ); + */ } else if (aspect instanceof Status) { result.setStatus(StatusMapper.map((Status) aspect)); } else if (aspect instanceof GlobalTags) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java index 88ceeac4345b7..031a049cd8d33 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java @@ -26,6 +26,7 @@ import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; +import graphql.execution.DataFetcherResult; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.ArrayList; @@ -58,7 +59,7 @@ public EntityType type() { } @Override - public List batchLoad(final List urns, final QueryContext context) { + public List> batchLoad(final List urns, final QueryContext context) { final List glossaryTermUrns = urns.stream() .map(GlossaryTermUtils::getGlossaryTermUrn) .collect(Collectors.toList()); @@ -76,7 +77,9 @@ public List batchLoad(final List urns, final QueryContext return gmsResults.stream() .map(gmsGlossaryTerm -> gmsGlossaryTerm == null ? null - : GlossaryTermSnapshotMapper.map(gmsGlossaryTerm.getValue().getGlossaryTermSnapshot())) + : DataFetcherResult.newResult().data( + GlossaryTermSnapshotMapper.map( + gmsGlossaryTerm.getValue().getGlossaryTermSnapshot())).build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load GlossaryTerms", e); @@ -122,7 +125,8 @@ public BrowseResults browse(@Nonnull List path, start, count); final List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List glossaryTerms = batchLoad(urns, context); + final List glossaryTerms = batchLoad(urns, context).stream().map(term -> term.getData()).collect( + Collectors.toList()); final BrowseResults browseResults = new BrowseResults(); browseResults.setStart(result.getFrom()); browseResults.setCount(result.getPageSize()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/DataFlowDataJobsRelationshipsType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/DataFlowDataJobsRelationshipsType.java index 3a8a03189182e..0ef2a97be5ceb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/DataFlowDataJobsRelationshipsType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/DataFlowDataJobsRelationshipsType.java @@ -8,6 +8,7 @@ import com.linkedin.metadata.query.RelationshipDirection; import com.linkedin.r2.RemoteInvocationException; +import graphql.execution.DataFetcherResult; import java.net.URISyntaxException; import java.util.List; import java.util.stream.Collectors; @@ -27,14 +28,13 @@ public Class objectClass() { } @Override - public List batchLoad(final List keys, final QueryContext context) { - + public List> batchLoad(final List keys, final QueryContext context) { try { return keys.stream().map(urn -> { try { com.linkedin.common.EntityRelationships relationships = _relationshipsClient.getRelationships(urn, _direction, "IsPartOf"); - return DataFlowDataJobsRelationshipsMapper.map(relationships); + return DataFetcherResult.newResult().data(DataFlowDataJobsRelationshipsMapper.map(relationships)).build(); } catch (RemoteInvocationException | URISyntaxException e) { throw new RuntimeException(String.format("Failed to batch load DataJobs for DataFlow %s", urn), e); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/DownstreamLineageType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/DownstreamLineageType.java index 1974fea6e63d4..ba6ca3c4caf6d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/DownstreamLineageType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/DownstreamLineageType.java @@ -8,6 +8,7 @@ import com.linkedin.metadata.query.RelationshipDirection; import com.linkedin.r2.RemoteInvocationException; +import graphql.execution.DataFetcherResult; import java.net.URISyntaxException; import java.util.List; import java.util.stream.Collectors; @@ -27,14 +28,14 @@ public Class objectClass() { } @Override - public List batchLoad(final List keys, final QueryContext context) { + public List> batchLoad(final List keys, final QueryContext context) { try { return keys.stream().map(urn -> { try { com.linkedin.common.EntityRelationships relationships = _lineageClient.getLineage(urn, _direction); - return DownstreamEntityRelationshipsMapper.map(relationships); + return DataFetcherResult.newResult().data(DownstreamEntityRelationshipsMapper.map(relationships)).build(); } catch (RemoteInvocationException | URISyntaxException e) { throw new RuntimeException(String.format("Failed to batch load DownstreamLineage for entity %s", urn), e); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/UpstreamLineageType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/UpstreamLineageType.java index c69e616662242..8a21168d667d3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/UpstreamLineageType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/lineage/UpstreamLineageType.java @@ -8,6 +8,7 @@ import com.linkedin.metadata.query.RelationshipDirection; import com.linkedin.r2.RemoteInvocationException; +import graphql.execution.DataFetcherResult; import java.net.URISyntaxException; import java.util.List; import java.util.stream.Collectors; @@ -27,14 +28,14 @@ public Class objectClass() { } @Override - public List batchLoad(final List keys, final QueryContext context) { + public List> batchLoad(final List keys, final QueryContext context) { try { return keys.stream().map(urn -> { try { com.linkedin.common.EntityRelationships relationships = _lineageClient.getLineage(urn, _direction); - return UpstreamEntityRelationshipsMapper.map(relationships); + return DataFetcherResult.newResult().data(UpstreamEntityRelationshipsMapper.map(relationships)).build(); } catch (RemoteInvocationException | URISyntaxException e) { throw new RuntimeException(String.format("Failed to batch load DownstreamLineage for entity %s", urn), e); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java index a77aa2e00b21b..3e30f45930a01 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java @@ -7,6 +7,7 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.Entity; import com.linkedin.metadata.query.SearchResult; +import graphql.execution.DataFetcherResult; import java.util.List; import java.util.Map; import java.util.Objects; @@ -49,7 +50,7 @@ public Class objectClass() { } @Override - public List batchLoad(final List urns, final QueryContext context) throws Exception { + public List> batchLoad(final List urns, final QueryContext context) throws Exception { final List mlModelUrns = urns.stream() .map(MLModelUtils::getMLModelUrn) .collect(Collectors.toList()); @@ -64,8 +65,9 @@ public List batchLoad(final List urns, final QueryContext conte .map(modelUrn -> mlModelMap.getOrDefault(modelUrn, null)).collect(Collectors.toList()); return gmsResults.stream() - .map(gmsMlModel -> gmsMlModel == null ? null : MLModelSnapshotMapper.map( - gmsMlModel.getValue().getMLModelSnapshot())) + .map(gmsMlModel -> gmsMlModel == null ? null + : DataFetcherResult.newResult().data(MLModelSnapshotMapper.map( + gmsMlModel.getValue().getMLModelSnapshot())).build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load MLModels", e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java index 53ccc84aca5c0..2812469745460 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java @@ -37,6 +37,7 @@ import com.linkedin.r2.RemoteInvocationException; import com.linkedin.tag.TagProperties; +import graphql.execution.DataFetcherResult; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URISyntaxException; @@ -72,7 +73,7 @@ public Class inputClass() { } @Override - public List batchLoad(final List urns, final QueryContext context) { + public List> batchLoad(final List urns, final QueryContext context) { final List tagUrns = urns.stream() .map(this::getTagUrn) @@ -89,7 +90,9 @@ public List batchLoad(final List urns, final QueryContext context) gmsResults.add(tagMap.getOrDefault(urn, null)); } return gmsResults.stream() - .map(gmsTag -> gmsTag == null ? null : TagSnapshotMapper.map(gmsTag.getValue().getTagSnapshot())) + .map(gmsTag -> gmsTag == null ? null + : DataFetcherResult.newResult() + .data(TagSnapshotMapper.map(gmsTag.getValue().getTagSnapshot())).build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load Tags", e); @@ -155,7 +158,7 @@ public Tag update(@Nonnull TagUpdate input, @Nonnull QueryContext context) throw throw new RuntimeException(String.format("Failed to write entity with urn %s", input.getUrn()), e); } - return load(input.getUrn(), context); + return load(input.getUrn(), context).getData(); } private TagUrn getTagUrn(final String urnStr) { diff --git a/datahub-graphql-core/src/main/resources/gms.graphql b/datahub-graphql-core/src/main/resources/gms.graphql index 45c32c01551c7..c1a275c9cc6b6 100644 --- a/datahub-graphql-core/src/main/resources/gms.graphql +++ b/datahub-graphql-core/src/main/resources/gms.graphql @@ -21,6 +21,11 @@ interface Entity { type: EntityType! } +interface Aspect { + aspectVersion: Long + createdAt: Long +} + interface EntityWithRelationships implements Entity { """ GMS Entity urn @@ -265,7 +270,12 @@ type Dataset implements EntityWithRelationships & Entity { """ Schema metadata of the dataset """ - schema: Schema + schema: Schema @deprecated(reason: "Use `schemaMetadata`") + + """ + Schema metadata of the dataset + """ + schemaMetadata(version: Int): SchemaMetadata """ Editable schema metadata of the dataset @@ -296,8 +306,6 @@ type Dataset implements EntityWithRelationships & Entity { The structured glossary terms associated with the dataset """ glossaryTerms: GlossaryTerms - - versionedSchema(version: Int, aspect: String): Schema } type GlossaryTerm implements Entity { @@ -518,7 +526,54 @@ type InstitutionalMemoryMetadata { created: AuditStamp! } -type Schema { +type SchemaMetadata implements Aspect { + aspectVersion: Long + createdAt: Long + """ + Dataset this schema metadata is associated with + """ + datasetUrn: String + + """ + Schema name + """ + name: String! + + """ + Platform this schema metadata is associated with + """ + platformUrn: String! + + """ + The version of the GMS Schema metadata + """ + version: Long! + + """ + The cluster this schema metadata is derived from + """ + cluster: String + + """ + The SHA1 hash of the schema content + """ + hash: String! + + """ + The native schema in the datasets platform, schemaless if it was not provided + """ + platformSchema: PlatformSchema + """ + Client provided a list of fields from value schema + """ + fields: [SchemaField!]! + """ + Client provided list of fields that define primary keys to access record + """ + primaryKeys: [String!] +} + +type Schema implements Aspect { """ Dataset this schema metadata is associated with """ @@ -583,8 +638,6 @@ type KeyValueSchema { valueSchema: String! } -union Aspects = Schema | GlobalTags - type SchemaField { """ Flattened name of the field computed from jsonPath field diff --git a/metadata-io/src/main/java/com/linkedin/metadata/extractor/SnapshotToAspectMap.java b/metadata-io/src/main/java/com/linkedin/metadata/extractor/SnapshotToAspectMap.java new file mode 100644 index 0000000000000..3c71e369de05d --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/extractor/SnapshotToAspectMap.java @@ -0,0 +1,53 @@ +package com.linkedin.metadata.extractor; + +import com.google.common.collect.ImmutableList; +import com.linkedin.data.DataMap; +import com.linkedin.data.element.DataElement; +import com.linkedin.data.it.IterationOrder; +import com.linkedin.data.it.ObjectIterator; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.PegasusUtils; +import com.linkedin.metadata.models.FieldSpec; +import com.linkedin.util.Pair; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.collections.map.HashedMap; +import org.apache.commons.lang3.StringUtils; + + +/** + * Extracts fields from a RecordTemplate based on the appropriate {@link FieldSpec}. + */ +public class SnapshotToAspectMap { + private SnapshotToAspectMap() { + } + + /** + * Function to extract the fields that match the input fieldSpecs + */ + public static Map extractAspectMap(RecordTemplate snapshot) { + + final ObjectIterator iterator = new ObjectIterator(snapshot.data(), snapshot.schema(), IterationOrder.PRE_ORDER); + final Map aspectsByName = new HashMap<>(); + + for (DataElement dataElement = iterator.next(); dataElement != null; dataElement = iterator.next()) { + final PathSpec pathSpec = dataElement.getSchemaPathSpec(); + List pathComponents = pathSpec.getPathComponents(); + // three components representing /aspect/*/ + if (pathComponents.size() != 3) { + continue; + } + String aspectName = PegasusUtils.getAspectNameFromFullyQualifiedName(pathComponents.get(2)); + aspectsByName.put(aspectName, (DataMap) dataElement.getValue()); + } + + return aspectsByName; + } +} From fd831aa1e0908689133d12cb691bc6c5595b9dd1 Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Thu, 10 Jun 2021 15:08:45 -0700 Subject: [PATCH 3/8] checkpointing here w/ everything being functional --- .../datahub/graphql/GmsClientFactory.java | 12 +- .../datahub/graphql/GmsGraphQLEngine.java | 9 +- .../graphql/resolvers/ResolverUtils.java | 47 + .../resolvers/load/AspectResolver.java | 22 +- .../graphql/types/aspect/AspectMapper.java | 28 + .../graphql/types/aspect/AspectType.java | 31 +- .../graphql/types/dataset/DatasetType.java | 29 +- .../mappers/DatasetSnapshotMapper.java | 4 +- .../types/dataset/mappers/SchemaMapper.java | 34 + .../dataset/mappers/SchemaMetadataMapper.java | 17 +- .../src/main/resources/gms.graphql | 4 +- datahub-web-react/src/graphql/dataset.graphql | 4 +- .../com.linkedin.entity.aspects.restspec.json | 30 + .../com.linkedin.chart.charts.snapshot.json | 14 +- ...inkedin.dashboard.dashboards.snapshot.json | 15 +- ....linkedin.dataflow.dataFlows.snapshot.json | 24 +- ...om.linkedin.datajob.dataJobs.snapshot.json | 28 +- ...in.dataprocess.dataProcesses.snapshot.json | 18 +- ...om.linkedin.dataset.datasets.snapshot.json | 33 +- .../com.linkedin.entity.aspects.snapshot.json | 2558 +++++++++++++++++ ...com.linkedin.entity.entities.snapshot.json | 148 +- ...kedin.glossary.glossaryNodes.snapshot.json | 9 +- ...kedin.glossary.glossaryTerms.snapshot.json | 11 +- ...linkedin.identity.corpGroups.snapshot.json | 3 +- ....linkedin.identity.corpUsers.snapshot.json | 14 +- .../com.linkedin.ml.mlModels.snapshot.json | 18 +- .../com.linkedin.tag.tags.snapshot.json | 6 +- .../linkedin/entity/client/AspectClient.java | 41 + .../resources/entity/AspectResource.java | 54 + .../examples/mce_files/bootstrap_mce.json | 10 +- .../metadata/entity/EntityService.java | 6 + .../metadata/entity/ebean/EbeanAspectDao.java | 2 +- .../entity/ebean/EbeanEntityService.java | 38 + .../extractor/SnapshotToAspectMap.java | 16 +- .../com/linkedin/metadata/aspect/Aspect.pdl | 98 + .../metadata/aspect/AspectWithMetadata.pdl | 8 + 36 files changed, 3164 insertions(+), 279 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java create mode 100644 gms/api/src/main/idl/com.linkedin.entity.aspects.restspec.json create mode 100644 gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json create mode 100644 gms/client/src/main/java/com/linkedin/entity/client/AspectClient.java create mode 100644 gms/impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/Aspect.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/AspectWithMetadata.pdl diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsClientFactory.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsClientFactory.java index 4e03de54ead17..dd790d228d4e8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsClientFactory.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsClientFactory.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql; import com.linkedin.dataplatform.client.DataPlatforms; +import com.linkedin.entity.client.AspectClient; import com.linkedin.entity.client.EntityClient; import com.linkedin.lineage.client.Lineages; import com.linkedin.lineage.client.Relationships; @@ -34,6 +35,7 @@ public class GmsClientFactory { private static Lineages _lineages; private static Relationships _relationships; private static EntityClient _entities; + private static AspectClient _aspects; private GmsClientFactory() { } @@ -82,14 +84,14 @@ public static EntityClient getEntitiesClient() { return _entities; } - public static EntityClient getAspectType() { - if (_entities == null) { + public static AspectClient getAspectsClient() { + if (_aspects == null) { synchronized (GmsClientFactory.class) { - if (_entities == null) { - _entities = new EntityClient(REST_CLIENT); + if (_aspects == null) { + _aspects = new AspectClient(REST_CLIENT); } } } - return _entities; + return _aspects; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 0aaedf0c96582..1f797d7e76de8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -55,11 +55,9 @@ import com.linkedin.datahub.graphql.types.glossary.GlossaryTermType; import graphql.execution.DataFetcherResult; -import graphql.schema.DataFetcher; import graphql.schema.idl.RuntimeWiring; import org.apache.commons.io.IOUtils; import org.dataloader.BatchLoaderContextProvider; -import org.dataloader.BatchLoaderEnvironment; import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; @@ -104,7 +102,7 @@ public class GmsGraphQLEngine { GmsClientFactory.getRelationshipsClient() ); public static final GlossaryTermType GLOSSARY_TERM_TYPE = new GlossaryTermType(GmsClientFactory.getEntitiesClient()); - public static final AspectType ASPECT_TYPE = new AspectType(); + public static final AspectType ASPECT_TYPE = new AspectType(GmsClientFactory.getAspectsClient()); /** @@ -306,7 +304,7 @@ private static void configureDatasetResolvers(final RuntimeWiring.Builder builde UPSTREAM_LINEAGE_TYPE, (env) -> ((Entity) env.getSource()).getUrn())) ) - .dataFetcher("schema", new AuthenticatedResolver<>( + .dataFetcher("schemaMetadata", new AuthenticatedResolver<>( new AspectResolver()) ) ) @@ -551,10 +549,9 @@ private static DataLoader> createDataLoader(fin private static DataLoader> createAspectLoader(final QueryContext queryContext) { BatchLoaderContextProvider contextProvider = () -> queryContext; DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setBatchLoaderContextProvider(contextProvider); - AspectType aspectType = new AspectType(); return DataLoader.newDataLoader((keys, context) -> CompletableFuture.supplyAsync(() -> { try { - return aspectType.batchLoad(keys, context.getContext()); + return ASPECT_TYPE.batchLoad(keys, context.getContext()); } catch (Exception e) { throw new RuntimeException(String.format("Failed to retrieve entities of type Aspect", e)); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java index fe444103a6b39..f4860f6a3426d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java @@ -1,9 +1,17 @@ package com.linkedin.datahub.graphql.resolvers; import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.data.DataMap; +import com.linkedin.data.element.DataElement; import com.linkedin.datahub.graphql.exception.ValidationException; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.types.aspect.AspectMapper; +import com.linkedin.metadata.aspect.AspectWithMetadata; +import graphql.schema.DataFetchingEnvironment; +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.CompletableFuture; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.Collections; @@ -11,6 +19,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.commons.lang.reflect.ConstructorUtils; public class ResolverUtils { @@ -55,4 +64,42 @@ public static Map buildFacetFilters(@Nullable List) localContext).getOrDefault(fieldName, null); + if (prefetchedAspect != null) { + try { + Object result = Class.forName(prefetchedAspect.getSchema().getUnionMemberKey()) + .cast((ConstructorUtils.getMatchingAccessibleConstructor( + Class.forName(prefetchedAspect.getSchema().getUnionMemberKey()), + new Class[]{DataMap.class}).newInstance(prefetchedAspect.getValue()))); + + AspectWithMetadata resultWithMetadata = new AspectWithMetadata(); + + resultWithMetadata.setAspect( + (com.linkedin.metadata.aspect.Aspect) + com.linkedin.metadata.aspect.Aspect.class.getMethod("create", result.getClass()) + .invoke(com.linkedin.metadata.aspect.Aspect.class, result)); + + resultWithMetadata.setVersion(0); + + return resultWithMetadata; + } catch (IllegalAccessException | InstantiationException | InvocationTargetException | ClassNotFoundException | NoSuchMethodException e) { + e.printStackTrace(); + } + } + } + } + return null; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java index c893d866aeb11..2d6402e879a7c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java @@ -1,14 +1,14 @@ package com.linkedin.datahub.graphql.resolvers.load; -import com.linkedin.data.Data; -import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.AspectLoadKey; import com.linkedin.datahub.graphql.generated.Aspect; import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.LoadableType; +import com.linkedin.datahub.graphql.types.aspect.AspectMapper; +import com.linkedin.metadata.aspect.AspectWithMetadata; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; -import java.util.Map; import java.util.concurrent.CompletableFuture; import org.dataloader.DataLoader; @@ -30,24 +30,18 @@ public AspectResolver() { @Override public CompletableFuture get(DataFetchingEnvironment environment) { - final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader("aspect"); + final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader("Aspect"); + String fieldName = environment.getField().getName(); Long version = environment.getArgument("version"); String urn = ((Entity) environment.getSource()).getUrn(); - Object localContext = environment.getLocalContext(); - // if we have context & the version is 0, we should try to retrieve it from the fetched entity - if (localContext == null && version == 0 || version == null) { - if (localContext instanceof Map) { - DataMap aspect = ((Map) localContext).getOrDefault(fieldName, null); - if (aspect != null) { - return CompletableFuture.completedFuture(AspectMapper.map(aspect)); - } - } + AspectWithMetadata aspectFromContext = ResolverUtils.getAspectFromLocalContext(environment); + if (aspectFromContext != null) { + return CompletableFuture.completedFuture(AspectMapper.map(aspectFromContext)); } // if the aspect is not in the cache, we need to fetch it return loader.load(new AspectLoadKey(urn, fieldName, version)); - // return loader.load(urn); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java new file mode 100644 index 0000000000000..7b8befd3a5ec2 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java @@ -0,0 +1,28 @@ +package com.linkedin.datahub.graphql.types.aspect; + +import com.linkedin.datahub.graphql.generated.Aspect; +import com.linkedin.datahub.graphql.types.dataset.mappers.SchemaMapper; +import com.linkedin.datahub.graphql.types.dataset.mappers.SchemaMetadataMapper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.schema.SchemaMetadata; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + + +public class AspectMapper implements ModelMapper { + + public static final AspectMapper INSTANCE = new AspectMapper(); + + public static Aspect map(@Nonnull final AspectWithMetadata restliAspect) { + return INSTANCE.apply(restliAspect); + } + + @Override + public Aspect apply(@Nonnull final AspectWithMetadata restliAspect) { + if (restliAspect.getAspect().isSchemaMetadata()) { + return SchemaMetadataMapper.map(restliAspect); + } + return null; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java index 0a93ebe8b2614..40a04ddbd83cd 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java @@ -1,14 +1,32 @@ package com.linkedin.datahub.graphql.types.aspect; +import com.linkedin.data.DataMap; +import com.linkedin.data.template.RecordTemplate; import com.linkedin.datahub.graphql.AspectLoadKey; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Aspect; +import com.linkedin.datahub.graphql.generated.DownstreamEntityRelationships; +import com.linkedin.datahub.graphql.types.relationships.mappers.DownstreamEntityRelationshipsMapper; +import com.linkedin.entity.Entity; +import com.linkedin.entity.client.AspectClient; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.lineage.client.Lineages; +import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.metadata.query.RelationshipDirection; +import com.linkedin.r2.RemoteInvocationException; import graphql.execution.DataFetcherResult; +import java.net.URISyntaxException; import java.util.List; +import java.util.stream.Collectors; import javax.annotation.Nonnull; public class AspectType { + private final AspectClient _aspectClient; + + public AspectType(final AspectClient aspectClient) { + _aspectClient = aspectClient; + } /** * Retrieves an list of entities given a list of urn strings. The list returned is expected to * be of same length of the list of urns, where nulls are provided in place of an entity object if an entity cannot be found. @@ -16,6 +34,17 @@ public class AspectType { * @param context the {@link QueryContext} corresponding to the request. */ public List> batchLoad(@Nonnull List keys, @Nonnull QueryContext context) throws Exception { - return null; + try { + return keys.stream().map(key -> { + try { + AspectWithMetadata entity = _aspectClient.getAspect(key.getUrn(), key.getAspectName(), key.getVersion()); + return DataFetcherResult.newResult().data(AspectMapper.map(entity)).build(); + } catch (RemoteInvocationException e) { + throw new RuntimeException(String.format("Failed to load Aspect for entity %s", key.getUrn()), e); + } + }).collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to batch load Aspects", e); + } } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index 743cf1e02ce56..e0d806769ec03 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -73,32 +73,6 @@ public EntityType type() { return EntityType.DATASET; } - public List legacyBatchLoad(final List urns, final QueryContext context) { - - final List datasetUrns = urns.stream() - .map(DatasetUtils::getDatasetUrn) - .collect(Collectors.toList()); - - try { - final Map datasetMap = _datasetsClient.batchGet(datasetUrns - .stream() - .filter(Objects::nonNull) - .collect(Collectors.toSet())); - - final List gmsResults = new ArrayList<>(); - for (DatasetUrn urn : datasetUrns) { - gmsResults.add(datasetMap.getOrDefault(urn, null)); - } - return gmsResults.stream() - .map(gmsDataset -> - gmsDataset == null ? null : DatasetSnapshotMapper.map(gmsDataset.getValue().getDatasetSnapshot())) - .collect(Collectors.toList()); - } catch (Exception e) { - throw new RuntimeException("Failed to batch load Datasets", e); - } - } - - @Override public List> batchLoad(final List urns, final QueryContext context) { @@ -166,7 +140,8 @@ public BrowseResults browse(@Nonnull List path, start, count); final List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List datasets = legacyBatchLoad(urns, context); + final List datasets = batchLoad(urns, context) + .stream().map(datasetDataFetcherResult -> datasetDataFetcherResult.getData()).collect(Collectors.toList()); final BrowseResults browseResults = new BrowseResults(); browseResults.setStart(result.getFrom()); browseResults.setCount(result.getPageSize()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetSnapshotMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetSnapshotMapper.java index 65d5eaf218a4d..792fbb7dc56f9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetSnapshotMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetSnapshotMapper.java @@ -70,11 +70,9 @@ public Dataset apply(@Nonnull final DatasetSnapshot dataset) { } else if (aspect instanceof Ownership) { result.setOwnership(OwnershipMapper.map((Ownership) aspect)); } else if (aspect instanceof SchemaMetadata) { - /* result.setSchema( - SchemaMetadataMapper.map((SchemaMetadata) aspect) + SchemaMapper.map((SchemaMetadata) aspect) ); - */ } else if (aspect instanceof Status) { result.setStatus(StatusMapper.map((Status) aspect)); } else if (aspect instanceof GlobalTags) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java new file mode 100644 index 0000000000000..05db591f42e63 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMapper.java @@ -0,0 +1,34 @@ +package com.linkedin.datahub.graphql.types.dataset.mappers; + +import com.linkedin.datahub.graphql.generated.Schema; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.schema.SchemaMetadata; + +import javax.annotation.Nonnull; +import java.util.stream.Collectors; + +public class SchemaMapper implements ModelMapper { + + public static final SchemaMapper INSTANCE = new SchemaMapper(); + + public static Schema map(@Nonnull final SchemaMetadata metadata) { + return INSTANCE.apply(metadata); + } + + @Override + public Schema apply(@Nonnull final com.linkedin.schema.SchemaMetadata input) { + final Schema result = new Schema(); + if (input.hasDataset()) { + result.setDatasetUrn(input.getDataset().toString()); + } + result.setName(input.getSchemaName()); + result.setPlatformUrn(input.getPlatform().toString()); + result.setVersion(input.getVersion()); + result.setCluster(input.getCluster()); + result.setHash(input.getHash()); + result.setPrimaryKeys(input.getPrimaryKeys()); + result.setFields(input.getFields().stream().map(SchemaFieldMapper::map).collect(Collectors.toList())); + result.setPlatformSchema(PlatformSchemaMapper.map(input.getPlatformSchema())); + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java index b36957cbd38ff..1a4b86b68c5ac 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java @@ -2,22 +2,26 @@ import com.linkedin.datahub.graphql.generated.Schema; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.metadata.aspect.AspectWithMetadata; import com.linkedin.schema.SchemaMetadata; - -import javax.annotation.Nonnull; import java.util.stream.Collectors; +import javax.annotation.Nonnull; -public class SchemaMetadataMapper implements ModelMapper { + +public class SchemaMetadataMapper implements ModelMapper { public static final SchemaMetadataMapper INSTANCE = new SchemaMetadataMapper(); - public static Schema map(@Nonnull final SchemaMetadata metadata) { + public static com.linkedin.datahub.graphql.generated.SchemaMetadata map(@Nonnull final AspectWithMetadata metadata) { return INSTANCE.apply(metadata); } @Override - public Schema apply(@Nonnull final com.linkedin.schema.SchemaMetadata input) { - final Schema result = new Schema(); + public com.linkedin.datahub.graphql.generated.SchemaMetadata apply(@Nonnull final AspectWithMetadata inputWithMetadata) { + SchemaMetadata input = inputWithMetadata.getAspect().getSchemaMetadata(); + final com.linkedin.datahub.graphql.generated.SchemaMetadata result = + new com.linkedin.datahub.graphql.generated.SchemaMetadata(); + if (input.hasDataset()) { result.setDatasetUrn(input.getDataset().toString()); } @@ -29,6 +33,7 @@ public Schema apply(@Nonnull final com.linkedin.schema.SchemaMetadata input) { result.setPrimaryKeys(input.getPrimaryKeys()); result.setFields(input.getFields().stream().map(SchemaFieldMapper::map).collect(Collectors.toList())); result.setPlatformSchema(PlatformSchemaMapper.map(input.getPlatformSchema())); + result.setVersion(inputWithMetadata.getVersion()); return result; } } diff --git a/datahub-graphql-core/src/main/resources/gms.graphql b/datahub-graphql-core/src/main/resources/gms.graphql index c1a275c9cc6b6..c39c48cf175c8 100644 --- a/datahub-graphql-core/src/main/resources/gms.graphql +++ b/datahub-graphql-core/src/main/resources/gms.graphql @@ -275,7 +275,7 @@ type Dataset implements EntityWithRelationships & Entity { """ Schema metadata of the dataset """ - schemaMetadata(version: Int): SchemaMetadata + schemaMetadata(version: Long): SchemaMetadata """ Editable schema metadata of the dataset @@ -573,7 +573,7 @@ type SchemaMetadata implements Aspect { primaryKeys: [String!] } -type Schema implements Aspect { +type Schema { """ Dataset this schema metadata is associated with """ diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index 0d73102425816..7d2248390fe8f 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -47,7 +47,7 @@ query getDataset($urn: String!) { } } } - schema { + schemaMetadata { datasetUrn name platformUrn @@ -79,7 +79,7 @@ query getDataset($urn: String!) { } primaryKeys } - pastSchema: schema(version: -1) { + pastSchema: schemaMetadata(version: -1) { datasetUrn name platformUrn diff --git a/gms/api/src/main/idl/com.linkedin.entity.aspects.restspec.json b/gms/api/src/main/idl/com.linkedin.entity.aspects.restspec.json new file mode 100644 index 0000000000000..c033f044b24d1 --- /dev/null +++ b/gms/api/src/main/idl/com.linkedin.entity.aspects.restspec.json @@ -0,0 +1,30 @@ +{ + "name" : "aspects", + "namespace" : "com.linkedin.entity", + "path" : "/aspects", + "schema" : "com.linkedin.metadata.aspect.AspectWithMetadata", + "doc" : "Single unified resource for fetching, updating, searching, & browsing DataHub entities\n\ngenerated from: com.linkedin.metadata.resources.entity.AspectResource", + "collection" : { + "identifier" : { + "name" : "aspectsId", + "type" : "string" + }, + "supports" : [ "get" ], + "methods" : [ { + "method" : "get", + "doc" : "Retrieves the value for an entity that is made up of latest versions of specified aspects.", + "parameters" : [ { + "name" : "aspect", + "type" : "string", + "optional" : true + }, { + "name" : "version", + "type" : "long", + "optional" : true + } ] + } ], + "entity" : { + "path" : "/aspects/{aspectsId}" + } + } +} \ No newline at end of file diff --git a/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json index 5f67b4a0b0349..0543e176f8867 100644 --- a/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json @@ -85,15 +85,14 @@ "type" : "string", "doc" : "Title of the chart", "Searchable" : { + "enableAutocomplete" : true, "fieldType" : "TEXT_PARTIAL" } }, { "name" : "description", "type" : "string", "doc" : "Detailed description about the chart", - "Searchable" : { - "fieldType" : "TEXT" - } + "Searchable" : { } }, { "name" : "lastModified", "type" : { @@ -340,8 +339,7 @@ "Searchable" : { "fieldName" : "tags", "fieldType" : "URN_PARTIAL", - "hasValuesFieldName" : "hasTags", - "queryByDefault" : true + "hasValuesFieldName" : "hasTags" } } ] } @@ -367,7 +365,8 @@ "Searchable" : { "fieldName" : "owners", "fieldType" : "URN", - "hasValuesFieldName" : "hasOwners" + "hasValuesFieldName" : "hasOwners", + "queryByDefault" : false } }, { "name" : "type", @@ -536,8 +535,7 @@ "addToFilters" : true, "boostScore" : 4.0, "fieldName" : "tool", - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "chartId", diff --git a/gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json index f50d1a641be19..0f95ed3813bb1 100644 --- a/gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json @@ -227,8 +227,7 @@ "Searchable" : { "fieldName" : "tags", "fieldType" : "URN_PARTIAL", - "hasValuesFieldName" : "hasTags", - "queryByDefault" : true + "hasValuesFieldName" : "hasTags" } } ] } @@ -254,7 +253,8 @@ "Searchable" : { "fieldName" : "owners", "fieldType" : "URN", - "hasValuesFieldName" : "hasOwners" + "hasValuesFieldName" : "hasOwners", + "queryByDefault" : false } }, { "name" : "type", @@ -393,8 +393,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "description", @@ -402,8 +401,7 @@ "doc" : "Detailed description about the dashboard", "Searchable" : { "fieldType" : "TEXT", - "hasValuesFieldName" : "hasDescription", - "queryByDefault" : true + "hasValuesFieldName" : "hasDescription" } }, { "name" : "charts", @@ -483,8 +481,7 @@ "addToFilters" : true, "boostScore" : 4.0, "fieldName" : "tool", - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "dashboardId", diff --git a/gms/api/src/main/snapshot/com.linkedin.dataflow.dataFlows.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.dataflow.dataFlows.snapshot.json index 083e23df744f6..7c11c6043e919 100644 --- a/gms/api/src/main/snapshot/com.linkedin.dataflow.dataFlows.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.dataflow.dataFlows.snapshot.json @@ -189,8 +189,7 @@ "Searchable" : { "fieldName" : "tags", "fieldType" : "URN_PARTIAL", - "hasValuesFieldName" : "hasTags", - "queryByDefault" : true + "hasValuesFieldName" : "hasTags" } } ] } @@ -216,7 +215,8 @@ "Searchable" : { "fieldName" : "owners", "fieldType" : "URN", - "hasValuesFieldName" : "hasOwners" + "hasValuesFieldName" : "hasOwners", + "queryByDefault" : false } }, { "name" : "type", @@ -365,8 +365,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "description", @@ -375,8 +374,7 @@ "optional" : true, "Searchable" : { "fieldType" : "TEXT", - "hasValuesFieldName" : "hasDescription", - "queryByDefault" : true + "hasValuesFieldName" : "hasDescription" } }, { "name" : "project", @@ -384,7 +382,8 @@ "doc" : "Optional project/namespace associated with the flow", "optional" : true, "Searchable" : { - "fieldType" : "TEXT_PARTIAL" + "fieldType" : "TEXT_PARTIAL", + "queryByDefault" : false } } ], "Aspect" : { @@ -424,8 +423,7 @@ "type" : "string", "doc" : "Workflow manager like azkaban, airflow which orchestrates the flow", "Searchable" : { - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "flowId", @@ -433,16 +431,14 @@ "doc" : "Unique Identifier of the data flow", "Searchable" : { "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "cluster", "type" : "string", "doc" : "Cluster where the flow is executed", "Searchable" : { - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } } ], "Aspect" : { diff --git a/gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json index a22eb5258bb0f..4dda76d5d48b6 100644 --- a/gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json @@ -257,8 +257,7 @@ "Searchable" : { "fieldName" : "tags", "fieldType" : "URN_PARTIAL", - "hasValuesFieldName" : "hasTags", - "queryByDefault" : true + "hasValuesFieldName" : "hasTags" } } ] } @@ -284,7 +283,8 @@ "Searchable" : { "fieldName" : "owners", "fieldType" : "URN", - "hasValuesFieldName" : "hasOwners" + "hasValuesFieldName" : "hasOwners", + "queryByDefault" : false } }, { "name" : "type", @@ -420,8 +420,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "description", @@ -430,8 +429,7 @@ "optional" : true, "Searchable" : { "fieldType" : "TEXT", - "hasValuesFieldName" : "hasDescription", - "queryByDefault" : true + "hasValuesFieldName" : "hasDescription" } }, { "name" : "type", @@ -495,7 +493,8 @@ "/*" : { "fieldName" : "inputs", "fieldType" : "URN", - "numValuesFieldName" : "numInputDatasets" + "numValuesFieldName" : "numInputDatasets", + "queryByDefault" : false } } }, { @@ -515,7 +514,8 @@ "/*" : { "fieldName" : "outputs", "fieldType" : "URN", - "numValuesFieldName" : "numOutputDatasets" + "numValuesFieldName" : "numOutputDatasets", + "queryByDefault" : false } } }, { @@ -564,9 +564,14 @@ "name" : "flow", "type" : "com.linkedin.common.Urn", "doc" : "Standardized data processing flow urn representing the flow for the job", + "Relationship" : { + "entityTypes" : [ "dataFlow" ], + "name" : "IsPartOf" + }, "Searchable" : { "fieldName" : "dataFlow", - "fieldType" : "URN_PARTIAL" + "fieldType" : "URN_PARTIAL", + "queryByDefault" : false } }, { "name" : "jobId", @@ -574,8 +579,7 @@ "doc" : "Unique Identifier of the data job", "Searchable" : { "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } } ], "Aspect" : { diff --git a/gms/api/src/main/snapshot/com.linkedin.dataprocess.dataProcesses.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.dataprocess.dataProcesses.snapshot.json index cdd85cc29da51..7ab48587b8fa9 100644 --- a/gms/api/src/main/snapshot/com.linkedin.dataprocess.dataProcesses.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.dataprocess.dataProcesses.snapshot.json @@ -131,7 +131,8 @@ "Searchable" : { "fieldName" : "owners", "fieldType" : "URN", - "hasValuesFieldName" : "hasOwners" + "hasValuesFieldName" : "hasOwners", + "queryByDefault" : false } }, { "name" : "type", @@ -280,7 +281,8 @@ "/*" : { "fieldName" : "inputs", "fieldType" : "URN", - "numValuesFieldName" : "numInputDatasets" + "numValuesFieldName" : "numInputDatasets", + "queryByDefault" : false } } }, { @@ -301,7 +303,8 @@ "/*" : { "fieldName" : "outputs", "fieldType" : "URN", - "numValuesFieldName" : "numOutputDatasets" + "numValuesFieldName" : "numOutputDatasets", + "queryByDefault" : false } } } ], @@ -329,8 +332,7 @@ "Searchable" : { "boostScore" : 4.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "orchestrator", @@ -338,15 +340,15 @@ "doc" : "Standardized Orchestrator where data process is defined.\nTODO: Migrate towards something that can be validated like DataPlatform urn", "Searchable" : { "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "origin", "type" : "com.linkedin.common.FabricType", "doc" : "Fabric type where dataset belongs to or where it was generated.", "Searchable" : { - "fieldType" : "TEXT_PARTIAL" + "fieldType" : "TEXT_PARTIAL", + "queryByDefault" : false } } ], "Aspect" : { diff --git a/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json index 83d1ec3e4f268..4a92458edd666 100644 --- a/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json @@ -293,8 +293,7 @@ "Searchable" : { "fieldName" : "tags", "fieldType" : "URN_PARTIAL", - "hasValuesFieldName" : "hasTags", - "queryByDefault" : true + "hasValuesFieldName" : "hasTags" } } ] } @@ -409,7 +408,8 @@ "Searchable" : { "fieldName" : "owners", "fieldType" : "URN", - "hasValuesFieldName" : "hasOwners" + "hasValuesFieldName" : "hasOwners", + "queryByDefault" : false } }, { "name" : "type", @@ -816,8 +816,7 @@ "doc" : "Flattened name of the field. Field is computed from jsonPath field. For data translation rules refer to wiki page above.", "Searchable" : { "fieldName" : "fieldPaths", - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "jsonPath", @@ -837,8 +836,7 @@ "Searchable" : { "boostScore" : 0.1, "fieldName" : "fieldDescriptions", - "fieldType" : "TEXT", - "queryByDefault" : true + "fieldType" : "TEXT" } }, { "name" : "type", @@ -962,8 +960,7 @@ "/tags/*/tag" : { "boostScore" : 0.5, "fieldName" : "fieldTags", - "fieldType" : "URN_PARTIAL", - "queryByDefault" : true + "fieldType" : "URN_PARTIAL" } } }, { @@ -1064,8 +1061,7 @@ "Searchable" : { "boostScore" : 0.1, "fieldName" : "editedFieldDescriptions", - "fieldType" : "TEXT", - "queryByDefault" : true + "fieldType" : "TEXT" } }, { "name" : "globalTags", @@ -1076,8 +1072,7 @@ "/tags/*/tag" : { "boostScore" : 0.5, "fieldName" : "editedFieldTags", - "fieldType" : "URN_PARTIAL", - "queryByDefault" : true + "fieldType" : "URN_PARTIAL" } } } ] @@ -1124,7 +1119,8 @@ }, "Searchable" : { "fieldName" : "upstreams", - "fieldType" : "URN" + "fieldType" : "URN", + "queryByDefault" : false } }, { "name" : "type", @@ -1203,8 +1199,7 @@ "optional" : true, "Searchable" : { "fieldType" : "TEXT", - "hasValuesFieldName" : "hasDescription", - "queryByDefault" : true + "hasValuesFieldName" : "hasDescription" } }, { "name" : "uri", @@ -1308,8 +1303,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "origin", @@ -1317,7 +1311,8 @@ "doc" : "Fabric type where dataset belongs to or where it was generated.", "Searchable" : { "addToFilters" : true, - "fieldType" : "TEXT_PARTIAL" + "fieldType" : "TEXT_PARTIAL", + "queryByDefault" : false } } ], "Aspect" : { diff --git a/gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json new file mode 100644 index 0000000000000..4cafa0121ae9a --- /dev/null +++ b/gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json @@ -0,0 +1,2558 @@ +{ + "models" : [ { + "type" : "typeref", + "name" : "ChartDataSourceType", + "namespace" : "com.linkedin.chart", + "doc" : "Input source type for a chart such as dataset or metric", + "ref" : [ { + "type" : "typeref", + "name" : "DatasetUrn", + "namespace" : "com.linkedin.common", + "doc" : "Standardized dataset identifier.", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.DatasetUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Standardized dataset identifier.", + "entityType" : "dataset", + "fields" : [ { + "doc" : "Standardized platform urn where dataset is defined.", + "name" : "platform", + "type" : "com.linkedin.common.urn.DataPlatformUrn" + }, { + "doc" : "Dataset native name e.g. ., /dir/subdir/, or ", + "maxLength" : 210, + "name" : "datasetName", + "type" : "string" + }, { + "doc" : "Fabric type where dataset belongs to or where it was generated.", + "name" : "origin", + "type" : "com.linkedin.common.FabricType" + } ], + "maxLength" : 284, + "name" : "Dataset", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:datahub" + } + } + } ] + }, { + "type" : "record", + "name" : "ChartInfo", + "namespace" : "com.linkedin.chart", + "doc" : "Information about a chart", + "include" : [ { + "type" : "record", + "name" : "CustomProperties", + "namespace" : "com.linkedin.common", + "doc" : "Misc. properties about an entity.", + "fields" : [ { + "name" : "customProperties", + "type" : { + "type" : "map", + "values" : "string" + }, + "doc" : "Custom property bag.", + "default" : { } + } ] + }, { + "type" : "record", + "name" : "ExternalReference", + "namespace" : "com.linkedin.common", + "doc" : "A reference to an external platform.", + "fields" : [ { + "name" : "externalUrl", + "type" : { + "type" : "typeref", + "name" : "Url", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.url.Url", + "coercerClass" : "com.linkedin.common.url.UrlCoercer" + } + }, + "doc" : "URL where the reference exist", + "optional" : true + } ] + } ], + "fields" : [ { + "name" : "title", + "type" : "string", + "doc" : "Title of the chart", + "Searchable" : { + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "description", + "type" : "string", + "doc" : "Detailed description about the chart", + "Searchable" : { } + }, { + "name" : "lastModified", + "type" : { + "type" : "record", + "name" : "ChangeAuditStamps", + "namespace" : "com.linkedin.common", + "doc" : "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into various lifecycle stages, and who acted to move it into those lifecycle stages. The recommended best practice is to include this record in your record schema, and annotate its fields as @readOnly in your resource. See https://github.com/linkedin/rest.li/wiki/Validation-in-Rest.li#restli-validation-annotations", + "fields" : [ { + "name" : "created", + "type" : { + "type" : "record", + "name" : "AuditStamp", + "doc" : "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage.", + "fields" : [ { + "name" : "time", + "type" : { + "type" : "typeref", + "name" : "Time", + "doc" : "Number of milliseconds since midnight, January 1, 1970 UTC. It must be a positive number", + "ref" : "long" + }, + "doc" : "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent." + }, { + "name" : "actor", + "type" : { + "type" : "typeref", + "name" : "Urn", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.Urn" + } + }, + "doc" : "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change." + }, { + "name" : "impersonator", + "type" : "Urn", + "doc" : "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.", + "optional" : true + } ] + }, + "doc" : "An AuditStamp corresponding to the creation of this resource/association/sub-resource" + }, { + "name" : "lastModified", + "type" : "AuditStamp", + "doc" : "An AuditStamp corresponding to the last modification of this resource/association/sub-resource. If no modification has happened since creation, lastModified should be the same as created" + }, { + "name" : "deleted", + "type" : "AuditStamp", + "doc" : "An AuditStamp corresponding to the deletion of this resource/association/sub-resource. Logically, deleted MUST have a later timestamp than creation. It may or may not have the same time as lastModified depending upon the resource/association/sub-resource semantics.", + "optional" : true + } ] + }, + "doc" : "Captures information about who created/last modified/deleted this chart and when" + }, { + "name" : "chartUrl", + "type" : "com.linkedin.common.Url", + "doc" : "URL for the chart. This could be used as an external link on DataHub to allow users access/view the chart", + "optional" : true + }, { + "name" : "inputs", + "type" : { + "type" : "array", + "items" : "ChartDataSourceType" + }, + "doc" : "Data sources for the chart", + "optional" : true + }, { + "name" : "type", + "type" : { + "type" : "enum", + "name" : "ChartType", + "doc" : "The various types of charts", + "symbols" : [ "BAR", "PIE", "SCATTER", "TABLE", "TEXT", "LINE", "AREA", "HISTOGRAM", "BOX_PLOT" ], + "symbolDocs" : { + "BAR" : "Chart showing a Bar chart", + "PIE" : "Chart showing a Pie chart", + "SCATTER" : "Chart showing a Scatter plot", + "TABLE" : "Chart showing a table", + "TEXT" : "Chart showing Markdown formatted text" + } + }, + "doc" : "Type of the chart", + "optional" : true, + "Searchable" : { + "addToFilters" : true, + "fieldType" : "KEYWORD" + } + }, { + "name" : "access", + "type" : { + "type" : "enum", + "name" : "AccessLevel", + "namespace" : "com.linkedin.common", + "doc" : "The various access levels", + "symbols" : [ "PUBLIC", "PRIVATE" ], + "symbolDocs" : { + "PRIVATE" : "Private availability to certain set of users", + "PUBLIC" : "Publicly available access level" + } + }, + "doc" : "Access level for the chart", + "optional" : true, + "Searchable" : { + "addToFilters" : true, + "fieldType" : "KEYWORD" + } + }, { + "name" : "lastRefreshed", + "type" : "com.linkedin.common.Time", + "doc" : "The time when this chart last refreshed", + "optional" : true + } ], + "Aspect" : { + "name" : "chartInfo" + } + }, { + "type" : "record", + "name" : "ChartQuery", + "namespace" : "com.linkedin.chart", + "doc" : "Information for chart query which is used for getting data of the chart", + "fields" : [ { + "name" : "rawQuery", + "type" : "string", + "doc" : "Raw query to build a chart from input datasets" + }, { + "name" : "type", + "type" : { + "type" : "enum", + "name" : "ChartQueryType", + "symbols" : [ "LOOKML", "SQL" ], + "symbolDocs" : { + "LOOKML" : "LookML queries", + "SQL" : "SQL type queries" + } + }, + "doc" : "Chart query type", + "Searchable" : { + "addToFilters" : true, + "fieldType" : "KEYWORD" + } + } ], + "Aspect" : { + "name" : "chartQuery" + } + }, "com.linkedin.chart.ChartQueryType", "com.linkedin.chart.ChartType", "com.linkedin.common.AccessLevel", "com.linkedin.common.AuditStamp", { + "type" : "record", + "name" : "BrowsePaths", + "namespace" : "com.linkedin.common", + "doc" : "Shared aspect containing Browse Paths to be indexed for an entity.", + "fields" : [ { + "name" : "paths", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "A list of valid browse paths for the entity.\n\nBrowse paths are expected to be backslash-separated strings. For example: 'prod/snowflake/datasetName'", + "Searchable" : { + "/*" : { + "fieldName" : "browsePaths", + "fieldType" : "BROWSE_PATH" + } + } + } ], + "Aspect" : { + "name" : "browsePaths" + } + }, "com.linkedin.common.ChangeAuditStamps", { + "type" : "typeref", + "name" : "ChartUrn", + "namespace" : "com.linkedin.common", + "doc" : "Standardized chart identifier", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.ChartUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Standardized chart identifier", + "entityType" : "chart", + "fields" : [ { + "doc" : "The name of the dashboard tool such as looker, redash etc.", + "maxLength" : 20, + "name" : "dashboardTool", + "type" : "string" + }, { + "doc" : "Unique id for the chart. This id should be globally unique for a dashboarding tool even when there are multiple deployments of it. As an example, chart URL could be used here for Looker such as 'looker.linkedin.com/looks/1234'", + "maxLength" : 200, + "name" : "chartId", + "type" : "string" + } ], + "maxLength" : 236, + "name" : "Chart", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:datahub" + } + } + }, { + "type" : "typeref", + "name" : "CorpGroupUrn", + "namespace" : "com.linkedin.common", + "doc" : "Corporate group's AD/LDAP login", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.CorpGroupUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Corporate group's AD/LDAP login", + "entityType" : "corpGroup", + "fields" : [ { + "doc" : "The name of the AD/LDAP group.", + "maxLength" : 64, + "name" : "groupName", + "type" : "string" + } ], + "maxLength" : 81, + "name" : "CorpGroup", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:security" + } + } + }, { + "type" : "typeref", + "name" : "CorpuserUrn", + "namespace" : "com.linkedin.common", + "doc" : "Corporate user's AD/LDAP login", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.CorpuserUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Corporate user's AD/LDAP login", + "entityType" : "corpuser", + "fields" : [ { + "doc" : "The name of the AD/LDAP user.", + "maxLength" : 20, + "name" : "username", + "type" : "string" + } ], + "maxLength" : 36, + "name" : "Corpuser", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:security" + } + } + }, "com.linkedin.common.CustomProperties", { + "type" : "typeref", + "name" : "DataFlowUrn", + "namespace" : "com.linkedin.common", + "doc" : "Standardized data processing flow identifier.", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.DataFlowUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Standardized data processing flow identifier.", + "entityType" : "dataFlow", + "fields" : [ { + "doc" : "Workflow manager like azkaban, airflow which orchestrates the flow", + "maxLength" : 50, + "name" : "orchestrator", + "type" : "string" + }, { + "doc" : "Unique Identifier of the data flow", + "maxLength" : 200, + "name" : "flowId", + "type" : "string" + }, { + "doc" : "Cluster where the flow is executed", + "maxLength" : 100, + "name" : "cluster", + "type" : "string" + } ], + "maxLength" : 373, + "name" : "DataFlow", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:datahub" + } + } + }, { + "type" : "typeref", + "name" : "DataJobUrn", + "namespace" : "com.linkedin.common", + "doc" : "Standardized data processing job identifier.", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.DataJobUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Standardized data processing job identifier.", + "entityType" : "dataJob", + "fields" : [ { + "doc" : "Standardized data processing flow urn representing the flow for the job", + "name" : "flow", + "type" : "com.linkedin.common.urn.DataFlowUrn" + }, { + "doc" : "Unique identifier of the data job", + "maxLength" : 200, + "name" : "jobID", + "type" : "string" + } ], + "maxLength" : 594, + "name" : "DataJob", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:datahub" + } + } + }, { + "type" : "typeref", + "name" : "DataPlatformUrn", + "namespace" : "com.linkedin.common", + "doc" : "Standardized data platforms available", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.DataPlatformUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Standardized data platforms available", + "entityType" : "dataPlatform", + "fields" : [ { + "doc" : "data platform name i.e. hdfs, oracle, espresso", + "maxLength" : 25, + "name" : "platformName", + "type" : "string" + } ], + "maxLength" : 45, + "name" : "DataPlatform", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:wherehows" + } + } + }, "com.linkedin.common.DatasetUrn", { + "type" : "typeref", + "name" : "EmailAddress", + "namespace" : "com.linkedin.common", + "ref" : "string" + }, "com.linkedin.common.ExternalReference", { + "type" : "enum", + "name" : "FabricType", + "namespace" : "com.linkedin.common", + "doc" : "Fabric group type", + "symbols" : [ "DEV", "EI", "PROD", "CORP" ], + "symbolDocs" : { + "CORP" : "Designates corporation fabrics", + "DEV" : "Designates development fabrics", + "EI" : "Designates early-integration (staging) fabrics", + "PROD" : "Designates production fabrics" + } + }, { + "type" : "record", + "name" : "GlobalTags", + "namespace" : "com.linkedin.common", + "doc" : "Tag aspect used for applying tags to an entity", + "fields" : [ { + "name" : "tags", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "TagAssociation", + "doc" : "Properties of an applied tag. For now, just an Urn. In the future we can extend this with other properties, e.g.\npropagation parameters.", + "fields" : [ { + "name" : "tag", + "type" : { + "type" : "typeref", + "name" : "TagUrn", + "doc" : "Globally defined tag", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.TagUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Globally defined tags", + "entityType" : "tag", + "fields" : [ { + "doc" : "tag name", + "maxLength" : 200, + "name" : "name", + "type" : "string" + } ], + "maxLength" : 220, + "name" : "Tag", + "namespace" : "li", + "owners" : [ ], + "owningTeam" : "urn:li:internalTeam:datahub" + } + } + }, + "doc" : "Urn of the applied tag", + "Searchable" : { + "fieldName" : "tags", + "fieldType" : "URN_PARTIAL", + "hasValuesFieldName" : "hasTags" + } + } ] + } + }, + "doc" : "Tags associated with a given entity" + } ], + "Aspect" : { + "name" : "globalTags" + } + }, { + "type" : "typeref", + "name" : "GlossaryNodeUrn", + "namespace" : "com.linkedin.common", + "doc" : "Business Node", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.GlossaryNodeUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Standardized business node identifier", + "entityType" : "glossaryNode", + "fields" : [ { + "doc" : "The name of business node with hierarchy.", + "name" : "name", + "type" : "string" + } ], + "maxLength" : 56, + "name" : "GlossaryNode", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:datahub" + } + } + }, { + "type" : "record", + "name" : "GlossaryTermAssociation", + "namespace" : "com.linkedin.common", + "doc" : "Properties of an applied glossary term.", + "fields" : [ { + "name" : "urn", + "type" : { + "type" : "typeref", + "name" : "GlossaryTermUrn", + "doc" : "Business Term", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.GlossaryTermUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "business term", + "entityType" : "glossaryTerm", + "fields" : [ { + "doc" : "The name of business term with hierarchy.", + "name" : "name", + "type" : "string" + } ], + "maxLength" : 56, + "name" : "GlossaryTerm", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:datahub" + } + } + }, + "doc" : "Urn of the applied glossary term" + } ] + }, "com.linkedin.common.GlossaryTermUrn", { + "type" : "record", + "name" : "GlossaryTerms", + "namespace" : "com.linkedin.common", + "doc" : "Related business terms information", + "fields" : [ { + "name" : "terms", + "type" : { + "type" : "array", + "items" : "GlossaryTermAssociation" + }, + "doc" : "The related business terms" + }, { + "name" : "auditStamp", + "type" : "AuditStamp", + "doc" : "Audit stamp containing who reported the related business term" + } ], + "Aspect" : { + "name" : "glossaryTerms" + } + }, { + "type" : "record", + "name" : "InstitutionalMemory", + "namespace" : "com.linkedin.common", + "doc" : "Institutional memory of an entity. This is a way to link to relevant documentation and provide description of the documentation. Institutional or tribal knowledge is very important for users to leverage the entity.", + "fields" : [ { + "name" : "elements", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "InstitutionalMemoryMetadata", + "doc" : "Metadata corresponding to a record of institutional memory.", + "fields" : [ { + "name" : "url", + "type" : "Url", + "doc" : "Link to an engineering design document or a wiki page." + }, { + "name" : "description", + "type" : "string", + "doc" : "Description of the link." + }, { + "name" : "createStamp", + "type" : "AuditStamp", + "doc" : "Audit stamp associated with creation of this record" + } ] + } + }, + "doc" : "List of records that represent institutional memory of an entity. Each record consists of a link, description, creator and timestamps associated with that record." + } ], + "Aspect" : { + "name" : "institutionalMemory" + } + }, "com.linkedin.common.InstitutionalMemoryMetadata", { + "type" : "enum", + "name" : "MLFeatureDataType", + "namespace" : "com.linkedin.common", + "doc" : "MLFeature Data Type", + "symbols" : [ "USELESS", "NOMINAL", "ORDINAL", "BINARY", "COUNT", "TIME", "INTERVAL", "IMAGE", "VIDEO", "AUDIO", "TEXT", "MAP", "SEQUENCE", "SET" ], + "symbolDocs" : { + "AUDIO" : "Audio Data", + "BINARY" : "Binary data is discrete data that can be in only one of two categories — either yes or no, 1 or 0, off or on, etc", + "COUNT" : "Count data is discrete whole number data — no negative numbers here.\nCount data often has many small values, such as zero and one.", + "IMAGE" : "Image Data", + "INTERVAL" : "Interval data has equal spaces between the numbers and does not represent a temporal pattern.\nExamples include percentages, temperatures, and income.", + "MAP" : "Mapping Data Type ex: dict, map", + "NOMINAL" : "Nominal data is made of discrete values with no numerical relationship between the different categories — mean and median are meaningless.\nAnimal species is one example. For example, pig is not higher than bird and lower than fish.", + "ORDINAL" : "Ordinal data are discrete integers that can be ranked or sorted.\nFor example, the distance between first and second may not be the same as the distance between second and third.", + "SEQUENCE" : "Sequence Data Type ex: list, tuple, range", + "SET" : "Set Data Type ex: set, frozenset", + "TEXT" : "Text Data", + "TIME" : "Time data is a cyclical, repeating continuous form of data.\nThe relevant time features can be any period— daily, weekly, monthly, annual, etc.", + "USELESS" : "Useless data is unique, discrete data with no potential relationship with the outcome variable.\nA useless feature has high cardinality. An example would be bank account numbers that were generated randomly.", + "VIDEO" : "Video Data" + } + }, { + "type" : "typeref", + "name" : "MLFeatureUrn", + "namespace" : "com.linkedin.common", + "doc" : "Standardized MLFeature identifier.", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.MLFeatureUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Standardized MLFeature identifier.", + "entityType" : "mlFeature", + "fields" : [ { + "doc" : "Namespace for the MLFeature", + "name" : "mlFeatureNamespace", + "type" : "string" + }, { + "doc" : "Name of the MLFeature", + "maxLength" : 210, + "name" : "mlFeatureName", + "type" : "string" + } ], + "maxLength" : 284, + "name" : "MLFeature", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:datahub" + } + } + }, { + "type" : "record", + "name" : "Owner", + "namespace" : "com.linkedin.common", + "doc" : "Ownership information", + "fields" : [ { + "name" : "owner", + "type" : "Urn", + "doc" : "Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name\n(Caveat: only corpuser is currently supported in the frontend.)", + "Relationship" : { + "entityTypes" : [ "corpUser", "corpGroup" ], + "name" : "OwnedBy" + }, + "Searchable" : { + "fieldName" : "owners", + "fieldType" : "URN", + "hasValuesFieldName" : "hasOwners", + "queryByDefault" : false + } + }, { + "name" : "type", + "type" : { + "type" : "enum", + "name" : "OwnershipType", + "doc" : "Owner category or owner role", + "symbols" : [ "DEVELOPER", "DATAOWNER", "DELEGATE", "PRODUCER", "CONSUMER", "STAKEHOLDER" ], + "symbolDocs" : { + "CONSUMER" : "A person, group, or service that consumes the data", + "DATAOWNER" : "A person or group that is owning the data", + "DELEGATE" : "A person or a group that overseas the operation, e.g. a DBA or SRE.", + "DEVELOPER" : "A person or group that is in charge of developing the code", + "PRODUCER" : "A person, group, or service that produces/generates the data", + "STAKEHOLDER" : "A person or a group that has direct business interest" + } + }, + "doc" : "The type of the ownership" + }, { + "name" : "source", + "type" : { + "type" : "record", + "name" : "OwnershipSource", + "doc" : "Source/provider of the ownership information", + "fields" : [ { + "name" : "type", + "type" : { + "type" : "enum", + "name" : "OwnershipSourceType", + "symbols" : [ "AUDIT", "DATABASE", "FILE_SYSTEM", "ISSUE_TRACKING_SYSTEM", "MANUAL", "SERVICE", "SOURCE_CONTROL", "OTHER" ], + "symbolDocs" : { + "AUDIT" : "Auditing system or audit logs", + "DATABASE" : "Database, e.g. GRANTS table", + "FILE_SYSTEM" : "File system, e.g. file/directory owner", + "ISSUE_TRACKING_SYSTEM" : "Issue tracking system, e.g. Jira", + "MANUAL" : "Manually provided by a user", + "OTHER" : "Other sources", + "SERVICE" : "Other ownership-like service, e.g. Nuage, ACL service etc", + "SOURCE_CONTROL" : "SCM system, e.g. GIT, SVN" + } + }, + "doc" : "The type of the source" + }, { + "name" : "url", + "type" : "string", + "doc" : "A reference URL for the source", + "optional" : true + } ] + }, + "doc" : "Source information for the ownership", + "optional" : true + } ] + }, { + "type" : "record", + "name" : "Ownership", + "namespace" : "com.linkedin.common", + "doc" : "Ownership information of an entity.", + "fields" : [ { + "name" : "owners", + "type" : { + "type" : "array", + "items" : "Owner" + }, + "doc" : "List of owners of the entity." + }, { + "name" : "lastModified", + "type" : "AuditStamp", + "doc" : "Audit stamp containing who last modified the record and when." + } ], + "Aspect" : { + "name" : "ownership" + } + }, "com.linkedin.common.OwnershipSource", "com.linkedin.common.OwnershipSourceType", "com.linkedin.common.OwnershipType", { + "type" : "record", + "name" : "Status", + "namespace" : "com.linkedin.common", + "doc" : "The status metadata of an entity, e.g. dataset, metric, feature, etc.", + "fields" : [ { + "name" : "removed", + "type" : "boolean", + "doc" : "whether the entity is removed or not", + "default" : false, + "Searchable" : { + "fieldType" : "BOOLEAN" + } + } ], + "Aspect" : { + "name" : "status" + } + }, "com.linkedin.common.TagAssociation", "com.linkedin.common.TagUrn", "com.linkedin.common.Time", { + "type" : "typeref", + "name" : "Uri", + "namespace" : "com.linkedin.common", + "ref" : "string", + "java" : { + "class" : "java.net.URI" + } + }, "com.linkedin.common.Url", "com.linkedin.common.Urn", { + "type" : "record", + "name" : "VersionTag", + "namespace" : "com.linkedin.common", + "doc" : "A resource-defined string representing the resource state for the purpose of concurrency control", + "fields" : [ { + "name" : "versionTag", + "type" : "string", + "optional" : true + } ] + }, { + "type" : "record", + "name" : "DashboardInfo", + "namespace" : "com.linkedin.dashboard", + "doc" : "Information about a dashboard", + "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ], + "fields" : [ { + "name" : "title", + "type" : "string", + "doc" : "Title of the dashboard", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "description", + "type" : "string", + "doc" : "Detailed description about the dashboard", + "Searchable" : { + "fieldType" : "TEXT", + "hasValuesFieldName" : "hasDescription" + } + }, { + "name" : "charts", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.ChartUrn" + }, + "doc" : "Charts in a dashboard", + "default" : [ ], + "Relationship" : { + "/*" : { + "entityTypes" : [ "chart" ], + "name" : "Contains" + } + } + }, { + "name" : "lastModified", + "type" : "com.linkedin.common.ChangeAuditStamps", + "doc" : "Captures information about who created/last modified/deleted this dashboard and when" + }, { + "name" : "dashboardUrl", + "type" : "com.linkedin.common.Url", + "doc" : "URL for the dashboard. This could be used as an external link on DataHub to allow users access/view the dashboard", + "optional" : true + }, { + "name" : "access", + "type" : "com.linkedin.common.AccessLevel", + "doc" : "Access level for the dashboard", + "optional" : true, + "Searchable" : { + "addToFilters" : true, + "fieldType" : "KEYWORD" + } + }, { + "name" : "lastRefreshed", + "type" : "com.linkedin.common.Time", + "doc" : "The time when this dashboard last refreshed", + "optional" : true + } ], + "Aspect" : { + "name" : "dashboardInfo" + } + }, { + "type" : "record", + "name" : "DataFlowInfo", + "namespace" : "com.linkedin.datajob", + "doc" : "Information about a Data processing flow", + "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ], + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Flow name", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "description", + "type" : "string", + "doc" : "Flow description", + "optional" : true, + "Searchable" : { + "fieldType" : "TEXT", + "hasValuesFieldName" : "hasDescription" + } + }, { + "name" : "project", + "type" : "string", + "doc" : "Optional project/namespace associated with the flow", + "optional" : true, + "Searchable" : { + "fieldType" : "TEXT_PARTIAL", + "queryByDefault" : false + } + } ], + "Aspect" : { + "name" : "dataFlowInfo" + } + }, { + "type" : "record", + "name" : "DataJobInfo", + "namespace" : "com.linkedin.datajob", + "doc" : "Information about a Data processing job", + "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ], + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Job name", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "description", + "type" : "string", + "doc" : "Job description", + "optional" : true, + "Searchable" : { + "fieldType" : "TEXT", + "hasValuesFieldName" : "hasDescription" + } + }, { + "name" : "type", + "type" : [ { + "type" : "enum", + "name" : "AzkabanJobType", + "namespace" : "com.linkedin.datajob.azkaban", + "doc" : "The various types of support azkaban jobs", + "symbols" : [ "COMMAND", "HADOOP_JAVA", "HADOOP_SHELL", "HIVE", "PIG", "SQL" ], + "symbolDocs" : { + "COMMAND" : "The command job type is one of the basic built-in types. It runs multiple UNIX commands using java processbuilder.\nUpon execution, Azkaban spawns off a process to run the command.", + "HADOOP_JAVA" : "Runs a java program with ability to access Hadoop cluster.\nhttps://azkaban.readthedocs.io/en/latest/jobTypes.html#java-job-type", + "HADOOP_SHELL" : "In large part, this is the same Command type. The difference is its ability to talk to a Hadoop cluster\nsecurely, via Hadoop tokens.", + "HIVE" : "Hive type is for running Hive jobs.", + "PIG" : "Pig type is for running Pig jobs.", + "SQL" : "SQL is for running Presto, mysql queries etc" + } + } ], + "doc" : "Datajob type" + }, { + "name" : "flowUrn", + "type" : "com.linkedin.common.DataFlowUrn", + "doc" : "DataFlow urn that this job is part of", + "optional" : true, + "Relationship" : { + "entityTypes" : [ "dataFlow" ], + "name" : "IsPartOf" + } + } ], + "Aspect" : { + "name" : "dataJobInfo" + } + }, { + "type" : "record", + "name" : "DataJobInputOutput", + "namespace" : "com.linkedin.datajob", + "doc" : "Information about the inputs and outputs of a Data processing job", + "fields" : [ { + "name" : "inputDatasets", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.DatasetUrn" + }, + "doc" : "Input datasets consumed by the data job during processing", + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataset" ], + "name" : "Consumes" + } + }, + "Searchable" : { + "/*" : { + "fieldName" : "inputs", + "fieldType" : "URN", + "numValuesFieldName" : "numInputDatasets", + "queryByDefault" : false + } + } + }, { + "name" : "outputDatasets", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.DatasetUrn" + }, + "doc" : "Output datasets produced by the data job during processing", + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataset" ], + "name" : "Produces" + } + }, + "Searchable" : { + "/*" : { + "fieldName" : "outputs", + "fieldType" : "URN", + "numValuesFieldName" : "numOutputDatasets", + "queryByDefault" : false + } + } + }, { + "name" : "inputDatajobs", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.DataJobUrn" + }, + "doc" : "Input datajobs that this data job depends on", + "optional" : true, + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataJob" ], + "name" : "DownstreamOf" + } + } + } ], + "Aspect" : { + "name" : "dataJobInputOutput" + } + }, "com.linkedin.datajob.azkaban.AzkabanJobType", { + "type" : "record", + "name" : "DatasetDeprecation", + "namespace" : "com.linkedin.dataset", + "doc" : "Dataset deprecation status", + "fields" : [ { + "name" : "deprecated", + "type" : "boolean", + "doc" : "Whether the dataset is deprecated by owner.", + "Searchable" : { + "fieldType" : "BOOLEAN", + "weightsPerFieldValue" : { + "true" : 0.5 + } + } + }, { + "name" : "decommissionTime", + "type" : "long", + "doc" : "The time user plan to decommission this dataset.", + "optional" : true + }, { + "name" : "note", + "type" : "string", + "doc" : "Additional information about the dataset deprecation plan, such as the wiki, doc, RB." + }, { + "name" : "actor", + "type" : "com.linkedin.common.Urn", + "doc" : "The corpuser URN which will be credited for modifying this deprecation content.", + "optional" : true + } ], + "Aspect" : { + "name" : "datasetDeprecation" + } + }, { + "type" : "enum", + "name" : "DatasetLineageType", + "namespace" : "com.linkedin.dataset", + "doc" : "The various types of supported dataset lineage", + "symbols" : [ "COPY", "TRANSFORMED", "VIEW" ], + "symbolDocs" : { + "COPY" : "Direct copy without modification", + "TRANSFORMED" : "Transformed data with modification (format or content change)", + "VIEW" : "Represents a view defined on the sources e.g. Hive view defined on underlying hive tables or a Hive table pointing to a HDFS dataset or DALI view defined on multiple sources" + } + }, { + "type" : "record", + "name" : "DatasetProperties", + "namespace" : "com.linkedin.dataset", + "doc" : "Properties associated with a Dataset", + "include" : [ "com.linkedin.common.CustomProperties", "com.linkedin.common.ExternalReference" ], + "fields" : [ { + "name" : "description", + "type" : "string", + "doc" : "Documentation of the dataset", + "optional" : true, + "Searchable" : { + "fieldType" : "TEXT", + "hasValuesFieldName" : "hasDescription" + } + }, { + "name" : "uri", + "type" : "com.linkedin.common.Uri", + "doc" : "The abstracted URI such as hdfs:///data/tracking/PageViewEvent, file:///dir/file_name. Uri should not include any environment specific properties. Some datasets might not have a standardized uri, which makes this field optional (i.e. kafka topic).", + "optional" : true + }, { + "name" : "tags", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "[Legacy] Unstructured tags for the dataset. Structured tags can be applied via the `GlobalTags` aspect.", + "default" : [ ] + } ], + "Aspect" : { + "name" : "datasetProperties" + } + }, { + "type" : "typeref", + "name" : "SchemaFieldPath", + "namespace" : "com.linkedin.dataset", + "doc" : "Schema field path. TODO: Add formal documentation on normalization rules.", + "ref" : "string" + }, { + "type" : "record", + "name" : "Upstream", + "namespace" : "com.linkedin.dataset", + "doc" : "Upstream lineage information about a dataset including the source reporting the lineage", + "fields" : [ { + "name" : "auditStamp", + "type" : "com.linkedin.common.AuditStamp", + "doc" : "Audit stamp containing who reported the lineage and when" + }, { + "name" : "dataset", + "type" : "com.linkedin.common.DatasetUrn", + "doc" : "The upstream dataset the lineage points to", + "Relationship" : { + "entityTypes" : [ "dataset" ], + "name" : "DownstreamOf" + }, + "Searchable" : { + "fieldName" : "upstreams", + "fieldType" : "URN", + "queryByDefault" : false + } + }, { + "name" : "type", + "type" : "DatasetLineageType", + "doc" : "The type of the lineage" + } ] + }, { + "type" : "record", + "name" : "UpstreamLineage", + "namespace" : "com.linkedin.dataset", + "doc" : "Upstream lineage of a dataset", + "fields" : [ { + "name" : "upstreams", + "type" : { + "type" : "array", + "items" : "Upstream" + }, + "doc" : "List of upstream dataset lineage information" + } ], + "Aspect" : { + "name" : "upstreamLineage" + } + }, { + "type" : "record", + "name" : "GlossaryNodeInfo", + "namespace" : "com.linkedin.glossary", + "doc" : "Properties associated with a GlossaryNode", + "fields" : [ { + "name" : "definition", + "type" : "string", + "doc" : "Definition of business node", + "Searchable" : { } + }, { + "name" : "parentNode", + "type" : "com.linkedin.common.GlossaryNodeUrn", + "doc" : "Parent node of the glossary term", + "optional" : true + } ], + "Aspect" : { + "name" : "glossaryNodeInfo" + } + }, { + "type" : "record", + "name" : "GlossaryTermInfo", + "namespace" : "com.linkedin.glossary", + "doc" : "Properties associated with a GlossaryTerm", + "fields" : [ { + "name" : "definition", + "type" : "string", + "doc" : "Definition of business term", + "Searchable" : { } + }, { + "name" : "parentNode", + "type" : "com.linkedin.common.GlossaryNodeUrn", + "doc" : "Parent node of the glossary term", + "optional" : true + }, { + "name" : "termSource", + "type" : "string", + "doc" : "Source of the Business Term (INTERNAL or EXTERNAL) with default value as INTERNAL", + "Searchable" : { + "fieldType" : "KEYWORD" + } + }, { + "name" : "sourceRef", + "type" : "string", + "doc" : "External Reference to the business-term", + "optional" : true, + "Searchable" : { + "fieldType" : "KEYWORD" + } + }, { + "name" : "sourceUrl", + "type" : "com.linkedin.common.Url", + "doc" : "The abstracted URL such as https://spec.edmcouncil.org/fibo/ontology/FBC/FinancialInstruments/FinancialInstruments/CashInstrument.", + "optional" : true + }, { + "name" : "customProperties", + "type" : { + "type" : "map", + "values" : "string" + }, + "doc" : "A key-value map to capture any other non-standardized properties for the glossary term", + "default" : { } + } ], + "Aspect" : { + "name" : "glossaryTermInfo" + } + }, { + "type" : "record", + "name" : "CorpGroupInfo", + "namespace" : "com.linkedin.identity", + "doc" : "group of corpUser, it may contains nested group", + "fields" : [ { + "name" : "email", + "type" : "com.linkedin.common.EmailAddress", + "doc" : "email of this group" + }, { + "name" : "admins", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.CorpuserUrn" + }, + "doc" : "owners of this group", + "Relationship" : { + "/*" : { + "entityTypes" : [ "corpUser" ], + "name" : "OwnedBy" + } + } + }, { + "name" : "members", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.CorpuserUrn" + }, + "doc" : "List of ldap urn in this group.", + "Relationship" : { + "/*" : { + "entityTypes" : [ "corpUser" ], + "name" : "IsPartOf" + } + } + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.CorpGroupUrn" + }, + "doc" : "List of groups in this group.", + "Relationship" : { + "/*" : { + "entityTypes" : [ "corpGroup" ], + "name" : "IsPartOf" + } + } + } ], + "Aspect" : { + "EntityUrns" : [ "com.linkedin.common.CorpGroupUrn" ], + "name" : "corpGroupInfo" + } + }, { + "type" : "record", + "name" : "CorpUserEditableInfo", + "namespace" : "com.linkedin.identity", + "doc" : "Linkedin corp user information that can be edited from UI", + "fields" : [ { + "name" : "aboutMe", + "type" : "string", + "doc" : "About me section of the user", + "optional" : true + }, { + "name" : "teams", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Teams that the user belongs to e.g. Metadata", + "default" : [ ], + "Searchable" : { + "/*" : { + "fieldType" : "TEXT" + } + } + }, { + "name" : "skills", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Skills that the user possesses e.g. Machine Learning", + "default" : [ ], + "Searchable" : { + "/*" : { + "fieldType" : "TEXT" + } + } + }, { + "name" : "pictureLink", + "type" : "com.linkedin.common.Url", + "doc" : "A URL which points to a picture which user wants to set as a profile photo", + "default" : "https://raw.githubusercontent.com/linkedin/datahub/master/datahub-web/packages/data-portal/public/assets/images/default_avatar.png" + } ], + "Aspect" : { + "EntityUrns" : [ "com.linkedin.common.CorpuserUrn" ], + "name" : "corpUserEditableInfo" + } + }, { + "type" : "record", + "name" : "CorpUserInfo", + "namespace" : "com.linkedin.identity", + "doc" : "Linkedin corp user information", + "fields" : [ { + "name" : "active", + "type" : "boolean", + "doc" : "Whether the corpUser is active, ref: https://iwww.corp.linkedin.com/wiki/cf/display/GTSD/Accessing+Active+Directory+via+LDAP+tools", + "Searchable" : { + "fieldType" : "BOOLEAN", + "weightsPerFieldValue" : { + "true" : 2.0 + } + } + }, { + "name" : "displayName", + "type" : "string", + "doc" : "displayName of this user , e.g. Hang Zhang(DataHQ)", + "optional" : true + }, { + "name" : "email", + "type" : "com.linkedin.common.EmailAddress", + "doc" : "email address of this user", + "Searchable" : { + "fieldType" : "KEYWORD", + "queryByDefault" : true + } + }, { + "name" : "title", + "type" : "string", + "doc" : "title of this user", + "optional" : true, + "Searchable" : { + "fieldType" : "KEYWORD", + "queryByDefault" : true + } + }, { + "name" : "managerUrn", + "type" : "com.linkedin.common.CorpuserUrn", + "doc" : "direct manager of this user", + "optional" : true, + "Relationship" : { + "entityTypes" : [ "corpUser" ], + "name" : "ReportsTo" + }, + "Searchable" : { + "fieldName" : "managerLdap", + "fieldType" : "URN", + "queryByDefault" : true + } + }, { + "name" : "departmentId", + "type" : "long", + "doc" : "department id this user belong to", + "optional" : true + }, { + "name" : "departmentName", + "type" : "string", + "doc" : "department name this user belong to", + "optional" : true + }, { + "name" : "firstName", + "type" : "string", + "doc" : "first name of this user", + "optional" : true + }, { + "name" : "lastName", + "type" : "string", + "doc" : "last name of this user", + "optional" : true + }, { + "name" : "fullName", + "type" : "string", + "doc" : "Common name of this user, format is firstName + lastName (split by a whitespace)", + "optional" : true, + "Searchable" : { + "boostScore" : 10.0, + "fieldType" : "TEXT_PARTIAL", + "queryByDefault" : true + } + }, { + "name" : "countryCode", + "type" : "string", + "doc" : "two uppercase letters country code. e.g. US", + "optional" : true + } ], + "Aspect" : { + "EntityUrns" : [ "com.linkedin.common.CorpuserUrn" ], + "name" : "corpUserInfo" + } + }, { + "type" : "typeref", + "name" : "Aspect", + "namespace" : "com.linkedin.metadata.aspect", + "doc" : "A union of all supported metadata aspects for a Chart", + "ref" : [ { + "type" : "record", + "name" : "ChartKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a Chart", + "fields" : [ { + "name" : "dashboardTool", + "type" : "string", + "doc" : "The name of the dashboard tool such as looker, redash etc.", + "Searchable" : { + "addToFilters" : true, + "boostScore" : 4.0, + "fieldName" : "tool", + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "chartId", + "type" : "string", + "doc" : "Unique id for the chart. This id should be globally unique for a dashboarding tool even when there are multiple deployments of it. As an example, chart URL could be used here for Looker such as 'looker.linkedin.com/looks/1234'" + } ], + "Aspect" : { + "name" : "chartKey" + } + }, "com.linkedin.chart.ChartInfo", "com.linkedin.chart.ChartQuery", { + "type" : "record", + "name" : "CorpGroupKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a CorpGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "The name of the AD/LDAP group." + } ], + "Aspect" : { + "name" : "corpGroupKey" + } + }, "com.linkedin.identity.CorpGroupInfo", { + "type" : "record", + "name" : "CorpUserKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a CorpUser", + "fields" : [ { + "name" : "username", + "type" : "string", + "doc" : "The name of the AD/LDAP user.", + "Searchable" : { + "boostScore" : 2.0, + "enableAutocomplete" : true, + "fieldName" : "ldap", + "fieldType" : "TEXT_PARTIAL" + } + } ], + "Aspect" : { + "name" : "corpUserKey" + } + }, "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", { + "type" : "record", + "name" : "DashboardKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a Dashboard", + "fields" : [ { + "name" : "dashboardTool", + "type" : "string", + "doc" : "The name of the dashboard tool such as looker, redash etc.", + "Searchable" : { + "addToFilters" : true, + "boostScore" : 4.0, + "fieldName" : "tool", + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "dashboardId", + "type" : "string", + "doc" : "Unique id for the dashboard. This id should be globally unique for a dashboarding tool even when there are multiple deployments of it. As an example, dashboard URL could be used here for Looker such as 'looker.linkedin.com/dashboards/1234'" + } ], + "Aspect" : { + "name" : "dashboardKey" + } + }, "com.linkedin.dashboard.DashboardInfo", { + "type" : "record", + "name" : "DataFlowKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a Data Flow", + "fields" : [ { + "name" : "orchestrator", + "type" : "string", + "doc" : "Workflow manager like azkaban, airflow which orchestrates the flow", + "Searchable" : { + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "flowId", + "type" : "string", + "doc" : "Unique Identifier of the data flow", + "Searchable" : { + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "cluster", + "type" : "string", + "doc" : "Cluster where the flow is executed", + "Searchable" : { + "fieldType" : "TEXT_PARTIAL" + } + } ], + "Aspect" : { + "name" : "dataFlowKey" + } + }, "com.linkedin.datajob.DataFlowInfo", { + "type" : "record", + "name" : "DataJobKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a Data Job", + "fields" : [ { + "name" : "flow", + "type" : "com.linkedin.common.Urn", + "doc" : "Standardized data processing flow urn representing the flow for the job", + "Relationship" : { + "entityTypes" : [ "dataFlow" ], + "name" : "IsPartOf" + }, + "Searchable" : { + "fieldName" : "dataFlow", + "fieldType" : "URN_PARTIAL", + "queryByDefault" : false + } + }, { + "name" : "jobId", + "type" : "string", + "doc" : "Unique Identifier of the data job", + "Searchable" : { + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + } ], + "Aspect" : { + "name" : "dataJobKey" + } + }, "com.linkedin.datajob.DataJobInfo", "com.linkedin.datajob.DataJobInputOutput", "com.linkedin.dataset.DatasetDeprecation", "com.linkedin.dataset.DatasetProperties", "com.linkedin.dataset.UpstreamLineage", { + "type" : "record", + "name" : "SchemaMetadata", + "namespace" : "com.linkedin.schema", + "doc" : "SchemaMetadata to describe metadata related to store schema", + "include" : [ { + "type" : "record", + "name" : "SchemaMetadataKey", + "doc" : "Key to retrieve schema metadata.", + "fields" : [ { + "name" : "schemaName", + "type" : "string", + "doc" : "Schema name e.g. PageViewEvent, identity.Profile, ams.account_management_tracking", + "validate" : { + "strlen" : { + "max" : 500, + "min" : 1 + } + } + }, { + "name" : "platform", + "type" : "com.linkedin.common.DataPlatformUrn", + "doc" : "Standardized platform urn where schema is defined. The data platform Urn (urn:li:platform:{platform_name})" + }, { + "name" : "version", + "type" : "long", + "doc" : "Every change to SchemaMetadata in the resource results in a new version. Version is server assigned. This version is differ from platform native schema version." + } ] + }, "com.linkedin.common.ChangeAuditStamps" ], + "fields" : [ { + "name" : "dataset", + "type" : "com.linkedin.common.DatasetUrn", + "doc" : "Dataset this schema metadata is associated with.", + "optional" : true + }, { + "name" : "cluster", + "type" : "string", + "doc" : "The cluster this schema metadata resides from", + "optional" : true + }, { + "name" : "hash", + "type" : "string", + "doc" : "the SHA1 hash of the schema content" + }, { + "name" : "platformSchema", + "type" : [ { + "type" : "record", + "name" : "EspressoSchema", + "doc" : "Schema text of an espresso table schema.", + "fields" : [ { + "name" : "documentSchema", + "type" : "string", + "doc" : "The native espresso document schema." + }, { + "name" : "tableSchema", + "type" : "string", + "doc" : "The espresso table schema definition." + } ] + }, { + "type" : "record", + "name" : "OracleDDL", + "doc" : "Schema holder for oracle data definition language that describes an oracle table.", + "fields" : [ { + "name" : "tableSchema", + "type" : "string", + "doc" : "The native schema in the dataset's platform. This is a human readable (json blob) table schema." + } ] + }, { + "type" : "record", + "name" : "MySqlDDL", + "doc" : "Schema holder for MySql data definition language that describes an MySql table.", + "fields" : [ { + "name" : "tableSchema", + "type" : "string", + "doc" : "The native schema in the dataset's platform. This is a human readable (json blob) table schema." + } ] + }, { + "type" : "record", + "name" : "PrestoDDL", + "doc" : "Schema holder for presto data definition language that describes a presto view.", + "fields" : [ { + "name" : "rawSchema", + "type" : "string", + "doc" : "The raw schema in the dataset's platform. This includes the DDL and the columns extracted from DDL." + } ] + }, { + "type" : "record", + "name" : "KafkaSchema", + "doc" : "Schema holder for kafka schema.", + "fields" : [ { + "name" : "documentSchema", + "type" : "string", + "doc" : "The native kafka document schema. This is a human readable avro document schema." + } ] + }, { + "type" : "record", + "name" : "BinaryJsonSchema", + "doc" : "Schema text of binary JSON schema.", + "fields" : [ { + "name" : "schema", + "type" : "string", + "doc" : "The native schema text for binary JSON file format." + } ] + }, { + "type" : "record", + "name" : "OrcSchema", + "doc" : "Schema text of an ORC schema.", + "fields" : [ { + "name" : "schema", + "type" : "string", + "doc" : "The native schema for ORC file format." + } ] + }, { + "type" : "record", + "name" : "Schemaless", + "doc" : "The dataset has no specific schema associated with it", + "fields" : [ ] + }, { + "type" : "record", + "name" : "KeyValueSchema", + "doc" : "Schema text of a key-value store schema.", + "fields" : [ { + "name" : "keySchema", + "type" : "string", + "doc" : "The raw schema for the key in the key-value store." + }, { + "name" : "valueSchema", + "type" : "string", + "doc" : "The raw schema for the value in the key-value store." + } ] + }, { + "type" : "record", + "name" : "OtherSchema", + "doc" : "Schema holder for undefined schema types.", + "fields" : [ { + "name" : "rawSchema", + "type" : "string", + "doc" : "The native schema in the dataset's platform." + } ] + } ], + "doc" : "The native schema in the dataset's platform." + }, { + "name" : "fields", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "SchemaField", + "doc" : "SchemaField to describe metadata related to dataset schema. Schema normalization rules: http://go/tms-schema", + "fields" : [ { + "name" : "fieldPath", + "type" : "com.linkedin.dataset.SchemaFieldPath", + "doc" : "Flattened name of the field. Field is computed from jsonPath field. For data translation rules refer to wiki page above.", + "Searchable" : { + "fieldName" : "fieldPaths", + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "jsonPath", + "type" : "string", + "doc" : "Flattened name of a field in JSON Path notation.", + "optional" : true + }, { + "name" : "nullable", + "type" : "boolean", + "doc" : "Indicates if this field is optional or nullable", + "default" : false + }, { + "name" : "description", + "type" : "string", + "doc" : "Description", + "optional" : true, + "Searchable" : { + "boostScore" : 0.1, + "fieldName" : "fieldDescriptions", + "fieldType" : "TEXT" + } + }, { + "name" : "type", + "type" : { + "type" : "record", + "name" : "SchemaFieldDataType", + "doc" : "Schema field data types", + "fields" : [ { + "name" : "type", + "type" : [ { + "type" : "record", + "name" : "BooleanType", + "doc" : "Boolean field type.", + "fields" : [ ] + }, { + "type" : "record", + "name" : "FixedType", + "doc" : "Fixed field type.", + "fields" : [ ] + }, { + "type" : "record", + "name" : "StringType", + "doc" : "String field type.", + "fields" : [ ] + }, { + "type" : "record", + "name" : "BytesType", + "doc" : "Bytes field type.", + "fields" : [ ] + }, { + "type" : "record", + "name" : "NumberType", + "doc" : "Number data type: long, integer, short, etc..", + "fields" : [ ] + }, { + "type" : "record", + "name" : "DateType", + "doc" : "Date field type.", + "fields" : [ ] + }, { + "type" : "record", + "name" : "TimeType", + "doc" : "Time field type. This should also be used for datetimes.", + "fields" : [ ] + }, { + "type" : "record", + "name" : "EnumType", + "doc" : "Enum field type.", + "fields" : [ ] + }, { + "type" : "record", + "name" : "NullType", + "doc" : "Null field type.", + "fields" : [ ] + }, { + "type" : "record", + "name" : "MapType", + "doc" : "Map field type.", + "fields" : [ { + "name" : "keyType", + "type" : "string", + "doc" : "Key type in a map", + "optional" : true + }, { + "name" : "valueType", + "type" : "string", + "doc" : "Type of the value in a map", + "optional" : true + } ] + }, { + "type" : "record", + "name" : "ArrayType", + "doc" : "Array field type.", + "fields" : [ { + "name" : "nestedType", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "List of types this array holds.", + "optional" : true + } ] + }, { + "type" : "record", + "name" : "UnionType", + "doc" : "Union field type.", + "fields" : [ { + "name" : "nestedTypes", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "List of types in union type.", + "optional" : true + } ] + }, { + "type" : "record", + "name" : "RecordType", + "doc" : "Record field type.", + "fields" : [ ] + } ], + "doc" : "Data platform specific types" + } ] + }, + "doc" : "Platform independent field type of the field." + }, { + "name" : "nativeDataType", + "type" : "string", + "doc" : "The native type of the field in the dataset's platform as declared by platform schema." + }, { + "name" : "recursive", + "type" : "boolean", + "doc" : "There are use cases when a field in type B references type A. A field in A references field of type B. In such cases, we will mark the first field as recursive.", + "default" : false + }, { + "name" : "globalTags", + "type" : "com.linkedin.common.GlobalTags", + "doc" : "Tags associated with the field", + "optional" : true, + "Searchable" : { + "/tags/*/tag" : { + "boostScore" : 0.5, + "fieldName" : "fieldTags", + "fieldType" : "URN_PARTIAL" + } + } + }, { + "name" : "glossaryTerms", + "type" : "com.linkedin.common.GlossaryTerms", + "doc" : "Glossary terms associated with the field", + "optional" : true + } ] + } + }, + "doc" : "Client provided a list of fields from document schema." + }, { + "name" : "primaryKeys", + "type" : { + "type" : "array", + "items" : "com.linkedin.dataset.SchemaFieldPath" + }, + "doc" : "Client provided list of fields that define primary keys to access record. Field order defines hierarchical espresso keys. Empty lists indicates absence of primary key access patter. Value is a SchemaField@fieldPath.", + "optional" : true + }, { + "name" : "foreignKeysSpecs", + "type" : { + "type" : "map", + "values" : { + "type" : "record", + "name" : "ForeignKeySpec", + "doc" : "Description of a foreign key in a schema.", + "fields" : [ { + "name" : "foreignKey", + "type" : [ { + "type" : "record", + "name" : "DatasetFieldForeignKey", + "doc" : "For non-urn based foregin keys.", + "fields" : [ { + "name" : "parentDataset", + "type" : "com.linkedin.common.DatasetUrn", + "doc" : "dataset that stores the resource." + }, { + "name" : "currentFieldPaths", + "type" : { + "type" : "array", + "items" : "com.linkedin.dataset.SchemaFieldPath" + }, + "doc" : "List of fields in hosting(current) SchemaMetadata that conform a foreign key. List can contain a single entry or multiple entries if several entries in hosting schema conform a foreign key in a single parent dataset." + }, { + "name" : "parentField", + "type" : "com.linkedin.dataset.SchemaFieldPath", + "doc" : "SchemaField@fieldPath that uniquely identify field in parent dataset that this field references." + } ] + }, { + "type" : "record", + "name" : "UrnForeignKey", + "doc" : "If SchemaMetadata fields make any external references and references are of type com.linkedin.common.Urn or any children, this models can be used to mark it.", + "fields" : [ { + "name" : "currentFieldPath", + "type" : "com.linkedin.dataset.SchemaFieldPath", + "doc" : "Field in hosting(current) SchemaMetadata." + } ] + } ], + "doc" : "Foreign key definition in metadata schema." + } ] + } + }, + "doc" : "Map captures all the references schema makes to external datasets. Map key is ForeignKeySpecName typeref.", + "optional" : true + } ], + "Aspect" : { + "name" : "schemaMetadata" + } + }, { + "type" : "record", + "name" : "EditableSchemaMetadata", + "namespace" : "com.linkedin.schema", + "doc" : "EditableSchemaMetadata stores editable changes made to schema metadata. This separates changes made from\ningestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines.", + "include" : [ "com.linkedin.common.ChangeAuditStamps" ], + "fields" : [ { + "name" : "editableSchemaFieldInfo", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "EditableSchemaFieldInfo", + "doc" : "SchemaField to describe metadata related to dataset schema.", + "fields" : [ { + "name" : "fieldPath", + "type" : "string", + "doc" : "FieldPath uniquely identifying the SchemaField this metadata is associated with" + }, { + "name" : "description", + "type" : "string", + "doc" : "Description", + "optional" : true, + "Searchable" : { + "boostScore" : 0.1, + "fieldName" : "editedFieldDescriptions", + "fieldType" : "TEXT" + } + }, { + "name" : "globalTags", + "type" : "com.linkedin.common.GlobalTags", + "doc" : "Tags associated with the field", + "optional" : true, + "Searchable" : { + "/tags/*/tag" : { + "boostScore" : 0.5, + "fieldName" : "editedFieldTags", + "fieldType" : "URN_PARTIAL" + } + } + } ] + } + }, + "doc" : "Client provided a list of fields from document schema." + } ], + "Aspect" : { + "name" : "editableSchemaMetadata" + } + }, "com.linkedin.common.InstitutionalMemory", "com.linkedin.glossary.GlossaryNodeInfo", { + "type" : "record", + "name" : "GlossaryNodeKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a GlossaryNode", + "fields" : [ { + "name" : "name", + "type" : "string", + "Searchable" : { + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + } ], + "Aspect" : { + "name" : "glossaryNodeKey" + } + }, "com.linkedin.glossary.GlossaryTermInfo", { + "type" : "record", + "name" : "GlossaryTermKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a GlossaryTerm", + "fields" : [ { + "name" : "name", + "type" : "string", + "Searchable" : { + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + } ], + "Aspect" : { + "name" : "glossaryTermKey" + } + }, { + "type" : "record", + "name" : "MLFeatureKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for an ML model", + "fields" : [ { + "name" : "featureNamespace", + "type" : "string", + "doc" : "Namespace for the feature" + }, { + "name" : "name", + "type" : "string", + "doc" : "Name of the feature" + } ], + "Aspect" : { + "name" : "mlFeatureKey" + } + }, { + "type" : "record", + "name" : "MLFeatureProperties", + "namespace" : "com.linkedin.ml.metadata", + "doc" : "Properties associated with a MLFeature", + "fields" : [ { + "name" : "description", + "type" : "string", + "doc" : "Documentation of the MLFeature", + "optional" : true + }, { + "name" : "dataType", + "type" : "com.linkedin.common.MLFeatureDataType", + "doc" : "Data Type of the MLFeature", + "optional" : true + }, { + "name" : "version", + "type" : "com.linkedin.common.VersionTag", + "doc" : "Version of the MLFeature", + "optional" : true + } ], + "Aspect" : { + "name" : "mlFeatureProperties" + } + }, { + "type" : "record", + "name" : "MLModelKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for an ML model", + "fields" : [ { + "name" : "platform", + "type" : "com.linkedin.common.Urn", + "doc" : "Standardized platform urn for the model", + "Searchable" : { + "boostScore" : 0.1, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "name", + "type" : "string", + "doc" : "Name of the MLModel", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "origin", + "type" : "com.linkedin.common.FabricType", + "doc" : "Fabric type where model belongs to or where it was generated", + "Searchable" : { + "fieldType" : "TEXT_PARTIAL", + "queryByDefault" : false + } + } ], + "Aspect" : { + "name" : "mlModelKey" + } + }, { + "type" : "record", + "name" : "CaveatsAndRecommendations", + "namespace" : "com.linkedin.ml.metadata", + "doc" : "This section should list additional concerns that were not covered in the previous sections. For example, did the results suggest any further testing? Were there any relevant groups that were not represented in the evaluation dataset? Are there additional recommendations for model use?", + "fields" : [ { + "name" : "caveats", + "type" : { + "type" : "record", + "name" : "CaveatDetails", + "doc" : "This section should list additional concerns that were not covered in the previous sections. For example, did the results suggest any further testing? Were there any relevant groups that were not represented in the evaluation dataset? Are there additional recommendations for model use?", + "fields" : [ { + "name" : "needsFurtherTesting", + "type" : "boolean", + "doc" : "Did the results suggest any further testing?", + "optional" : true + }, { + "name" : "caveatDescription", + "type" : "string", + "doc" : "Caveat Description\nFor ex: Given gender classes are binary (male/not male), which we include as male/female. Further work needed to evaluate across a spectrum of genders.", + "optional" : true + }, { + "name" : "groupsNotRepresented", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Relevant groups that were not represented in the evaluation dataset?", + "optional" : true + } ] + }, + "doc" : "This section should list additional concerns that were not covered in the previous sections. For example, did the results suggest any further testing? Were there any relevant groups that were not represented in the evaluation dataset?", + "optional" : true + }, { + "name" : "recommendations", + "type" : "string", + "doc" : "Recommendations on where this MLModel should be used.", + "optional" : true + }, { + "name" : "idealDatasetCharacteristics", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Ideal characteristics of an evaluation dataset for this MLModel", + "optional" : true + } ], + "Aspect" : { + "name" : "mlModelCaveatsAndRecommendations" + } + }, { + "type" : "record", + "name" : "EthicalConsiderations", + "namespace" : "com.linkedin.ml.metadata", + "doc" : "This section is intended to demonstrate the ethical considerations that went into MLModel development, surfacing ethical challenges and solutions to stakeholders.", + "fields" : [ { + "name" : "data", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Does the MLModel use any sensitive data (e.g., protected classes)?", + "optional" : true + }, { + "name" : "humanLife", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : " Is the MLModel intended to inform decisions about matters central to human life or flourishing – e.g., health or safety? Or could it be used in such a way?", + "optional" : true + }, { + "name" : "mitigations", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "What risk mitigation strategies were used during MLModel development?", + "optional" : true + }, { + "name" : "risksAndHarms", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "What risks may be present in MLModel usage? Try to identify the potential recipients, likelihood, and magnitude of harms. If these cannot be determined, note that they were considered but remain unknown.", + "optional" : true + }, { + "name" : "useCases", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Are there any known MLModel use cases that are especially fraught? This may connect directly to the intended use section", + "optional" : true + } ], + "Aspect" : { + "name" : "mlModelEthicalConsiderations" + } + }, { + "type" : "record", + "name" : "EvaluationData", + "namespace" : "com.linkedin.ml.metadata", + "doc" : "All referenced datasets would ideally point to any set of documents that provide visibility into the source and composition of the dataset.", + "fields" : [ { + "name" : "evaluationData", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BaseData", + "doc" : "BaseData record", + "fields" : [ { + "name" : "dataset", + "type" : "com.linkedin.common.DatasetUrn", + "doc" : "What dataset were used in the MLModel?" + }, { + "name" : "motivation", + "type" : "string", + "doc" : "Why was this dataset chosen?", + "optional" : true + }, { + "name" : "preProcessing", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "How was the data preprocessed (e.g., tokenization of sentences, cropping of images, any filtering such as dropping images without faces)?", + "optional" : true + } ] + } + }, + "doc" : "Details on the dataset(s) used for the quantitative analyses in the MLModel" + } ], + "Aspect" : { + "name" : "mlModelEvaluationData" + } + }, { + "type" : "record", + "name" : "IntendedUse", + "namespace" : "com.linkedin.ml.metadata", + "doc" : "Intended Use for the ML Model", + "fields" : [ { + "name" : "primaryUses", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Primary Use cases for the MLModel.", + "optional" : true + }, { + "name" : "primaryUsers", + "type" : { + "type" : "array", + "items" : { + "type" : "enum", + "name" : "IntendedUserType", + "symbols" : [ "ENTERPRISE", "HOBBY", "ENTERTAINMENT" ] + } + }, + "doc" : "Primary Intended Users - For example, was the MLModel developed for entertainment purposes, for hobbyists, or enterprise solutions?", + "optional" : true + }, { + "name" : "outOfScopeUses", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Highlight technology that the MLModel might easily be confused with, or related contexts that users could try to apply the MLModel to.", + "optional" : true + } ], + "Aspect" : { + "name" : "intendedUse" + } + }, { + "type" : "record", + "name" : "Metrics", + "namespace" : "com.linkedin.ml.metadata", + "doc" : "Metrics to be featured for the MLModel.", + "fields" : [ { + "name" : "performanceMeasures", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Measures of MLModel performance", + "optional" : true + }, { + "name" : "decisionThreshold", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Decision Thresholds used (if any)?", + "optional" : true + } ], + "Aspect" : { + "name" : "mlModelMetrics" + } + }, { + "type" : "record", + "name" : "MLModelFactorPrompts", + "namespace" : "com.linkedin.ml.metadata", + "doc" : "Prompts which affect the performance of the MLModel", + "fields" : [ { + "name" : "relevantFactors", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "MLModelFactors", + "doc" : "Factors affecting the performance of the MLModel.", + "fields" : [ { + "name" : "groups", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Groups refers to distinct categories with similar characteristics that are present in the evaluation data instances.\nFor human-centric machine learning MLModels, groups are people who share one or multiple characteristics.", + "optional" : true + }, { + "name" : "instrumentation", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "The performance of a MLModel can vary depending on what instruments were used to capture the input to the MLModel.\nFor example, a face detection model may perform differently depending on the camera’s hardware and software,\nincluding lens, image stabilization, high dynamic range techniques, and background blurring for portrait mode.", + "optional" : true + }, { + "name" : "environment", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "A further factor affecting MLModel performance is the environment in which it is deployed.", + "optional" : true + } ] + } + }, + "doc" : "What are foreseeable salient factors for which MLModel performance may vary, and how were these determined?", + "optional" : true + }, { + "name" : "evaluationFactors", + "type" : { + "type" : "array", + "items" : "MLModelFactors" + }, + "doc" : "Which factors are being reported, and why were these chosen?", + "optional" : true + } ], + "Aspect" : { + "name" : "mlModelFactorPrompts" + } + }, { + "type" : "record", + "name" : "MLModelProperties", + "namespace" : "com.linkedin.ml.metadata", + "doc" : "Properties associated with a ML Model", + "fields" : [ { + "name" : "description", + "type" : "string", + "doc" : "Documentation of the MLModel", + "optional" : true, + "Searchable" : { + "fieldType" : "TEXT", + "hasValuesFieldName" : "hasDescription" + } + }, { + "name" : "date", + "type" : "com.linkedin.common.Time", + "doc" : "Date when the MLModel was developed", + "optional" : true + }, { + "name" : "version", + "type" : "com.linkedin.common.VersionTag", + "doc" : "Version of the MLModel", + "optional" : true + }, { + "name" : "type", + "type" : "string", + "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc", + "optional" : true, + "Searchable" : { + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "hyperParameters", + "type" : { + "type" : "map", + "values" : { + "type" : "typeref", + "name" : "HyperParameterValueType", + "doc" : "A union of all supported metadata aspects for HyperParameter Value", + "ref" : [ "string", "int", "float", "double", "boolean" ] + } + }, + "doc" : "Hyper Parameters of the MLModel", + "optional" : true + }, { + "name" : "mlFeatures", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.MLFeatureUrn" + }, + "doc" : "List of features used for MLModel training", + "optional" : true + }, { + "name" : "tags", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Tags for the MLModel", + "default" : [ ] + } ], + "Aspect" : { + "name" : "mlModelProperties" + } + }, { + "type" : "record", + "name" : "QuantitativeAnalyses", + "namespace" : "com.linkedin.ml.metadata", + "doc" : "Quantitative analyses should be disaggregated, that is, broken down by the chosen factors. Quantitative analyses should provide the results of evaluating the MLModel according to the chosen metrics, providing confidence interval values when possible.", + "fields" : [ { + "name" : "unitaryResults", + "type" : { + "type" : "typeref", + "name" : "ResultsType", + "doc" : "A union of all supported metadata aspects for ResultsType", + "ref" : [ "string" ] + }, + "doc" : "Link to a dashboard with results showing how the MLModel performed with respect to each factor", + "optional" : true + }, { + "name" : "intersectionalResults", + "type" : "ResultsType", + "doc" : "Link to a dashboard with results showing how the MLModel performed with respect to the intersection of evaluated factors?", + "optional" : true + } ], + "Aspect" : { + "name" : "mlModelQuantitativeAnalyses" + } + }, { + "type" : "record", + "name" : "TrainingData", + "namespace" : "com.linkedin.ml.metadata", + "doc" : "Ideally, the MLModel card would contain as much information about the training data as the evaluation data. However, there might be cases where it is not feasible to provide this level of detailed information about the training data. For example, the data may be proprietary, or require a non-disclosure agreement. In these cases, we advocate for basic details about the distributions over groups in the data, as well as any other details that could inform stakeholders on the kinds of biases the model may have encoded.", + "fields" : [ { + "name" : "trainingData", + "type" : { + "type" : "array", + "items" : "BaseData" + }, + "doc" : "Details on the dataset(s) used for training the MLModel" + } ], + "Aspect" : { + "name" : "mlModelTrainingData" + } + }, { + "type" : "record", + "name" : "SourceCode", + "namespace" : "com.linkedin.ml.metadata", + "doc" : "Source Code", + "fields" : [ { + "name" : "sourceCode", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "SourceCodeUrl", + "doc" : "Source Code Url Entity", + "fields" : [ { + "name" : "type", + "type" : { + "type" : "enum", + "name" : "SourceCodeUrlType", + "symbols" : [ "ML_MODEL_SOURCE_CODE", "TRAINING_PIPELINE_SOURCE_CODE", "EVALUATION_PIPELINE_SOURCE_CODE" ] + }, + "doc" : "Source Code Url Types" + }, { + "name" : "sourceCodeUrl", + "type" : "com.linkedin.common.Url", + "doc" : "Source Code Url" + } ] + } + }, + "doc" : "Source Code along with types" + } ], + "Aspect" : { + "name" : "sourceCode" + } + }, { + "type" : "record", + "name" : "TagKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a Tag", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "The unique tag name", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + } ], + "Aspect" : { + "name" : "tagKey" + } + }, { + "type" : "record", + "name" : "TagProperties", + "namespace" : "com.linkedin.tag", + "doc" : "Properties associated with a Tag", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the tag" + }, { + "name" : "description", + "type" : "string", + "doc" : "Documentation of the tag", + "optional" : true + } ], + "Aspect" : { + "name" : "tagProperties" + } + }, "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.common.GlobalTags", "com.linkedin.common.BrowsePaths" ] + }, { + "type" : "record", + "name" : "AspectWithMetadata", + "namespace" : "com.linkedin.metadata.aspect", + "fields" : [ { + "name" : "aspect", + "type" : "Aspect" + }, { + "name" : "version", + "type" : "long" + } ] + }, "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.TagKey", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", "com.linkedin.schema.ArrayType", "com.linkedin.schema.BinaryJsonSchema", "com.linkedin.schema.BooleanType", "com.linkedin.schema.BytesType", "com.linkedin.schema.DatasetFieldForeignKey", "com.linkedin.schema.DateType", "com.linkedin.schema.EditableSchemaFieldInfo", "com.linkedin.schema.EditableSchemaMetadata", "com.linkedin.schema.EnumType", "com.linkedin.schema.EspressoSchema", "com.linkedin.schema.FixedType", "com.linkedin.schema.ForeignKeySpec", "com.linkedin.schema.KafkaSchema", "com.linkedin.schema.KeyValueSchema", "com.linkedin.schema.MapType", "com.linkedin.schema.MySqlDDL", "com.linkedin.schema.NullType", "com.linkedin.schema.NumberType", "com.linkedin.schema.OracleDDL", "com.linkedin.schema.OrcSchema", "com.linkedin.schema.OtherSchema", "com.linkedin.schema.PrestoDDL", "com.linkedin.schema.RecordType", "com.linkedin.schema.SchemaField", "com.linkedin.schema.SchemaFieldDataType", "com.linkedin.schema.SchemaMetadata", "com.linkedin.schema.SchemaMetadataKey", "com.linkedin.schema.Schemaless", "com.linkedin.schema.StringType", "com.linkedin.schema.TimeType", "com.linkedin.schema.UnionType", "com.linkedin.schema.UrnForeignKey", "com.linkedin.tag.TagProperties" ], + "schema" : { + "name" : "aspects", + "namespace" : "com.linkedin.entity", + "path" : "/aspects", + "schema" : "com.linkedin.metadata.aspect.AspectWithMetadata", + "doc" : "Single unified resource for fetching, updating, searching, & browsing DataHub entities\n\ngenerated from: com.linkedin.metadata.resources.entity.AspectResource", + "collection" : { + "identifier" : { + "name" : "aspectsId", + "type" : "string" + }, + "supports" : [ "get" ], + "methods" : [ { + "method" : "get", + "doc" : "Retrieves the value for an entity that is made up of latest versions of specified aspects.", + "parameters" : [ { + "name" : "aspect", + "type" : "string", + "optional" : true + }, { + "name" : "version", + "type" : "long", + "optional" : true + } ] + } ], + "entity" : { + "path" : "/aspects/{aspectsId}" + } + } + } +} \ No newline at end of file diff --git a/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index ef2a43a9d8f0f..d4174980301fe 100644 --- a/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -85,15 +85,14 @@ "type" : "string", "doc" : "Title of the chart", "Searchable" : { + "enableAutocomplete" : true, "fieldType" : "TEXT_PARTIAL" } }, { "name" : "description", "type" : "string", "doc" : "Detailed description about the chart", - "Searchable" : { - "fieldType" : "TEXT" - } + "Searchable" : { } }, { "name" : "lastModified", "type" : { @@ -712,8 +711,7 @@ "Searchable" : { "fieldName" : "tags", "fieldType" : "URN_PARTIAL", - "hasValuesFieldName" : "hasTags", - "queryByDefault" : true + "hasValuesFieldName" : "hasTags" } } ] } @@ -945,7 +943,8 @@ "Searchable" : { "fieldName" : "owners", "fieldType" : "URN", - "hasValuesFieldName" : "hasOwners" + "hasValuesFieldName" : "hasOwners", + "queryByDefault" : false } }, { "name" : "type", @@ -1066,8 +1065,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "description", @@ -1075,8 +1073,7 @@ "doc" : "Detailed description about the dashboard", "Searchable" : { "fieldType" : "TEXT", - "hasValuesFieldName" : "hasDescription", - "queryByDefault" : true + "hasValuesFieldName" : "hasDescription" } }, { "name" : "charts", @@ -1132,8 +1129,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "description", @@ -1142,8 +1138,7 @@ "optional" : true, "Searchable" : { "fieldType" : "TEXT", - "hasValuesFieldName" : "hasDescription", - "queryByDefault" : true + "hasValuesFieldName" : "hasDescription" } }, { "name" : "project", @@ -1151,7 +1146,8 @@ "doc" : "Optional project/namespace associated with the flow", "optional" : true, "Searchable" : { - "fieldType" : "TEXT_PARTIAL" + "fieldType" : "TEXT_PARTIAL", + "queryByDefault" : false } } ], "Aspect" : { @@ -1170,8 +1166,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "description", @@ -1180,8 +1175,7 @@ "optional" : true, "Searchable" : { "fieldType" : "TEXT", - "hasValuesFieldName" : "hasDescription", - "queryByDefault" : true + "hasValuesFieldName" : "hasDescription" } }, { "name" : "type", @@ -1236,7 +1230,8 @@ "/*" : { "fieldName" : "inputs", "fieldType" : "URN", - "numValuesFieldName" : "numInputDatasets" + "numValuesFieldName" : "numInputDatasets", + "queryByDefault" : false } } }, { @@ -1256,7 +1251,8 @@ "/*" : { "fieldName" : "outputs", "fieldType" : "URN", - "numValuesFieldName" : "numOutputDatasets" + "numValuesFieldName" : "numOutputDatasets", + "queryByDefault" : false } } }, { @@ -1352,7 +1348,8 @@ "/*" : { "fieldName" : "inputs", "fieldType" : "URN", - "numValuesFieldName" : "numInputDatasets" + "numValuesFieldName" : "numInputDatasets", + "queryByDefault" : false } } }, { @@ -1373,7 +1370,8 @@ "/*" : { "fieldName" : "outputs", "fieldType" : "URN", - "numValuesFieldName" : "numOutputDatasets" + "numValuesFieldName" : "numOutputDatasets", + "queryByDefault" : false } } } ], @@ -1460,8 +1458,7 @@ "optional" : true, "Searchable" : { "fieldType" : "TEXT", - "hasValuesFieldName" : "hasDescription", - "queryByDefault" : true + "hasValuesFieldName" : "hasDescription" } }, { "name" : "uri", @@ -1521,7 +1518,8 @@ }, "Searchable" : { "fieldName" : "upstreams", - "fieldType" : "URN" + "fieldType" : "URN", + "queryByDefault" : false } }, { "name" : "type", @@ -1585,8 +1583,7 @@ "addToFilters" : true, "boostScore" : 4.0, "fieldName" : "tool", - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "chartId", @@ -1725,9 +1722,9 @@ "doc" : "The name of the AD/LDAP user.", "Searchable" : { "boostScore" : 2.0, + "enableAutocomplete" : true, "fieldName" : "ldap", - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } } ], "Aspect" : { @@ -1811,7 +1808,6 @@ "optional" : true, "Searchable" : { "boostScore" : 10.0, - "enableAutocomplete" : true, "fieldType" : "TEXT_PARTIAL", "queryByDefault" : true } @@ -1845,8 +1841,7 @@ "default" : [ ], "Searchable" : { "/*" : { - "fieldType" : "TEXT", - "queryByDefault" : true + "fieldType" : "TEXT" } } }, { @@ -1859,8 +1854,7 @@ "default" : [ ], "Searchable" : { "/*" : { - "fieldType" : "TEXT", - "queryByDefault" : true + "fieldType" : "TEXT" } } }, { @@ -1912,8 +1906,7 @@ "addToFilters" : true, "boostScore" : 4.0, "fieldName" : "tool", - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "dashboardId", @@ -1959,8 +1952,7 @@ "type" : "string", "doc" : "Workflow manager like azkaban, airflow which orchestrates the flow", "Searchable" : { - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "flowId", @@ -1968,16 +1960,14 @@ "doc" : "Unique Identifier of the data flow", "Searchable" : { "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "cluster", "type" : "string", "doc" : "Cluster where the flow is executed", "Searchable" : { - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } } ], "Aspect" : { @@ -2018,9 +2008,14 @@ "name" : "flow", "type" : "com.linkedin.common.Urn", "doc" : "Standardized data processing flow urn representing the flow for the job", + "Relationship" : { + "entityTypes" : [ "dataFlow" ], + "name" : "IsPartOf" + }, "Searchable" : { "fieldName" : "dataFlow", - "fieldType" : "URN_PARTIAL" + "fieldType" : "URN_PARTIAL", + "queryByDefault" : false } }, { "name" : "jobId", @@ -2028,8 +2023,7 @@ "doc" : "Unique Identifier of the data job", "Searchable" : { "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } } ], "Aspect" : { @@ -2080,8 +2074,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "origin", @@ -2089,7 +2082,8 @@ "doc" : "Fabric type where dataset belongs to or where it was generated.", "Searchable" : { "addToFilters" : true, - "fieldType" : "TEXT_PARTIAL" + "fieldType" : "TEXT_PARTIAL", + "queryByDefault" : false } } ], "Aspect" : { @@ -2250,8 +2244,7 @@ "doc" : "Flattened name of the field. Field is computed from jsonPath field. For data translation rules refer to wiki page above.", "Searchable" : { "fieldName" : "fieldPaths", - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "jsonPath", @@ -2271,8 +2264,7 @@ "Searchable" : { "boostScore" : 0.1, "fieldName" : "fieldDescriptions", - "fieldType" : "TEXT", - "queryByDefault" : true + "fieldType" : "TEXT" } }, { "name" : "type", @@ -2396,8 +2388,7 @@ "/tags/*/tag" : { "boostScore" : 0.5, "fieldName" : "fieldTags", - "fieldType" : "URN_PARTIAL", - "queryByDefault" : true + "fieldType" : "URN_PARTIAL" } } }, { @@ -2493,8 +2484,7 @@ "Searchable" : { "boostScore" : 0.1, "fieldName" : "editedFieldDescriptions", - "fieldType" : "TEXT", - "queryByDefault" : true + "fieldType" : "TEXT" } }, { "name" : "globalTags", @@ -2505,8 +2495,7 @@ "/tags/*/tag" : { "boostScore" : 0.5, "fieldName" : "editedFieldTags", - "fieldType" : "URN_PARTIAL", - "queryByDefault" : true + "fieldType" : "URN_PARTIAL" } } } ] @@ -2555,8 +2544,7 @@ "Searchable" : { "boostScore" : 4.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "orchestrator", @@ -2564,15 +2552,15 @@ "doc" : "Standardized Orchestrator where data process is defined.\nTODO: Migrate towards something that can be validated like DataPlatform urn", "Searchable" : { "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "origin", "type" : "com.linkedin.common.FabricType", "doc" : "Fabric type where dataset belongs to or where it was generated.", "Searchable" : { - "fieldType" : "TEXT_PARTIAL" + "fieldType" : "TEXT_PARTIAL", + "queryByDefault" : false } } ], "Aspect" : { @@ -2654,8 +2642,7 @@ "doc" : "Standardized platform urn for the model", "Searchable" : { "boostScore" : 0.1, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "name", @@ -2664,15 +2651,15 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "origin", "type" : "com.linkedin.common.FabricType", "doc" : "Fabric type where model belongs to or where it was generated", "Searchable" : { - "fieldType" : "TEXT_PARTIAL" + "fieldType" : "TEXT_PARTIAL", + "queryByDefault" : false } } ], "Aspect" : { @@ -2690,8 +2677,7 @@ "optional" : true, "Searchable" : { "fieldType" : "TEXT", - "hasValuesFieldName" : "hasDescription", - "queryByDefault" : true + "hasValuesFieldName" : "hasDescription" } }, { "name" : "date", @@ -2709,8 +2695,7 @@ "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc", "optional" : true, "Searchable" : { - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "hyperParameters", @@ -3175,8 +3160,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } } ], "Aspect" : { @@ -3236,8 +3220,7 @@ "type" : "string", "Searchable" : { "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } } ], "Aspect" : { @@ -3252,10 +3235,7 @@ "name" : "definition", "type" : "string", "doc" : "Definition of business term", - "Searchable" : { - "fieldType" : "TEXT", - "queryByDefault" : true - } + "Searchable" : { } }, { "name" : "parentNode", "type" : "com.linkedin.common.GlossaryNodeUrn", @@ -3328,6 +3308,7 @@ "name" : "name", "type" : "string", "Searchable" : { + "enableAutocomplete" : true, "fieldType" : "TEXT_PARTIAL" } } ], @@ -3343,10 +3324,7 @@ "name" : "definition", "type" : "string", "doc" : "Definition of business node", - "Searchable" : { - "fieldType" : "TEXT", - "queryByDefault" : true - } + "Searchable" : { } }, { "name" : "parentNode", "type" : "com.linkedin.common.GlossaryNodeUrn", diff --git a/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryNodes.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryNodes.snapshot.json index 6f8a336dbe8ce..d619f72af96f0 100644 --- a/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryNodes.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryNodes.snapshot.json @@ -73,7 +73,8 @@ "Searchable" : { "fieldName" : "owners", "fieldType" : "URN", - "hasValuesFieldName" : "hasOwners" + "hasValuesFieldName" : "hasOwners", + "queryByDefault" : false } }, { "name" : "type", @@ -182,10 +183,7 @@ "name" : "definition", "type" : "string", "doc" : "Definition of business node", - "Searchable" : { - "fieldType" : "TEXT", - "queryByDefault" : true - } + "Searchable" : { } }, { "name" : "parentNode", "type" : "com.linkedin.common.GlossaryNodeUrn", @@ -233,6 +231,7 @@ "name" : "name", "type" : "string", "Searchable" : { + "enableAutocomplete" : true, "fieldType" : "TEXT_PARTIAL" } } ], diff --git a/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json index 8d80f5a12019f..45efdd86bcb46 100644 --- a/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json @@ -100,7 +100,8 @@ "Searchable" : { "fieldName" : "owners", "fieldType" : "URN", - "hasValuesFieldName" : "hasOwners" + "hasValuesFieldName" : "hasOwners", + "queryByDefault" : false } }, { "name" : "type", @@ -218,10 +219,7 @@ "name" : "definition", "type" : "string", "doc" : "Definition of business term", - "Searchable" : { - "fieldType" : "TEXT", - "queryByDefault" : true - } + "Searchable" : { } }, { "name" : "parentNode", "type" : "com.linkedin.common.GlossaryNodeUrn", @@ -298,8 +296,7 @@ "type" : "string", "Searchable" : { "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } } ], "Aspect" : { diff --git a/gms/api/src/main/snapshot/com.linkedin.identity.corpGroups.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.identity.corpGroups.snapshot.json index e6f8e0a346100..6d0b4e9c88257 100644 --- a/gms/api/src/main/snapshot/com.linkedin.identity.corpGroups.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.identity.corpGroups.snapshot.json @@ -107,8 +107,7 @@ "Searchable" : { "fieldName" : "tags", "fieldType" : "URN_PARTIAL", - "hasValuesFieldName" : "hasTags", - "queryByDefault" : true + "hasValuesFieldName" : "hasTags" } } ] } diff --git a/gms/api/src/main/snapshot/com.linkedin.identity.corpUsers.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.identity.corpUsers.snapshot.json index 8cd7341a3ccc1..08c60f22374f6 100644 --- a/gms/api/src/main/snapshot/com.linkedin.identity.corpUsers.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.identity.corpUsers.snapshot.json @@ -79,8 +79,7 @@ "Searchable" : { "fieldName" : "tags", "fieldType" : "URN_PARTIAL", - "hasValuesFieldName" : "hasTags", - "queryByDefault" : true + "hasValuesFieldName" : "hasTags" } } ] } @@ -212,7 +211,6 @@ "optional" : true, "Searchable" : { "boostScore" : 10.0, - "enableAutocomplete" : true, "fieldType" : "TEXT_PARTIAL", "queryByDefault" : true } @@ -250,8 +248,7 @@ "default" : [ ], "Searchable" : { "/*" : { - "fieldType" : "TEXT", - "queryByDefault" : true + "fieldType" : "TEXT" } } }, { @@ -264,8 +261,7 @@ "default" : [ ], "Searchable" : { "/*" : { - "fieldType" : "TEXT", - "queryByDefault" : true + "fieldType" : "TEXT" } } }, { @@ -319,9 +315,9 @@ "doc" : "The name of the AD/LDAP user.", "Searchable" : { "boostScore" : 2.0, + "enableAutocomplete" : true, "fieldName" : "ldap", - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } } ], "Aspect" : { diff --git a/gms/api/src/main/snapshot/com.linkedin.ml.mlModels.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.ml.mlModels.snapshot.json index fcc5220def998..a1d1ca5bd502c 100644 --- a/gms/api/src/main/snapshot/com.linkedin.ml.mlModels.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.ml.mlModels.snapshot.json @@ -365,7 +365,8 @@ "Searchable" : { "fieldName" : "owners", "fieldType" : "URN", - "hasValuesFieldName" : "hasOwners" + "hasValuesFieldName" : "hasOwners", + "queryByDefault" : false } }, { "name" : "type", @@ -481,8 +482,7 @@ "doc" : "Standardized platform urn for the model", "Searchable" : { "boostScore" : 0.1, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "name", @@ -491,15 +491,15 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "origin", "type" : "com.linkedin.common.FabricType", "doc" : "Fabric type where model belongs to or where it was generated", "Searchable" : { - "fieldType" : "TEXT_PARTIAL" + "fieldType" : "TEXT_PARTIAL", + "queryByDefault" : false } } ], "Aspect" : { @@ -517,8 +517,7 @@ "optional" : true, "Searchable" : { "fieldType" : "TEXT", - "hasValuesFieldName" : "hasDescription", - "queryByDefault" : true + "hasValuesFieldName" : "hasDescription" } }, { "name" : "date", @@ -536,8 +535,7 @@ "doc" : "Type of Algorithm or MLModel such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc", "optional" : true, "Searchable" : { - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } }, { "name" : "hyperParameters", diff --git a/gms/api/src/main/snapshot/com.linkedin.tag.tags.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.tag.tags.snapshot.json index 3db7b7fbb4f27..916052801a32c 100644 --- a/gms/api/src/main/snapshot/com.linkedin.tag.tags.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.tag.tags.snapshot.json @@ -65,7 +65,8 @@ "Searchable" : { "fieldName" : "owners", "fieldType" : "URN", - "hasValuesFieldName" : "hasOwners" + "hasValuesFieldName" : "hasOwners", + "queryByDefault" : false } }, { "name" : "type", @@ -200,8 +201,7 @@ "Searchable" : { "boostScore" : 10.0, "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL", - "queryByDefault" : true + "fieldType" : "TEXT_PARTIAL" } } ], "Aspect" : { diff --git a/gms/client/src/main/java/com/linkedin/entity/client/AspectClient.java b/gms/client/src/main/java/com/linkedin/entity/client/AspectClient.java new file mode 100644 index 0000000000000..f95a8ce41006b --- /dev/null +++ b/gms/client/src/main/java/com/linkedin/entity/client/AspectClient.java @@ -0,0 +1,41 @@ +package com.linkedin.entity.client; + +import com.linkedin.entity.AspectsGetRequestBuilder; +import com.linkedin.entity.AspectsRequestBuilders; +import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.restli.client.Client; +import javax.annotation.Nonnull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class AspectClient { + + private static final AspectsRequestBuilders ASPECTS_REQUEST_BUILDERS = new AspectsRequestBuilders(); + + private final Client _client; + private final Logger _logger = LoggerFactory.getLogger("AspectClient"); + + public AspectClient(@Nonnull final Client restliClient) { + _client = restliClient; + } + + /** + * Gets aspect at veresion for an entity + * + * @param urn urn for the entity + * @return list of paths given urn + * @throws RemoteInvocationException + */ + @Nonnull + public AspectWithMetadata getAspect(@Nonnull String urn, @Nonnull String aspect, @Nonnull Long version) + throws RemoteInvocationException { + + AspectsGetRequestBuilder requestBuilder = + ASPECTS_REQUEST_BUILDERS.get().id(urn).aspectParam(aspect).versionParam(version); + + return _client.sendRequest(requestBuilder.build()).getResponse().getEntity(); + + } +} diff --git a/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java new file mode 100644 index 0000000000000..51ee24ff55e36 --- /dev/null +++ b/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java @@ -0,0 +1,54 @@ +package com.linkedin.metadata.resources.entity; + +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.restli.RestliUtils; +import com.linkedin.parseq.Task; +import com.linkedin.restli.server.annotations.Optional; +import com.linkedin.restli.server.annotations.QueryParam; +import com.linkedin.restli.server.annotations.RestLiCollection; +import com.linkedin.restli.server.annotations.RestMethod; +import com.linkedin.restli.server.resources.CollectionResourceTaskTemplate; +import java.net.URISyntaxException; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.inject.Named; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Single unified resource for fetching, updating, searching, & browsing DataHub entities + */ +@RestLiCollection(name = "aspects", namespace = "com.linkedin.entity") +public class AspectResource extends CollectionResourceTaskTemplate { + + private final Logger _logger = LoggerFactory.getLogger("EntityResource"); + + @Inject + @Named("entityService") + private EntityService _entityService; + + /** + * Retrieves the value for an entity that is made up of latest versions of specified aspects. + */ + @RestMethod.Get + @Nonnull + public Task get( + @Nonnull String urnStr, + @QueryParam("aspect") @Optional @Nullable String aspectName, + @QueryParam("version") @Optional @Nullable Long version + ) throws URISyntaxException { + _logger.info("GET ASPECT urn: {} aspect: {} version: {}", urnStr, aspectName, version); + final Urn urn = Urn.createFromString(urnStr); + return RestliUtils.toTask(() -> { + final AspectWithMetadata aspect = _entityService.getAspectWithMetadata(urn, aspectName, version); + if (aspect == null) { + throw RestliUtils.resourceNotFoundException(); + } + return aspect; + }); + } + +} diff --git a/metadata-ingestion/examples/mce_files/bootstrap_mce.json b/metadata-ingestion/examples/mce_files/bootstrap_mce.json index 499f2798c0b07..4b025a38036e5 100644 --- a/metadata-ingestion/examples/mce_files/bootstrap_mce.json +++ b/metadata-ingestion/examples/mce_files/bootstrap_mce.json @@ -208,7 +208,7 @@ }, "fields": [ { - "fieldPath": "field_foo", + "fieldPath": "field_foo_2", "jsonPath": null, "nullable": false, "description": { @@ -1118,13 +1118,7 @@ "deleted": null }, "chartUrl": null, - "inputs": { - "array": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)" - } - ] - }, + "inputs": ["urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"], "type": null, "access": null, "lastRefreshed": null diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index f46effd31b5dd..326e918a70e90 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -8,6 +8,7 @@ import com.linkedin.data.template.UnionTemplate; import com.linkedin.entity.Entity; import com.linkedin.metadata.PegasusUtils; +import com.linkedin.metadata.aspect.AspectWithMetadata; import com.linkedin.metadata.dao.exception.ModelConversionException; import com.linkedin.metadata.dao.utils.RecordUtils; import com.linkedin.metadata.event.EntityEventProducer; @@ -103,6 +104,11 @@ public abstract RecordTemplate getAspect( @Nonnull final String aspectName, long version); + public abstract AspectWithMetadata getAspectWithMetadata( + @Nonnull final Urn urn, + @Nonnull final String aspectName, + long version); + /** * Retrieves a list of all persisted aspects with a specific name, sorted by corresponding urn. * diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java index 16f03a0eed4e3..923c08ed7e591 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java @@ -183,7 +183,7 @@ public long getMaxVersion(@Nonnull final String urn, @Nonnull final String aspec validateConnection(); List result = _server.find(EbeanAspectV2.class) .where() - .eq("urn", "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)").eq("aspect", "browsePaths") + .eq("urn", urn).eq("aspect", aspectName) .orderBy() .desc("version") .findList(); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index 01b21c55775f4..f01be8c9880a8 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -2,9 +2,13 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; +import com.linkedin.data.DataMap; import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.aspect.Aspect; +import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.metadata.dao.exception.ModelConversionException; import com.linkedin.metadata.dao.utils.RecordUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.ListResult; @@ -12,6 +16,8 @@ import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.HashMap; @@ -24,6 +30,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Value; +import org.apache.commons.lang.reflect.ConstructorUtils; import static com.linkedin.metadata.PegasusUtils.*; @@ -96,6 +103,37 @@ public RecordTemplate getAspect(@Nonnull final Urn urn, @Nonnull final String as .orElse(null); } + @Override + public AspectWithMetadata getAspectWithMetadata(@Nonnull Urn urn, @Nonnull String aspectName, long version) { + AspectWithMetadata result = new AspectWithMetadata(); + + if (version < 0) { + version = _entityDao.getMaxVersion(urn.toString(), aspectName) + version + 1; + } + + final EbeanAspectV2.PrimaryKey primaryKey = new EbeanAspectV2.PrimaryKey(urn.toString(), aspectName, version); + final Optional maybeAspect = Optional.ofNullable(_entityDao.getAspect(primaryKey)); + RecordTemplate aspect = maybeAspect + .map(ebeanAspect -> toAspectRecord(urn, aspectName, ebeanAspect.getMetadata())) + .orElse(null); + + if (aspect == null) { + return null; + } + + Aspect resultAspect = new Aspect(); + + RecordUtils.setSelectedRecordTemplateInUnion( + resultAspect, + aspect + ); +; + result.setAspect(resultAspect); + result.setVersion(version); + + return result; + } + @Override @Nonnull public ListResult listLatestAspects( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/extractor/SnapshotToAspectMap.java b/metadata-io/src/main/java/com/linkedin/metadata/extractor/SnapshotToAspectMap.java index 3c71e369de05d..1ab68be4ca7e8 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/extractor/SnapshotToAspectMap.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/extractor/SnapshotToAspectMap.java @@ -1,7 +1,5 @@ package com.linkedin.metadata.extractor; -import com.google.common.collect.ImmutableList; -import com.linkedin.data.DataMap; import com.linkedin.data.element.DataElement; import com.linkedin.data.it.IterationOrder; import com.linkedin.data.it.ObjectIterator; @@ -9,17 +7,9 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.metadata.PegasusUtils; import com.linkedin.metadata.models.FieldSpec; -import com.linkedin.util.Pair; -import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.commons.collections.map.HashedMap; -import org.apache.commons.lang3.StringUtils; /** @@ -32,10 +22,10 @@ private SnapshotToAspectMap() { /** * Function to extract the fields that match the input fieldSpecs */ - public static Map extractAspectMap(RecordTemplate snapshot) { + public static Map extractAspectMap(RecordTemplate snapshot) { final ObjectIterator iterator = new ObjectIterator(snapshot.data(), snapshot.schema(), IterationOrder.PRE_ORDER); - final Map aspectsByName = new HashMap<>(); + final Map aspectsByName = new HashMap<>(); for (DataElement dataElement = iterator.next(); dataElement != null; dataElement = iterator.next()) { final PathSpec pathSpec = dataElement.getSchemaPathSpec(); @@ -45,7 +35,7 @@ public static Map extractAspectMap(RecordTemplate snapshot) { continue; } String aspectName = PegasusUtils.getAspectNameFromFullyQualifiedName(pathComponents.get(2)); - aspectsByName.put(aspectName, (DataMap) dataElement.getValue()); + aspectsByName.put(aspectName, dataElement); } return aspectsByName; diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/Aspect.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/Aspect.pdl new file mode 100644 index 0000000000000..43c6ae05e9ad3 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/Aspect.pdl @@ -0,0 +1,98 @@ +namespace com.linkedin.metadata.aspect + +import com.linkedin.metadata.key.ChartKey +import com.linkedin.chart.ChartInfo +import com.linkedin.chart.ChartQuery +import com.linkedin.common.Ownership +import com.linkedin.common.Status +import com.linkedin.common.GlobalTags +import com.linkedin.common.BrowsePaths +import com.linkedin.metadata.key.CorpGroupKey +import com.linkedin.identity.CorpGroupInfo +import com.linkedin.metadata.key.CorpUserKey +import com.linkedin.identity.CorpUserEditableInfo +import com.linkedin.identity.CorpUserInfo +import com.linkedin.metadata.key.DashboardKey +import com.linkedin.dashboard.DashboardInfo +import com.linkedin.metadata.key.DataFlowKey +import com.linkedin.datajob.DataFlowInfo +import com.linkedin.metadata.key.DataJobKey +import com.linkedin.datajob.DataJobInfo +import com.linkedin.datajob.DataJobInputOutput +import com.linkedin.metadata.key.DatasetKey +import com.linkedin.dataset.DatasetDeprecation +import com.linkedin.dataset.DatasetProperties +import com.linkedin.dataset.DatasetUpstreamLineage +import com.linkedin.dataset.UpstreamLineage +import com.linkedin.schema.SchemaMetadata +import com.linkedin.schema.EditableSchemaMetadata +import com.linkedin.common.InstitutionalMemory +import com.linkedin.glossary.GlossaryNodeInfo +import com.linkedin.metadata.key.GlossaryNodeKey +import com.linkedin.glossary.GlossaryTermInfo +import com.linkedin.metadata.key.GlossaryTermKey +import com.linkedin.metadata.key.MLFeatureKey +import com.linkedin.ml.metadata.MLFeatureProperties +import com.linkedin.metadata.key.MLModelKey +import com.linkedin.ml.metadata.CaveatsAndRecommendations +import com.linkedin.ml.metadata.EthicalConsiderations +import com.linkedin.ml.metadata.EvaluationData +import com.linkedin.ml.metadata.IntendedUse +import com.linkedin.ml.metadata.Metrics +import com.linkedin.ml.metadata.MLModelFactorPrompts +import com.linkedin.ml.metadata.MLModelProperties +import com.linkedin.ml.metadata.QuantitativeAnalyses +import com.linkedin.ml.metadata.TrainingData +import com.linkedin.ml.metadata.SourceCode +import com.linkedin.metadata.key.TagKey +import com.linkedin.tag.TagProperties + +/** + * A union of all supported metadata aspects for a Chart + */ +typeref Aspect = union[ + ChartKey, + ChartInfo, + ChartQuery, + CorpGroupKey, + CorpGroupInfo, + CorpUserKey, + CorpUserEditableInfo, + CorpUserInfo, + DashboardKey, + DashboardInfo, + DataFlowKey, + DataFlowInfo, + DataJobKey, + DataJobInfo, + DataJobInputOutput, + DatasetDeprecation, + DatasetProperties, + UpstreamLineage, + SchemaMetadata, + EditableSchemaMetadata, + InstitutionalMemory, + GlossaryNodeInfo, + GlossaryNodeKey, + GlossaryTermInfo, + GlossaryTermKey, + MLFeatureKey, + MLFeatureProperties, + MLModelKey, + CaveatsAndRecommendations, + EthicalConsiderations, + EvaluationData, + IntendedUse, + Metrics, + MLModelFactorPrompts, + MLModelProperties, + QuantitativeAnalyses, + TrainingData, + SourceCode, + TagKey, + TagProperties, + Ownership, + Status, + GlobalTags, + BrowsePaths, +] diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/AspectWithMetadata.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/AspectWithMetadata.pdl new file mode 100644 index 0000000000000..e75cbc4b94c26 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/AspectWithMetadata.pdl @@ -0,0 +1,8 @@ +namespace com.linkedin.metadata.aspect + +import com.linkedin.common.AuditStamp + +record AspectWithMetadata { + aspect: Aspect + version: long +} From ddc1bcb20c30a3a349e15b8426ed8f4853b09f97 Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Thu, 10 Jun 2021 16:09:44 -0700 Subject: [PATCH 4/8] small fixes & adding tests --- .../datahub/graphql/AspectLoadKey.java | 1 - .../datahub/graphql/GmsGraphQLEngine.java | 8 ++--- .../graphql/resolvers/ResolverUtils.java | 11 +++++- .../resolvers/load/AspectResolver.java | 11 +++--- .../type/AspectInterfaceTypeResolver.java | 10 +++--- .../graphql/types/aspect/AspectType.java | 13 ++----- .../types/dataset/mappers/DatasetMapper.java | 2 +- .../dataset/mappers/SchemaMetadataMapper.java | 2 +- .../src/main/resources/gms.graphql | 1 - datahub-web-react/src/graphql/dataset.graphql | 3 +- .../entity/EbeanEntityServiceTest.java | 36 +++++++++++++++++++ 11 files changed, 63 insertions(+), 35 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java index 3be600868549a..ac2ee8243d6ee 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java @@ -2,7 +2,6 @@ import lombok.Data; - @Data public class AspectLoadKey { private String aspectName; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 1f797d7e76de8..b57ba8652c4c5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -104,8 +104,7 @@ public class GmsGraphQLEngine { public static final GlossaryTermType GLOSSARY_TERM_TYPE = new GlossaryTermType(GmsClientFactory.getEntitiesClient()); public static final AspectType ASPECT_TYPE = new AspectType(GmsClientFactory.getAspectsClient()); - - /** + /** * Configures the graph objects that can be fetched primary key. */ public static final List> ENTITY_TYPES = ImmutableList.of( @@ -134,10 +133,7 @@ public class GmsGraphQLEngine { /** * Configures all graph objects */ - public static final List> LOADABLE_TYPES = Stream.concat( - ENTITY_TYPES.stream(), - RELATIONSHIP_TYPES.stream() - ).collect(Collectors.toList()); + public static final List> LOADABLE_TYPES = Stream.concat(ENTITY_TYPES.stream(), RELATIONSHIP_TYPES.stream()).collect(Collectors.toList()); /** * Configures the graph objects for owner diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java index f4860f6a3426d..c991ec65eda5e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java @@ -20,12 +20,16 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang.reflect.ConstructorUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ResolverUtils { private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final Logger _logger = LoggerFactory.getLogger("ResolverUtils"); + private ResolverUtils() { } @Nonnull @@ -69,7 +73,6 @@ public static Map buildFacetFilters(@Nullable List> { @@ -36,12 +32,13 @@ public CompletableFuture get(DataFetchingEnvironment environment) { Long version = environment.getArgument("version"); String urn = ((Entity) environment.getSource()).getUrn(); + // first, we try fetching the aspect from the local cache AspectWithMetadata aspectFromContext = ResolverUtils.getAspectFromLocalContext(environment); if (aspectFromContext != null) { return CompletableFuture.completedFuture(AspectMapper.map(aspectFromContext)); } - // if the aspect is not in the cache, we need to fetch it + // if the aspect is not in the cache, we need to fetch it from GMS Aspect Resource return loader.load(new AspectLoadKey(urn, fieldName, version)); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/AspectInterfaceTypeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/AspectInterfaceTypeResolver.java index f7894c8ed474d..45998bdae45b0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/AspectInterfaceTypeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/AspectInterfaceTypeResolver.java @@ -5,16 +5,16 @@ import graphql.schema.TypeResolver; /** - * Responsible for resolving the {@link com.linkedin.datahub.graphql.generated.Entity} interface type. + * Responsible for resolving the {@link com.linkedin.datahub.graphql.generated.Aspect} interface type. */ public class AspectInterfaceTypeResolver implements TypeResolver { public AspectInterfaceTypeResolver() { } - @Override public GraphQLObjectType getType(TypeResolutionEnvironment env) { - String fieldName = env.getField().getName(); - String typeName = fieldName.substring(0, 1).toUpperCase() + fieldName.substring(1); - return env.getSchema().getObjectType(typeName); + // TODO(Gabe): Fill this out. This method is not called today. We will need to fill this + // out in the case we ever want to return fields of type Aspect in graphql. Right now + // we just use Aspect to define the shared `version` field. + return null; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java index 40a04ddbd83cd..edfdfec722c3d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java @@ -1,21 +1,12 @@ package com.linkedin.datahub.graphql.types.aspect; -import com.linkedin.data.DataMap; -import com.linkedin.data.template.RecordTemplate; import com.linkedin.datahub.graphql.AspectLoadKey; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Aspect; -import com.linkedin.datahub.graphql.generated.DownstreamEntityRelationships; -import com.linkedin.datahub.graphql.types.relationships.mappers.DownstreamEntityRelationshipsMapper; -import com.linkedin.entity.Entity; import com.linkedin.entity.client.AspectClient; -import com.linkedin.entity.client.EntityClient; -import com.linkedin.lineage.client.Lineages; import com.linkedin.metadata.aspect.AspectWithMetadata; -import com.linkedin.metadata.query.RelationshipDirection; import com.linkedin.r2.RemoteInvocationException; import graphql.execution.DataFetcherResult; -import java.net.URISyntaxException; import java.util.List; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -28,8 +19,8 @@ public AspectType(final AspectClient aspectClient) { _aspectClient = aspectClient; } /** - * Retrieves an list of entities given a list of urn strings. The list returned is expected to - * be of same length of the list of urns, where nulls are provided in place of an entity object if an entity cannot be found. + * Retrieves an list of aspects given a list of {@link AspectLoadKey} structs. The list returned is expected to + * be of same length of the list of keys, where nulls are provided in place of an aspect object if an entity cannot be found. * @param keys to retrieve * @param context the {@link QueryContext} corresponding to the request. */ diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index e4d873729c040..9bd61ddcab737 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -46,7 +46,7 @@ public Dataset apply(@Nonnull final com.linkedin.dataset.Dataset dataset) { result.setExternalUrl(dataset.getExternalUrl().toString()); } if (dataset.hasSchemaMetadata()) { - // result.setSchema(SchemaMetadataMapper.map(dataset.getSchemaMetadata())); + result.setSchema(SchemaMapper.map(dataset.getSchemaMetadata())); } if (dataset.hasEditableSchemaMetadata()) { result.setEditableSchemaMetadata(EditableSchemaMetadataMapper.map(dataset.getEditableSchemaMetadata())); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java index 1a4b86b68c5ac..9bcaa13e9bcbf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java @@ -33,7 +33,7 @@ public com.linkedin.datahub.graphql.generated.SchemaMetadata apply(@Nonnull fina result.setPrimaryKeys(input.getPrimaryKeys()); result.setFields(input.getFields().stream().map(SchemaFieldMapper::map).collect(Collectors.toList())); result.setPlatformSchema(PlatformSchemaMapper.map(input.getPlatformSchema())); - result.setVersion(inputWithMetadata.getVersion()); + result.setAspectVersion(inputWithMetadata.getVersion()); return result; } } diff --git a/datahub-graphql-core/src/main/resources/gms.graphql b/datahub-graphql-core/src/main/resources/gms.graphql index c39c48cf175c8..0951fbd36ee3e 100644 --- a/datahub-graphql-core/src/main/resources/gms.graphql +++ b/datahub-graphql-core/src/main/resources/gms.graphql @@ -23,7 +23,6 @@ interface Entity { interface Aspect { aspectVersion: Long - createdAt: Long } interface EntityWithRelationships implements Entity { diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index 7d2248390fe8f..4775342c48528 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -47,7 +47,7 @@ query getDataset($urn: String!) { } } } - schemaMetadata { + schema { datasetUrn name platformUrn @@ -110,6 +110,7 @@ query getDataset($urn: String!) { } } primaryKeys + aspectVersion } editableSchemaMetadata { editableSchemaFieldInfo { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java index 7305dc4e433de..e4344b9390ee9 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java @@ -9,6 +9,8 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.identity.CorpUserInfo; import com.linkedin.metadata.PegasusUtils; +import com.linkedin.metadata.aspect.Aspect; +import com.linkedin.metadata.aspect.AspectWithMetadata; import com.linkedin.metadata.aspect.CorpUserAspect; import com.linkedin.metadata.aspect.CorpUserAspectArray; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; @@ -242,6 +244,40 @@ public void testUpdateGetAspect() throws Exception { verifyNoMoreInteractions(_mockProducer); } + @Test + public void testGetAspectAtVersion() throws Exception { + // Test Writing a CorpUser Entity + Urn entityUrn = Urn.createFromString("urn:li:corpuser:test"); + + String aspectName = PegasusUtils.getAspectNameFromSchema(new CorpUserInfo().schema()); + + // Ingest CorpUserInfo Aspect #1 + CorpUserInfo writeAspect = createCorpUserInfo("email@test.com"); + + // Validate retrieval of CorpUserInfo Aspect #1 + _entityService.updateAspect(entityUrn, aspectName, writeAspect, TEST_AUDIT_STAMP, 1, true); + + + AspectWithMetadata writtenAspectWithMetadata = new AspectWithMetadata(); + writtenAspectWithMetadata.setAspect(Aspect.create(writeAspect)); + writtenAspectWithMetadata.setVersion(1); + + AspectWithMetadata readAspect1 = _entityService.getAspectWithMetadata(entityUrn, aspectName, 1); + assertTrue(DataTemplateUtil.areEqual(writtenAspectWithMetadata, readAspect1)); + verify(_mockProducer, times(1)).produceMetadataAuditEvent( + Mockito.eq(entityUrn), + Mockito.eq(null), + Mockito.any()); + + AspectWithMetadata readAspect2 = _entityService.getAspectWithMetadata(entityUrn, aspectName, -1); + assertTrue(DataTemplateUtil.areEqual(writtenAspectWithMetadata, readAspect2)); + + AspectWithMetadata readAspectVersion0 = _entityService.getAspectWithMetadata(entityUrn, aspectName, 0); + assertFalse(DataTemplateUtil.areEqual(writtenAspectWithMetadata, readAspectVersion0)); + + verifyNoMoreInteractions(_mockProducer); + } + @Nonnull private com.linkedin.entity.Entity createCorpUserEntity(Urn entityUrn, String email) throws Exception { CorpuserUrn corpuserUrn = CorpuserUrn.createFromUrn(entityUrn); From 3a83716202323dcbe4ee987b50d98c69f74ab71a Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Thu, 10 Jun 2021 16:48:34 -0700 Subject: [PATCH 5/8] fixing tests --- .../datahub/graphql/AspectLoadKey.java | 8 ++--- .../graphql/resolvers/ResolverUtils.java | 3 -- .../graphql/types/aspect/AspectMapper.java | 3 -- .../graphql/types/chart/ChartType.java | 6 +++- .../types/dashboard/DashboardType.java | 7 ++-- .../graphql/types/dataflow/DataFlowType.java | 7 ++-- .../graphql/types/datajob/DataJobType.java | 7 ++-- .../dataset/mappers/SchemaMetadataMapper.java | 1 - .../types/glossary/GlossaryTermType.java | 8 +++-- .../graphql/types/mlmodel/MLModelType.java | 7 ++-- .../datahub/graphql/types/tag/TagType.java | 5 ++- datahub-web-react/src/graphql/dataset.graphql | 33 ------------------- .../entity/ebean/EbeanEntityService.java | 5 --- 13 files changed, 38 insertions(+), 62 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java index ac2ee8243d6ee..e1113d4bc3268 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java @@ -8,9 +8,9 @@ public class AspectLoadKey { private String urn; private Long version; - public AspectLoadKey(String _urn, String _aspectName, Long _version) { - urn = _urn; - version = _version; - aspectName = _aspectName; + public AspectLoadKey(String urn, String aspectName, Long version) { + this.urn = urn; + this.version = version; + this.aspectName = aspectName; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java index c991ec65eda5e..01dd94b617ff5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java @@ -4,14 +4,11 @@ import com.linkedin.data.DataMap; import com.linkedin.data.element.DataElement; import com.linkedin.datahub.graphql.exception.ValidationException; -import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.FacetFilterInput; -import com.linkedin.datahub.graphql.types.aspect.AspectMapper; import com.linkedin.metadata.aspect.AspectWithMetadata; import graphql.schema.DataFetchingEnvironment; import java.lang.reflect.InvocationTargetException; -import java.util.concurrent.CompletableFuture; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.Collections; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java index 7b8befd3a5ec2..32f941d517d18 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java @@ -1,13 +1,10 @@ package com.linkedin.datahub.graphql.types.aspect; import com.linkedin.datahub.graphql.generated.Aspect; -import com.linkedin.datahub.graphql.types.dataset.mappers.SchemaMapper; import com.linkedin.datahub.graphql.types.dataset.mappers.SchemaMetadataMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.metadata.aspect.AspectWithMetadata; -import com.linkedin.schema.SchemaMetadata; import javax.annotation.Nonnull; -import javax.annotation.Nullable; public class AspectMapper implements ModelMapper { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java index f46206855e51a..16e9fdc10f2dd 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java @@ -25,6 +25,7 @@ import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.configs.ChartSearchConfig; +import com.linkedin.metadata.extractor.SnapshotToAspectMap; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; @@ -86,7 +87,10 @@ public List> batchLoad(@Nonnull List urns, @Non } return gmsResults.stream() .map(gmsChart -> gmsChart == null ? null - : DataFetcherResult.newResult().data(ChartSnapshotMapper.map(gmsChart.getValue().getChartSnapshot())).build()) + : DataFetcherResult.newResult() + .data(ChartSnapshotMapper.map(gmsChart.getValue().getChartSnapshot())) + .localContext(SnapshotToAspectMap.extractAspectMap(gmsChart.getValue().getChartSnapshot())) + .build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load Charts", e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java index beb74f61079c0..4d81eeba36776 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java @@ -27,6 +27,7 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.Entity; import com.linkedin.metadata.configs.DashboardSearchConfig; +import com.linkedin.metadata.extractor.SnapshotToAspectMap; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; @@ -88,8 +89,10 @@ public List> batchLoad(@Nonnull List urns, } return gmsResults.stream() .map(gmsDashboard -> gmsDashboard == null ? null - : DataFetcherResult.newResult().data(DashboardSnapshotMapper.map( - gmsDashboard.getValue().getDashboardSnapshot())).build()) + : DataFetcherResult.newResult() + .data(DashboardSnapshotMapper.map(gmsDashboard.getValue().getDashboardSnapshot())) + .localContext(SnapshotToAspectMap.extractAspectMap(gmsDashboard.getValue().getDashboardSnapshot())) + .build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load Dashboards", e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java index 06a47bbb31ad3..6314b55b0d030 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java @@ -29,6 +29,7 @@ import com.linkedin.entity.Entity; import com.linkedin.metadata.aspect.DataFlowAspect; import com.linkedin.metadata.dao.utils.ModelUtils; +import com.linkedin.metadata.extractor.SnapshotToAspectMap; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; @@ -89,8 +90,10 @@ public List> batchLoad(final List urns, fina .map(flowUrn -> dataFlowMap.getOrDefault(flowUrn, null)).collect(Collectors.toList()); return gmsResults.stream() - .map(gmsDataFlow -> gmsDataFlow == null ? null : DataFetcherResult.newResult().data(DataFlowSnapshotMapper.map( - gmsDataFlow.getValue().getDataFlowSnapshot())).build()) + .map(gmsDataFlow -> gmsDataFlow == null ? null : DataFetcherResult.newResult() + .data(DataFlowSnapshotMapper.map(gmsDataFlow.getValue().getDataFlowSnapshot())) + .localContext(SnapshotToAspectMap.extractAspectMap(gmsDataFlow.getValue().getDataFlowSnapshot())) + .build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load DataFlows", e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java index 208b5eb8ef040..f4a9cf27ec6bb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java @@ -29,6 +29,7 @@ import com.linkedin.entity.Entity; import com.linkedin.metadata.aspect.DataJobAspect; import com.linkedin.metadata.dao.utils.ModelUtils; +import com.linkedin.metadata.extractor.SnapshotToAspectMap; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; @@ -91,8 +92,10 @@ public List> batchLoad(final List urns, final return gmsResults.stream() .map(gmsDataJob -> gmsDataJob == null ? null - : DataFetcherResult.newResult().data( - DataJobSnapshotMapper.map(gmsDataJob.getValue().getDataJobSnapshot())).build()) + : DataFetcherResult.newResult() + .data(DataJobSnapshotMapper.map(gmsDataJob.getValue().getDataJobSnapshot())) + .localContext(SnapshotToAspectMap.extractAspectMap(gmsDataJob.getValue().getDataJobSnapshot())) + .build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load DataJobs", e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java index 9bcaa13e9bcbf..0cfc891ac90d9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java @@ -1,6 +1,5 @@ package com.linkedin.datahub.graphql.types.dataset.mappers; -import com.linkedin.datahub.graphql.generated.Schema; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.metadata.aspect.AspectWithMetadata; import com.linkedin.schema.SchemaMetadata; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java index 031a049cd8d33..34a75097b8ad4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java @@ -22,6 +22,7 @@ import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.Entity; +import com.linkedin.metadata.extractor.SnapshotToAspectMap; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; @@ -77,9 +78,10 @@ public List> batchLoad(final List urns, return gmsResults.stream() .map(gmsGlossaryTerm -> gmsGlossaryTerm == null ? null - : DataFetcherResult.newResult().data( - GlossaryTermSnapshotMapper.map( - gmsGlossaryTerm.getValue().getGlossaryTermSnapshot())).build()) + : DataFetcherResult.newResult() + .data(GlossaryTermSnapshotMapper.map(gmsGlossaryTerm.getValue().getGlossaryTermSnapshot())) + .localContext(SnapshotToAspectMap.extractAspectMap(gmsGlossaryTerm.getValue().getGlossaryTermSnapshot())) + .build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load GlossaryTerms", e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java index 3e30f45930a01..596f3837e32b6 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java @@ -6,6 +6,7 @@ import com.linkedin.datahub.graphql.types.mlmodel.mappers.MLModelSnapshotMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.Entity; +import com.linkedin.metadata.extractor.SnapshotToAspectMap; import com.linkedin.metadata.query.SearchResult; import graphql.execution.DataFetcherResult; import java.util.List; @@ -66,8 +67,10 @@ public List> batchLoad(final List urns, final return gmsResults.stream() .map(gmsMlModel -> gmsMlModel == null ? null - : DataFetcherResult.newResult().data(MLModelSnapshotMapper.map( - gmsMlModel.getValue().getMLModelSnapshot())).build()) + : DataFetcherResult.newResult() + .data(MLModelSnapshotMapper.map(gmsMlModel.getValue().getMLModelSnapshot())) + .localContext(SnapshotToAspectMap.extractAspectMap(gmsMlModel.getValue().getMLModelSnapshot())) + .build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load MLModels", e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java index 2812469745460..c4daadfff577f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/tag/TagType.java @@ -30,6 +30,7 @@ import com.linkedin.metadata.aspect.TagAspect; import com.linkedin.metadata.configs.TagSearchConfig; import com.linkedin.metadata.dao.utils.ModelUtils; +import com.linkedin.metadata.extractor.SnapshotToAspectMap; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.SearchResult; import com.linkedin.metadata.snapshot.Snapshot; @@ -92,7 +93,9 @@ public List> batchLoad(final List urns, final Que return gmsResults.stream() .map(gmsTag -> gmsTag == null ? null : DataFetcherResult.newResult() - .data(TagSnapshotMapper.map(gmsTag.getValue().getTagSnapshot())).build()) + .data(TagSnapshotMapper.map(gmsTag.getValue().getTagSnapshot())) + .localContext(SnapshotToAspectMap.extractAspectMap(gmsTag.getValue().getTagSnapshot())) + .build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to batch load Tags", e); diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index 4775342c48528..7d23bf4204ebe 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -79,39 +79,6 @@ query getDataset($urn: String!) { } primaryKeys } - pastSchema: schemaMetadata(version: -1) { - datasetUrn - name - platformUrn - version - hash - platformSchema { - ... on TableSchema { - schema - } - ... on KeyValueSchema { - keySchema - valueSchema - } - } - fields { - fieldPath - jsonPath - nullable - description - type - nativeDataType - recursive - globalTags { - ...globalTagsFields - } - glossaryTerms { - ...glossaryTerms - } - } - primaryKeys - aspectVersion - } editableSchemaMetadata { editableSchemaFieldInfo { fieldPath diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index f01be8c9880a8..ebf6e38149f5c 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -2,13 +2,11 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; -import com.linkedin.data.DataMap; import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; import com.linkedin.metadata.aspect.Aspect; import com.linkedin.metadata.aspect.AspectWithMetadata; -import com.linkedin.metadata.dao.exception.ModelConversionException; import com.linkedin.metadata.dao.utils.RecordUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.ListResult; @@ -16,8 +14,6 @@ import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.HashMap; @@ -30,7 +26,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Value; -import org.apache.commons.lang.reflect.ConstructorUtils; import static com.linkedin.metadata.PegasusUtils.*; From e8768a3e45d5b99bb8d1f5d8a73140a2cf81f15e Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Thu, 10 Jun 2021 17:26:01 -0700 Subject: [PATCH 6/8] updating snapshots --- .../com.linkedin.chart.charts.snapshot.json | 8 +++- .../com.linkedin.entity.aspects.snapshot.json | 45 ++++++++++++++++--- ...com.linkedin.entity.entities.snapshot.json | 14 ++++-- .../com.linkedin.ml.mlModels.snapshot.json | 6 +-- 4 files changed, 58 insertions(+), 15 deletions(-) diff --git a/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json index 0543e176f8867..968bbd3562b12 100644 --- a/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json @@ -158,7 +158,13 @@ "items" : "ChartDataSourceType" }, "doc" : "Data sources for the chart", - "optional" : true + "optional" : true, + "Relationship" : { + "/*/string" : { + "entityTypes" : [ "dataset" ], + "name" : "Consumes" + } + } }, { "name" : "type", "type" : { diff --git a/gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json index 4cafa0121ae9a..48b3a22eab629 100644 --- a/gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json @@ -158,7 +158,13 @@ "items" : "ChartDataSourceType" }, "doc" : "Data sources for the chart", - "optional" : true + "optional" : true, + "Relationship" : { + "/*/string" : { + "entityTypes" : [ "dataset" ], + "name" : "Consumes" + } + } }, { "name" : "type", "type" : { @@ -641,10 +647,12 @@ "name" : "MLFeatureDataType", "namespace" : "com.linkedin.common", "doc" : "MLFeature Data Type", - "symbols" : [ "USELESS", "NOMINAL", "ORDINAL", "BINARY", "COUNT", "TIME", "INTERVAL", "IMAGE", "VIDEO", "AUDIO", "TEXT", "MAP", "SEQUENCE", "SET" ], + "symbols" : [ "USELESS", "NOMINAL", "ORDINAL", "BINARY", "COUNT", "TIME", "INTERVAL", "IMAGE", "VIDEO", "AUDIO", "TEXT", "MAP", "SEQUENCE", "SET", "CONTINUOUS", "BYTE", "UNKNOWN" ], "symbolDocs" : { "AUDIO" : "Audio Data", "BINARY" : "Binary data is discrete data that can be in only one of two categories — either yes or no, 1 or 0, off or on, etc", + "BYTE" : "Bytes data are binary-encoded values that can represent complex objects.", + "CONTINUOUS" : "Continuous data are made of uncountable values, often the result of a measurement such as height, weight, age etc.", "COUNT" : "Count data is discrete whole number data — no negative numbers here.\nCount data often has many small values, such as zero and one.", "IMAGE" : "Image Data", "INTERVAL" : "Interval data has equal spaces between the numbers and does not represent a temporal pattern.\nExamples include percentages, temperatures, and income.", @@ -655,6 +663,7 @@ "SET" : "Set Data Type ex: set, frozenset", "TEXT" : "Text Data", "TIME" : "Time data is a cyclical, repeating continuous form of data.\nThe relevant time features can be any period— daily, weekly, monthly, annual, etc.", + "UNKNOWN" : "Unknown data are data that we don't know the type for.", "USELESS" : "Useless data is unique, discrete data with no potential relationship with the outcome variable.\nA useless feature has high cardinality. An example would be bank account numbers that were generated randomly.", "VIDEO" : "Video Data" } @@ -2014,15 +2023,23 @@ "type" : "record", "name" : "MLFeatureKey", "namespace" : "com.linkedin.metadata.key", - "doc" : "Key for an ML model", + "doc" : "Key for an MLFeature", "fields" : [ { "name" : "featureNamespace", "type" : "string", - "doc" : "Namespace for the feature" + "doc" : "Namespace for the feature", + "Searchable" : { + "fieldType" : "TEXT_PARTIAL" + } }, { "name" : "name", "type" : "string", - "doc" : "Name of the feature" + "doc" : "Name of the feature", + "Searchable" : { + "boostScore" : 8.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } } ], "Aspect" : { "name" : "mlFeatureKey" @@ -2047,6 +2064,20 @@ "type" : "com.linkedin.common.VersionTag", "doc" : "Version of the MLFeature", "optional" : true + }, { + "name" : "sources", + "type" : { + "type" : "array", + "items" : "com.linkedin.common.Urn" + }, + "doc" : "Source of the MLFeature", + "optional" : true, + "Relationship" : { + "/*" : { + "entityTypes" : [ "dataset" ], + "name" : "DerivedFrom" + } + } } ], "Aspect" : { "name" : "mlFeatureProperties" @@ -2061,8 +2092,8 @@ "type" : "com.linkedin.common.Urn", "doc" : "Standardized platform urn for the model", "Searchable" : { - "boostScore" : 0.1, - "fieldType" : "TEXT_PARTIAL" + "addToFilters" : true, + "fieldType" : "URN" } }, { "name" : "name", diff --git a/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 7c2206315e113..1aa5a7a434272 100644 --- a/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -158,7 +158,13 @@ "items" : "ChartDataSourceType" }, "doc" : "Data sources for the chart", - "optional" : true + "optional" : true, + "Relationship" : { + "/*/string" : { + "entityTypes" : [ "dataset" ], + "name" : "Consumes" + } + } }, { "name" : "type", "type" : { @@ -2644,8 +2650,8 @@ "type" : "com.linkedin.common.Urn", "doc" : "Standardized platform urn for the model", "Searchable" : { - "boostScore" : 0.1, - "fieldType" : "TEXT_PARTIAL" + "addToFilters" : true, + "fieldType" : "URN" } }, { "name" : "name", @@ -3937,4 +3943,4 @@ } } } -} +} \ No newline at end of file diff --git a/gms/api/src/main/snapshot/com.linkedin.ml.mlModels.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.ml.mlModels.snapshot.json index 19cdc88da2cea..356c3e029a026 100644 --- a/gms/api/src/main/snapshot/com.linkedin.ml.mlModels.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.ml.mlModels.snapshot.json @@ -481,8 +481,8 @@ "type" : "com.linkedin.common.Urn", "doc" : "Standardized platform urn for the model", "Searchable" : { - "boostScore" : 0.1, - "fieldType" : "TEXT_PARTIAL" + "addToFilters" : true, + "fieldType" : "URN" } }, { "name" : "name", @@ -1680,4 +1680,4 @@ } } } -} +} \ No newline at end of file From 5e20b2f139669fd148f6d5e04c20e207ae0f2408 Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Mon, 14 Jun 2021 17:08:28 -0700 Subject: [PATCH 7/8] renaming aspectloadkey and aspectwithmetadata --- .../datahub/graphql/GmsGraphQLEngine.java | 2 +- ...ctLoadKey.java => VersionedAspectKey.java} | 4 ++-- .../graphql/resolvers/ResolverUtils.java | 8 ++++---- .../resolvers/load/AspectResolver.java | 10 +++++----- .../graphql/types/aspect/AspectMapper.java | 8 ++++---- .../graphql/types/aspect/AspectType.java | 10 +++++----- .../dataset/mappers/SchemaMetadataMapper.java | 8 ++++---- .../addon/constants/metadata/aspect.ts | 4 ++-- .../com.linkedin.entity.aspects.restspec.json | 4 ++-- .../com.linkedin.entity.aspects.snapshot.json | 6 +++--- .../linkedin/entity/client/AspectClient.java | 4 ++-- .../resources/entity/AspectResource.java | 8 ++++---- .../metadata/entity/EntityService.java | 4 ++-- .../entity/ebean/EbeanEntityService.java | 6 +++--- .../entity/EbeanEntityServiceTest.java | 20 +++++++++---------- ...ctWithMetadata.pdl => VersionedAspect.pdl} | 2 +- 16 files changed, 54 insertions(+), 54 deletions(-) rename datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/{AspectLoadKey.java => VersionedAspectKey.java} (67%) rename metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/{AspectWithMetadata.pdl => VersionedAspect.pdl} (80%) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index b57ba8652c4c5..d7cfcec78f17c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -542,7 +542,7 @@ private static DataLoader> createDataLoader(fin }), loaderOptions); } - private static DataLoader> createAspectLoader(final QueryContext queryContext) { + private static DataLoader> createAspectLoader(final QueryContext queryContext) { BatchLoaderContextProvider contextProvider = () -> queryContext; DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setBatchLoaderContextProvider(contextProvider); return DataLoader.newDataLoader((keys, context) -> CompletableFuture.supplyAsync(() -> { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/VersionedAspectKey.java similarity index 67% rename from datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java rename to datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/VersionedAspectKey.java index e1113d4bc3268..b0c0436ffd891 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/AspectLoadKey.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/VersionedAspectKey.java @@ -3,12 +3,12 @@ import lombok.Data; @Data -public class AspectLoadKey { +public class VersionedAspectKey { private String aspectName; private String urn; private Long version; - public AspectLoadKey(String urn, String aspectName, Long version) { + public VersionedAspectKey(String urn, String aspectName, Long version) { this.urn = urn; this.version = version; this.aspectName = aspectName; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java index 01dd94b617ff5..82cfb4fa85cb3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java @@ -6,7 +6,7 @@ import com.linkedin.datahub.graphql.exception.ValidationException; import com.linkedin.datahub.graphql.generated.FacetFilterInput; -import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.metadata.aspect.VersionedAspect; import graphql.schema.DataFetchingEnvironment; import java.lang.reflect.InvocationTargetException; import javax.annotation.Nonnull; @@ -25,7 +25,7 @@ public class ResolverUtils { private static final ObjectMapper MAPPER = new ObjectMapper(); - private static final Logger _logger = LoggerFactory.getLogger("ResolverUtils"); + private static final Logger _logger = LoggerFactory.getLogger(ResolverUtils.class.getName()); private ResolverUtils() { } @@ -67,7 +67,7 @@ public static Map buildFacetFilters(@Nullable List get(DataFetchingEnvironment environment) { - final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader("Aspect"); + final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader("Aspect"); String fieldName = environment.getField().getName(); Long version = environment.getArgument("version"); String urn = ((Entity) environment.getSource()).getUrn(); // first, we try fetching the aspect from the local cache - AspectWithMetadata aspectFromContext = ResolverUtils.getAspectFromLocalContext(environment); + VersionedAspect aspectFromContext = ResolverUtils.getAspectFromLocalContext(environment); if (aspectFromContext != null) { return CompletableFuture.completedFuture(AspectMapper.map(aspectFromContext)); } // if the aspect is not in the cache, we need to fetch it from GMS Aspect Resource - return loader.load(new AspectLoadKey(urn, fieldName, version)); + return loader.load(new VersionedAspectKey(urn, fieldName, version)); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java index 32f941d517d18..2accd6b846555 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectMapper.java @@ -3,20 +3,20 @@ import com.linkedin.datahub.graphql.generated.Aspect; import com.linkedin.datahub.graphql.types.dataset.mappers.SchemaMetadataMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; -import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.metadata.aspect.VersionedAspect; import javax.annotation.Nonnull; -public class AspectMapper implements ModelMapper { +public class AspectMapper implements ModelMapper { public static final AspectMapper INSTANCE = new AspectMapper(); - public static Aspect map(@Nonnull final AspectWithMetadata restliAspect) { + public static Aspect map(@Nonnull final VersionedAspect restliAspect) { return INSTANCE.apply(restliAspect); } @Override - public Aspect apply(@Nonnull final AspectWithMetadata restliAspect) { + public Aspect apply(@Nonnull final VersionedAspect restliAspect) { if (restliAspect.getAspect().isSchemaMetadata()) { return SchemaMetadataMapper.map(restliAspect); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java index edfdfec722c3d..21d9bed26a7bb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/aspect/AspectType.java @@ -1,10 +1,10 @@ package com.linkedin.datahub.graphql.types.aspect; -import com.linkedin.datahub.graphql.AspectLoadKey; +import com.linkedin.datahub.graphql.VersionedAspectKey; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Aspect; import com.linkedin.entity.client.AspectClient; -import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.r2.RemoteInvocationException; import graphql.execution.DataFetcherResult; import java.util.List; @@ -19,16 +19,16 @@ public AspectType(final AspectClient aspectClient) { _aspectClient = aspectClient; } /** - * Retrieves an list of aspects given a list of {@link AspectLoadKey} structs. The list returned is expected to + * Retrieves an list of aspects given a list of {@link VersionedAspectKey} structs. The list returned is expected to * be of same length of the list of keys, where nulls are provided in place of an aspect object if an entity cannot be found. * @param keys to retrieve * @param context the {@link QueryContext} corresponding to the request. */ - public List> batchLoad(@Nonnull List keys, @Nonnull QueryContext context) throws Exception { + public List> batchLoad(@Nonnull List keys, @Nonnull QueryContext context) throws Exception { try { return keys.stream().map(key -> { try { - AspectWithMetadata entity = _aspectClient.getAspect(key.getUrn(), key.getAspectName(), key.getVersion()); + VersionedAspect entity = _aspectClient.getAspect(key.getUrn(), key.getAspectName(), key.getVersion()); return DataFetcherResult.newResult().data(AspectMapper.map(entity)).build(); } catch (RemoteInvocationException e) { throw new RuntimeException(String.format("Failed to load Aspect for entity %s", key.getUrn()), e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java index 0cfc891ac90d9..9b38fbf834737 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java @@ -1,22 +1,22 @@ package com.linkedin.datahub.graphql.types.dataset.mappers; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; -import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.schema.SchemaMetadata; import java.util.stream.Collectors; import javax.annotation.Nonnull; -public class SchemaMetadataMapper implements ModelMapper { +public class SchemaMetadataMapper implements ModelMapper { public static final SchemaMetadataMapper INSTANCE = new SchemaMetadataMapper(); - public static com.linkedin.datahub.graphql.generated.SchemaMetadata map(@Nonnull final AspectWithMetadata metadata) { + public static com.linkedin.datahub.graphql.generated.SchemaMetadata map(@Nonnull final VersionedAspect metadata) { return INSTANCE.apply(metadata); } @Override - public com.linkedin.datahub.graphql.generated.SchemaMetadata apply(@Nonnull final AspectWithMetadata inputWithMetadata) { + public com.linkedin.datahub.graphql.generated.SchemaMetadata apply(@Nonnull final VersionedAspect inputWithMetadata) { SchemaMetadata input = inputWithMetadata.getAspect().getSchemaMetadata(); final com.linkedin.datahub.graphql.generated.SchemaMetadata result = new com.linkedin.datahub.graphql.generated.SchemaMetadata(); diff --git a/datahub-web/@datahub/metadata-types/addon/constants/metadata/aspect.ts b/datahub-web/@datahub/metadata-types/addon/constants/metadata/aspect.ts index b20be77a121da..38cbbdb5ec7b6 100644 --- a/datahub-web/@datahub/metadata-types/addon/constants/metadata/aspect.ts +++ b/datahub-web/@datahub/metadata-types/addon/constants/metadata/aspect.ts @@ -6,7 +6,7 @@ type AspectsOfSnapshot = Snapshot extends { aspects: Array) => boolean)} */ -const getMetadataAspectWithMetadataAspectKey = ( +const getMetadataVersionedAspectAspectKey = ( metadataAspectKey: AspectKey ): ((aspect: Aspect) => boolean) => (aspect: Aspect): boolean => aspect.hasOwnProperty(metadataAspectKey); @@ -40,7 +40,7 @@ export const getMetadataAspect = < ): AspectsOfSnapshot[AspectKey] | undefined => { const { aspects = [] } = snapshot || {}; // Find the aspect with the metadata key that matches the passed in metadataAspectKey - const [relevantAspect] = aspects.filter(getMetadataAspectWithMetadataAspectKey(metadataAspectKey)); + const [relevantAspect] = aspects.filter(getMetadataVersionedAspectAspectKey(metadataAspectKey)); return relevantAspect ? getMetadataAspectValue(metadataAspectKey, relevantAspect) : undefined; }; diff --git a/gms/api/src/main/idl/com.linkedin.entity.aspects.restspec.json b/gms/api/src/main/idl/com.linkedin.entity.aspects.restspec.json index c033f044b24d1..98a372f180f37 100644 --- a/gms/api/src/main/idl/com.linkedin.entity.aspects.restspec.json +++ b/gms/api/src/main/idl/com.linkedin.entity.aspects.restspec.json @@ -2,7 +2,7 @@ "name" : "aspects", "namespace" : "com.linkedin.entity", "path" : "/aspects", - "schema" : "com.linkedin.metadata.aspect.AspectWithMetadata", + "schema" : "com.linkedin.metadata.aspect.VersionedAspect", "doc" : "Single unified resource for fetching, updating, searching, & browsing DataHub entities\n\ngenerated from: com.linkedin.metadata.resources.entity.AspectResource", "collection" : { "identifier" : { @@ -27,4 +27,4 @@ "path" : "/aspects/{aspectsId}" } } -} \ No newline at end of file +} diff --git a/gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json index 48b3a22eab629..c3f14eaf8188f 100644 --- a/gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json @@ -2546,7 +2546,7 @@ }, "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.common.GlobalTags", "com.linkedin.common.BrowsePaths" ] }, { "type" : "record", - "name" : "AspectWithMetadata", + "name" : "VersionedAspect", "namespace" : "com.linkedin.metadata.aspect", "fields" : [ { "name" : "aspect", @@ -2560,7 +2560,7 @@ "name" : "aspects", "namespace" : "com.linkedin.entity", "path" : "/aspects", - "schema" : "com.linkedin.metadata.aspect.AspectWithMetadata", + "schema" : "com.linkedin.metadata.aspect.VersionedAspect", "doc" : "Single unified resource for fetching, updating, searching, & browsing DataHub entities\n\ngenerated from: com.linkedin.metadata.resources.entity.AspectResource", "collection" : { "identifier" : { @@ -2586,4 +2586,4 @@ } } } -} \ No newline at end of file +} diff --git a/gms/client/src/main/java/com/linkedin/entity/client/AspectClient.java b/gms/client/src/main/java/com/linkedin/entity/client/AspectClient.java index f95a8ce41006b..a1c66aec96a58 100644 --- a/gms/client/src/main/java/com/linkedin/entity/client/AspectClient.java +++ b/gms/client/src/main/java/com/linkedin/entity/client/AspectClient.java @@ -2,7 +2,7 @@ import com.linkedin.entity.AspectsGetRequestBuilder; import com.linkedin.entity.AspectsRequestBuilders; -import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.r2.RemoteInvocationException; import com.linkedin.restli.client.Client; import javax.annotation.Nonnull; @@ -29,7 +29,7 @@ public AspectClient(@Nonnull final Client restliClient) { * @throws RemoteInvocationException */ @Nonnull - public AspectWithMetadata getAspect(@Nonnull String urn, @Nonnull String aspect, @Nonnull Long version) + public VersionedAspect getAspect(@Nonnull String urn, @Nonnull String aspect, @Nonnull Long version) throws RemoteInvocationException { AspectsGetRequestBuilder requestBuilder = diff --git a/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java index 51ee24ff55e36..207fbc0c63a33 100644 --- a/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java +++ b/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java @@ -1,7 +1,7 @@ package com.linkedin.metadata.resources.entity; import com.linkedin.common.urn.Urn; -import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.restli.RestliUtils; import com.linkedin.parseq.Task; @@ -22,7 +22,7 @@ * Single unified resource for fetching, updating, searching, & browsing DataHub entities */ @RestLiCollection(name = "aspects", namespace = "com.linkedin.entity") -public class AspectResource extends CollectionResourceTaskTemplate { +public class AspectResource extends CollectionResourceTaskTemplate { private final Logger _logger = LoggerFactory.getLogger("EntityResource"); @@ -35,7 +35,7 @@ public class AspectResource extends CollectionResourceTaskTemplate get( + public Task get( @Nonnull String urnStr, @QueryParam("aspect") @Optional @Nullable String aspectName, @QueryParam("version") @Optional @Nullable Long version @@ -43,7 +43,7 @@ public Task get( _logger.info("GET ASPECT urn: {} aspect: {} version: {}", urnStr, aspectName, version); final Urn urn = Urn.createFromString(urnStr); return RestliUtils.toTask(() -> { - final AspectWithMetadata aspect = _entityService.getAspectWithMetadata(urn, aspectName, version); + final VersionedAspect aspect = _entityService.getVersionedAspect(urn, aspectName, version); if (aspect == null) { throw RestliUtils.resourceNotFoundException(); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index 09e7709dcd472..b73d04fe4573d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -8,7 +8,7 @@ import com.linkedin.data.template.UnionTemplate; import com.linkedin.entity.Entity; import com.linkedin.metadata.PegasusUtils; -import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.metadata.dao.exception.ModelConversionException; import com.linkedin.metadata.dao.utils.RecordUtils; import com.linkedin.metadata.event.EntityEventProducer; @@ -104,7 +104,7 @@ public abstract RecordTemplate getAspect( @Nonnull final String aspectName, long version); - public abstract AspectWithMetadata getAspectWithMetadata( + public abstract VersionedAspect getVersionedAspect( @Nonnull final Urn urn, @Nonnull final String aspectName, long version); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index 36546767d4fff..d55af94ce91f4 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -6,7 +6,7 @@ import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; import com.linkedin.metadata.aspect.Aspect; -import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.metadata.dao.utils.RecordUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.ListResult; @@ -105,8 +105,8 @@ public RecordTemplate getAspect(@Nonnull final Urn urn, @Nonnull final String as } @Override - public AspectWithMetadata getAspectWithMetadata(@Nonnull Urn urn, @Nonnull String aspectName, long version) { - AspectWithMetadata result = new AspectWithMetadata(); + public VersionedAspect getVersionedAspect(@Nonnull Urn urn, @Nonnull String aspectName, long version) { + VersionedAspect result = new VersionedAspect(); if (version < 0) { version = _entityDao.getMaxVersion(urn.toString(), aspectName) + version + 1; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java index c2cd3bd46c040..76ecd1e2f1d3d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java @@ -10,7 +10,7 @@ import com.linkedin.identity.CorpUserInfo; import com.linkedin.metadata.PegasusUtils; import com.linkedin.metadata.aspect.Aspect; -import com.linkedin.metadata.aspect.AspectWithMetadata; +import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.metadata.aspect.CorpUserAspect; import com.linkedin.metadata.aspect.CorpUserAspectArray; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; @@ -258,22 +258,22 @@ public void testGetAspectAtVersion() throws Exception { _entityService.updateAspect(entityUrn, aspectName, writeAspect, TEST_AUDIT_STAMP, 1, true); - AspectWithMetadata writtenAspectWithMetadata = new AspectWithMetadata(); - writtenAspectWithMetadata.setAspect(Aspect.create(writeAspect)); - writtenAspectWithMetadata.setVersion(1); + VersionedAspect writtenVersionedAspect = new VersionedAspect(); + writtenVersionedAspect.setAspect(Aspect.create(writeAspect)); + writtenVersionedAspect.setVersion(1); - AspectWithMetadata readAspect1 = _entityService.getAspectWithMetadata(entityUrn, aspectName, 1); - assertTrue(DataTemplateUtil.areEqual(writtenAspectWithMetadata, readAspect1)); + VersionedAspect readAspect1 = _entityService.getVersionedAspect(entityUrn, aspectName, 1); + assertTrue(DataTemplateUtil.areEqual(writtenVersionedAspect, readAspect1)); verify(_mockProducer, times(1)).produceMetadataAuditEvent( Mockito.eq(entityUrn), Mockito.eq(null), Mockito.any()); - AspectWithMetadata readAspect2 = _entityService.getAspectWithMetadata(entityUrn, aspectName, -1); - assertTrue(DataTemplateUtil.areEqual(writtenAspectWithMetadata, readAspect2)); + VersionedAspect readAspect2 = _entityService.getVersionedAspect(entityUrn, aspectName, -1); + assertTrue(DataTemplateUtil.areEqual(writtenVersionedAspect, readAspect2)); - AspectWithMetadata readAspectVersion0 = _entityService.getAspectWithMetadata(entityUrn, aspectName, 0); - assertFalse(DataTemplateUtil.areEqual(writtenAspectWithMetadata, readAspectVersion0)); + VersionedAspect readAspectVersion0 = _entityService.getVersionedAspect(entityUrn, aspectName, 0); + assertFalse(DataTemplateUtil.areEqual(writtenVersionedAspect, readAspectVersion0)); verifyNoMoreInteractions(_mockProducer); } diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/AspectWithMetadata.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/VersionedAspect.pdl similarity index 80% rename from metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/AspectWithMetadata.pdl rename to metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/VersionedAspect.pdl index e75cbc4b94c26..48e16503eb409 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/AspectWithMetadata.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/VersionedAspect.pdl @@ -2,7 +2,7 @@ namespace com.linkedin.metadata.aspect import com.linkedin.common.AuditStamp -record AspectWithMetadata { +record VersionedAspect { aspect: Aspect version: long } From 9468949648f4a5eea22cf41fa4779c5e765706eb Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Wed, 16 Jun 2021 09:38:39 -0700 Subject: [PATCH 8/8] refactoring to be more clear --- .../graphql/resolvers/ResolverUtils.java | 36 ++++++++++++++----- .../resolvers/load/AspectResolver.java | 1 + 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java index 82cfb4fa85cb3..61b1e6a893bdd 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java @@ -66,6 +66,28 @@ public static Map buildFacetFilters(@Nullable List) localContext).getOrDefault(fieldName, null); + if (prefetchedAspect != null) { try { - Object result = Class.forName(prefetchedAspect.getSchema().getUnionMemberKey()) - .cast((ConstructorUtils.getMatchingAccessibleConstructor( - Class.forName(prefetchedAspect.getSchema().getUnionMemberKey()), - new Class[]{DataMap.class}).newInstance(prefetchedAspect.getValue()))); + Object constructedAspect = constructAspectFromDataElement(prefetchedAspect); VersionedAspect resultWithMetadata = new VersionedAspect(); - resultWithMetadata.setAspect( - (com.linkedin.metadata.aspect.Aspect) - com.linkedin.metadata.aspect.Aspect.class.getMethod("create", result.getClass()) - .invoke(com.linkedin.metadata.aspect.Aspect.class, result)); + resultWithMetadata.setAspect(constructAspectUnionInstanceFromAspect(constructedAspect)); resultWithMetadata.setVersion(0); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java index 2be2ee6115532..62e8078e1e46d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/AspectResolver.java @@ -33,6 +33,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) { String urn = ((Entity) environment.getSource()).getUrn(); // first, we try fetching the aspect from the local cache + // we need to convert it into a VersionedAspect so we can make use of existing mappers VersionedAspect aspectFromContext = ResolverUtils.getAspectFromLocalContext(environment); if (aspectFromContext != null) { return CompletableFuture.completedFuture(AspectMapper.map(aspectFromContext));