From da187158783d47e5d5b62e75d24b3185fc3e8935 Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Wed, 3 May 2023 22:26:00 -0400 Subject: [PATCH 01/71] Initial version - Joins Schema --- .../java/com/linkedin/metadata/Constants.java | 6 ++ .../com/linkedin/common/urn/JoinUrn.java | 66 ++++++++++++++++++ .../pegasus/com/linkedin/common/JoinUrn.pdl | 24 +++++++ .../linkedin/join/EditableJoinProperties.pdl | 31 +++++++++ .../pegasus/com/linkedin/join/FieldMap.pdl | 19 +++++ .../com/linkedin/join/JoinFieldMapping.pdl | 18 +++++ .../com/linkedin/join/JoinProperties.pdl | 69 +++++++++++++++++++ .../main/pegasus/com/linkedin/join/Joins.pdl | 15 ++++ .../linkedin/metadata/aspect/JoinAspect.pdl | 30 ++++++++ .../com/linkedin/metadata/key/JoinKey.pdl | 18 +++++ .../metadata/snapshot/JoinSnapshot.pdl | 24 +++++++ .../src/main/resources/entity-registry.yml | 13 ++++ 12 files changed, 333 insertions(+) create mode 100644 li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java create mode 100644 li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/EditableJoinProperties.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index d37eedbe4035e2..8ff8fb80ce8f0c 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -45,6 +45,7 @@ public class Constants { public static final String TAG_ENTITY_NAME = "tag"; public static final String CONTAINER_ENTITY_NAME = "container"; public static final String DOMAIN_ENTITY_NAME = "domain"; + public static final String JOIN_ENTITY_NAME = "join"; public static final String ASSERTION_ENTITY_NAME = "assertion"; public static final String INGESTION_SOURCE_ENTITY_NAME = "dataHubIngestionSource"; public static final String SECRETS_ENTITY_NAME = "dataHubSecret"; @@ -216,6 +217,11 @@ public class Constants { public static final String DOMAINS_ASPECT_NAME = "domains"; public static final String DOMAIN_CREATED_TIME_INDEX_FIELD_NAME = "createdTime"; + // Join + public static final String JOIN_KEY_ASPECT_NAME = "joinKey"; + public static final String JOIN_PROPERTIES_ASPECT_NAME = "joinProperties"; + public static final String EDITABLE_JOIN_PROPERTIES_ASPECT_NAME = "editableJoinProperties"; + // Assertion public static final String ASSERTION_KEY_ASPECT_NAME = "assertionKey"; public static final String ASSERTION_INFO_ASPECT_NAME = "assertionInfo"; diff --git a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java new file mode 100644 index 00000000000000..ed92bca555eeaa --- /dev/null +++ b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java @@ -0,0 +1,66 @@ +package com.linkedin.common.urn; + +import com.linkedin.data.template.Custom; +import com.linkedin.data.template.DirectCoercer; +import com.linkedin.data.template.TemplateOutputCastException; +import java.net.URISyntaxException; + + +public class JoinUrn extends Urn { + public static final String ENTITY_TYPE = "join"; + + private final String _joinId; + + public JoinUrn(String name) { + super(ENTITY_TYPE, TupleKey.create(name)); + this._joinId = name; + } + + public String getName() { + return _joinId; + } + + public static JoinUrn createFromString(String rawUrn) throws URISyntaxException { + return createFromUrn(Urn.createFromString(rawUrn)); + } + + public static JoinUrn createFromUrn(Urn urn) throws URISyntaxException { + if (!"li".equals(urn.getNamespace())) { + throw new URISyntaxException(urn.toString(), "Urn namespace type should be 'li'."); + } else if (!ENTITY_TYPE.equals(urn.getEntityType())) { + throw new URISyntaxException(urn.toString(), "Urn entity type should be 'join'."); + } else { + TupleKey key = urn.getEntityKey(); + if (key.size() != 1) { + throw new URISyntaxException(urn.toString(), "Invalid number of keys."); + } else { + try { + return new JoinUrn((String) key.getAs(0, String.class)); + } catch (Exception var3) { + throw new URISyntaxException(urn.toString(), "Invalid URN Parameter: '" + var3.getMessage()); + } + } + } + } + + public static JoinUrn deserialize(String rawUrn) throws URISyntaxException { + return createFromString(rawUrn); + } + + static { + Custom.initializeCustomClass(JoinUrn.class); + Custom.registerCoercer(new DirectCoercer() { + public Object coerceInput(JoinUrn object) throws ClassCastException { + return object.toString(); + } + + public JoinUrn coerceOutput(Object object) throws TemplateOutputCastException { + try { + return JoinUrn.createFromString((String) object); + } catch (URISyntaxException e) { + throw new TemplateOutputCastException("Invalid URN syntax: " + e.getMessage(), e); + } + } + }, JoinUrn.class); + } +} diff --git a/li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl b/li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl new file mode 100644 index 00000000000000..abfb5ba9a34cb4 --- /dev/null +++ b/li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl @@ -0,0 +1,24 @@ +namespace com.linkedin.common + +/** + * Standardized join identifier. + */ +@java.class = "com.linkedin.common.urn.JoinUrn" +@validate.`com.linkedin.common.validator.TypedUrnValidator` = { + "accessible" : true, + "owningTeam" : "urn:li:internalTeam:datahub", + "entityType" : "join", + "constructable" : true, + "namespace" : "li", + "name" : "Join", + "doc" : "Standardized Join identifier.", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "fields" : [ { + "name" : "joinId", + "doc" : "Join native name e.g. ., /dir/subdir/, or ", + "type" : "string", + "maxLength" : 284 + }], + "maxLength" : 284 +} +typeref JoinUrn = string \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/EditableJoinProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/EditableJoinProperties.pdl new file mode 100644 index 00000000000000..beba0d7d13c6f8 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/join/EditableJoinProperties.pdl @@ -0,0 +1,31 @@ +namespace com.linkedin.join + +import com.linkedin.common.ChangeAuditStamps + + +/** + * EditableJoinProperties stores editable changes made to join properties. This separates changes made from + * ingestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines + */ +@Aspect = { + "name": "editableJoinProperties" +} +record EditableJoinProperties includes ChangeAuditStamps { + /** + * Documentation of the join + */ + @Searchable = { + "fieldType": "TEXT", + "fieldName": "editedDescription", + } + description: optional string + + /** + * Display name of the Join + */ + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "fieldName": "editedName", + } + name: optional string +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl new file mode 100644 index 00000000000000..294dbe70e76849 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl @@ -0,0 +1,19 @@ +namespace com.linkedin.join + +import com.linkedin.dataset.SchemaFieldPath + +/** + * Field Mapping of 1:1 field + */ +record FieldMap { + /** + * All fields from dataset A that are required for the join, maps to bFields 1:1 + */ + afield: SchemaFieldPath + + /** + * All fields from dataset B that are required for the join, maps to aFields 1:1 + */ + bfield: SchemaFieldPath + +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl new file mode 100644 index 00000000000000..f875f36f420375 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl @@ -0,0 +1,18 @@ +namespace com.linkedin.join + +import com.linkedin.dataset.SchemaFieldPath + +/** + * Field Mapping about a join between two datasets + */ +record JoinFieldMapping { + /** + * All fields from dataset A that are required for the join to dataset B + */ + fieldMapping: array[FieldMap] + + /** + * Any transformation logic or notes pertaining to this specific join + */ + details: string +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl new file mode 100644 index 00000000000000..14831e34288699 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl @@ -0,0 +1,69 @@ +namespace com.linkedin.join + +import com.linkedin.common.TimeStamp +import com.linkedin.common.DatasetUrn +import com.linkedin.common.CustomProperties + +/** + * Properties associated with a Join + */ +@Aspect = { + "name": "joinProperties" +} +record JoinProperties includes CustomProperties { + + /** + * Display name of the Join + */ + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } + name: optional string + + /** + * First dataset in the join (no directionality) + */ + @Relationship = { + "name": "joinA", + "entityTypes": [ "dataset" ] + } + datasetA: DatasetUrn + + /** + * Second dataset in the join (no directionality) + */ + @Relationship = { + "name": "joinB", + "entityTypes": [ "dataset" ] + } + datasetB: DatasetUrn + + /** + * Array of JoinFieldMapping + */ + joinFieldMappings: array[JoinFieldMapping] + + /** + * A timestamp documenting when the asset was created in the source Data Platform (not on DataHub) + */ + @Searchable = { + "/time": { + "fieldName": "createdAt", + "fieldType": "DATETIME" + } + } + created: optional TimeStamp + + /** + * A timestamp documenting when the asset was last modified in the source Data Platform (not on DataHub) + */ + @Searchable = { + "/time": { + "fieldName": "lastModifiedAt", + "fieldType": "DATETIME" + } + } + lastModified: optional TimeStamp +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl new file mode 100644 index 00000000000000..2e339d15a2a2db --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl @@ -0,0 +1,15 @@ +namespace com.linkedin.join +import com.linkedin.common.JoinUrn + +/** + * Joins information of an entity. + */ +@Aspect = { + "name": "joins" +} +record Joins { + /** + * Join + */ + joins: array[JoinUrn] +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl new file mode 100644 index 00000000000000..ef8e48ad3252fa --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl @@ -0,0 +1,30 @@ +namespace com.linkedin.metadata.aspect + +import com.linkedin.metadata.key.JoinKey +import com.linkedin.join.JoinProperties +import com.linkedin.join.EditableJoinProperties + +import com.linkedin.common.InstitutionalMemory +import com.linkedin.common.Ownership +import com.linkedin.common.Status +import com.linkedin.container.Container +import com.linkedin.common.GlobalTags +import com.linkedin.common.GlossaryTerms +import com.linkedin.common.BrowsePaths + + +/** + * A union of all supported metadata aspects for a Join + */ +typeref JoinAspect = union[ + JoinKey, + JoinProperties, + EditableJoinProperties, + InstitutionalMemory, + Ownership, + Status, + Container, + GlobalTags, + GlossaryTerms, + BrowsePaths +] diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl new file mode 100644 index 00000000000000..fc209a548aa731 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl @@ -0,0 +1,18 @@ +namespace com.linkedin.metadata.key + +/** + * Key for a Join + */ +@Aspect = { + "name": "joinKey" +} +record JoinKey { + /* + * Unique guid for Join + */ + @Searchable = { + "fieldType": "TEXT", + } + joinId: string + +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl new file mode 100644 index 00000000000000..58bc2c59569ab3 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl @@ -0,0 +1,24 @@ +namespace com.linkedin.metadata.snapshot + +import com.linkedin.common.JoinUrn +import com.linkedin.metadata.aspect.JoinAspect + +/** + * A metadata snapshot for a specific join entity. + */ +@Entity = { + "name": "join", + "keyAspect": "joinKey" +} +record JoinSnapshot { + + /** + * URN for the entity the metadata snapshot is associated with. + */ + urn: JoinUrn + + /** + * The list of metadata aspects associated with the Join. Depending on the use case, this can either be all, or a selection, of supported aspects. + */ + aspects: array[JoinAspect] +} diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 041b79cb6cb20a..537611de2355ad 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -419,6 +419,19 @@ entities: keyAspect: dataHubViewKey aspects: - dataHubViewInfo + - name: join + category: core + doc: Join Dataset Fields + keyAspect: joinKey + aspects: + - joinProperties + - editableJoinProperties + - ownership + - status + - container + - globalTags + - glossaryTerms + - browsePaths - name: query category: core keyAspect: queryKey From bdf2bc6ad83ffcbe084f69d4fa18cee5d23ce9a1 Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Tue, 9 May 2023 15:55:46 -0400 Subject: [PATCH 02/71] Joins slight model changes --- .../src/main/pegasus/com/linkedin/join/JoinProperties.pdl | 2 +- .../main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl | 1 + metadata-models/src/main/resources/entity-registry.yml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl index 14831e34288699..0f7449d582c2b4 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl @@ -20,7 +20,7 @@ record JoinProperties includes CustomProperties { "enableAutocomplete": true, "boostScore": 10.0 } - name: optional string + name: string /** * First dataset in the join (no directionality) diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl index 91993724afbadc..9cf3e325967edb 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl @@ -25,4 +25,5 @@ typeref Snapshot = union[ DataHubPolicySnapshot, SchemaFieldSnapshot, DataHubRetentionSnapshot, + JoinSnapshot, ] diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 537611de2355ad..8a535d5f55af94 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -420,12 +420,12 @@ entities: aspects: - dataHubViewInfo - name: join - category: core doc: Join Dataset Fields keyAspect: joinKey aspects: - joinProperties - editableJoinProperties + - institutionalMemory - ownership - status - container From eeba4cb3097fe02d9b3771d2701b871a7f11c31c Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Wed, 31 May 2023 10:56:09 -0400 Subject: [PATCH 03/71] Added resolvers --- .../datahub/graphql/GmsGraphQLEngine.java | 45 +++ .../graphql/resolvers/EntityTypeMapper.java | 1 + .../common/mappers/UrnToEntityMapper.java | 6 + .../datahub/graphql/types/join/JoinType.java | 242 +++++++++++++ .../types/join/mappers/JoinMapper.java | 128 +++++++ .../join/mappers/JoinUpdateInputMapper.java | 119 +++++++ .../src/main/resources/entity.graphql | 325 ++++++++++++++++++ .../src/graphql/fragments.graphql | 22 ++ datahub-web-react/src/graphql/join.graphql | 42 +++ datahub-web-react/src/graphql/lineage.graphql | 11 + datahub-web-react/src/graphql/search.graphql | 55 +++ .../com/linkedin/common/urn/JoinUrn.java | 8 +- .../pegasus/com/linkedin/join/FieldMap.pdl | 10 + .../com/linkedin/join/JoinFieldMapping.pdl | 6 + .../com/linkedin/join/JoinProperties.pdl | 16 +- .../linkedin/metadata/snapshot/Snapshot.pdl | 2 +- 16 files changed, 1030 insertions(+), 8 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java create mode 100644 datahub-web-react/src/graphql/join.graphql diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 1289059e8ac312..8c655521b438f3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -58,11 +58,13 @@ import com.linkedin.datahub.graphql.generated.GlossaryTermAssociation; import com.linkedin.datahub.graphql.generated.IngestionSource; import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata; +import com.linkedin.datahub.graphql.generated.Join; import com.linkedin.datahub.graphql.generated.LineageRelationship; import com.linkedin.datahub.graphql.generated.ListAccessTokenResult; import com.linkedin.datahub.graphql.generated.ListDomainsResult; import com.linkedin.datahub.graphql.generated.ListGroupsResult; import com.linkedin.datahub.graphql.generated.ListQueriesResult; +import com.linkedin.datahub.graphql.generated.ListJoinResult; import com.linkedin.datahub.graphql.generated.ListTestsResult; import com.linkedin.datahub.graphql.generated.ListViewsResult; import com.linkedin.datahub.graphql.generated.MLFeature; @@ -88,6 +90,7 @@ import com.linkedin.datahub.graphql.generated.Test; import com.linkedin.datahub.graphql.generated.TestResult; import com.linkedin.datahub.graphql.generated.UserUsageCounts; +import com.linkedin.datahub.graphql.resolvers.AuthenticatedResolver; import com.linkedin.datahub.graphql.resolvers.MeResolver; import com.linkedin.datahub.graphql.resolvers.assertion.AssertionRunEventResolver; import com.linkedin.datahub.graphql.resolvers.assertion.DeleteAssertionResolver; @@ -279,6 +282,7 @@ import com.linkedin.datahub.graphql.types.tag.TagType; import com.linkedin.datahub.graphql.types.test.TestType; import com.linkedin.datahub.graphql.types.view.DataHubViewType; +import com.linkedin.datahub.graphql.types.join.JoinType; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.config.DataHubConfiguration; import com.linkedin.metadata.config.IngestionConfiguration; @@ -400,6 +404,7 @@ public class GmsGraphQLEngine { private final DataHubPolicyType dataHubPolicyType; private final DataHubRoleType dataHubRoleType; private final SchemaFieldType schemaFieldType; + private final JoinType joinType; private final DataHubViewType dataHubViewType; private final QueryType queryType; @@ -493,6 +498,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { this.dataHubPolicyType = new DataHubPolicyType(entityClient); this.dataHubRoleType = new DataHubRoleType(entityClient); this.schemaFieldType = new SchemaFieldType(); + this.joinType = new JoinType(entityClient); this.dataHubViewType = new DataHubViewType(entityClient); this.queryType = new QueryType(entityClient); @@ -525,6 +531,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { dataHubPolicyType, dataHubRoleType, schemaFieldType, + joinType, dataHubViewType, queryType ); @@ -586,6 +593,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureTestResultResolvers(builder); configureRoleResolvers(builder); configureSchemaFieldResolvers(builder); + configureJoinResolvers(builder); configureEntityPathResolvers(builder); configureViewResolvers(builder); configureQueryEntityResolvers(builder); @@ -671,6 +679,9 @@ private void configureDataPlatformInstanceResolvers(final RuntimeWiring.Builder private void configureQueryResolvers(final RuntimeWiring.Builder builder) { builder.type("Query", typeWiring -> typeWiring + .dataFetcher("join", new AuthenticatedResolver<>( + new LoadableTypeResolver<>(joinType, + (env) -> env.getArgument(URN_FIELD_NAME)))) .dataFetcher("appConfig", new AppConfigResolver(gitVersion, analyticsService != null, this.ingestionConfiguration, @@ -708,6 +719,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("glossaryTerm", getResolver(glossaryTermType)) .dataFetcher("glossaryNode", getResolver(glossaryNodeType)) .dataFetcher("domain", getResolver((domainType))) + .dataFetcher("join", getResolver(joinType)) .dataFetcher("dataPlatform", getResolver(dataPlatformType)) .dataFetcher("mlFeatureTable", getResolver(mlFeatureTableType)) .dataFetcher("mlFeature", getResolver(mlFeatureType)) @@ -806,6 +818,8 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("updateDataFlow", new MutableTypeResolver<>(dataFlowType)) .dataFetcher("updateCorpUserProperties", new MutableTypeResolver<>(corpUserType)) .dataFetcher("updateCorpGroupProperties", new MutableTypeResolver<>(corpGroupType)) + .dataFetcher("updateJoin", new AuthenticatedResolver<>(new MutableTypeResolver<>(joinType))) + .dataFetcher("createJoin", new MutableTypeResolver<>(joinType)) .dataFetcher("addTag", new AddTagResolver(entityService)) .dataFetcher("addTags", new AddTagsResolver(entityService)) .dataFetcher("batchAddTags", new BatchAddTagsResolver(entityService)) @@ -940,6 +954,12 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder .map(Domain::getUrn) .collect(Collectors.toList()))) ) + .type("ListJoinResult", typeWiring -> typeWiring + .dataFetcher("join", new LoadableTypeBatchResolver<>(joinType, + (env) -> ((ListJoinResult) env.getSource()).getJoin().stream() + .map(Join::getUrn) + .collect(Collectors.toList()))) + ) .type("GetRootGlossaryTermsResult", typeWiring -> typeWiring .dataFetcher("terms", new LoadableTypeBatchResolver<>(glossaryTermType, (env) -> ((GetRootGlossaryTermsResult) env.getSource()).getTerms().stream() @@ -1341,6 +1361,31 @@ private void configureTypeExtensions(final RuntimeWiring.Builder builder) { builder.scalar(GraphQLLong); } + /** + * Configures resolvers responsible for resolving the {@link Join} type. + */ + private void configureJoinResolvers(final RuntimeWiring.Builder builder) { + builder + .type("Join", typeWiring -> typeWiring + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.joinType)) + .dataFetcher("container", + new LoadableTypeResolver<>(containerType, + (env) -> { + final Join join = env.getSource(); + return join.getContainer() != null ? join.getContainer().getUrn() : null; + }) + )) + .type("Owner", typeWiring -> typeWiring + .dataFetcher("owner", new OwnerTypeResolver<>(ownerTypes, + (env) -> ((Owner) env.getSource()).getOwner())) + ) + .type("InstitutionalMemoryMetadata", typeWiring -> typeWiring + .dataFetcher("author", new LoadableTypeResolver<>(corpUserType, + (env) -> ((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn())) + ); + } + /** * Configures resolvers responsible for resolving the {@link com.linkedin.datahub.graphql.generated.DataJob} type. */ diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java index 958f78a98b7fde..5009f5b202c8a7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java @@ -36,6 +36,7 @@ public class EntityTypeMapper { .put(EntityType.NOTEBOOK, "notebook") .put(EntityType.DATA_PLATFORM_INSTANCE, "dataPlatformInstance") .put(EntityType.TEST, "test") + .put(EntityType.JOIN, Constants.JOIN_ENTITY_NAME) .put(EntityType.DATAHUB_VIEW, Constants.DATAHUB_VIEW_ENTITY_NAME) .build(); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java index 23385130fcc7e9..f38e316d480617 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java @@ -20,6 +20,7 @@ import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.GlossaryNode; import com.linkedin.datahub.graphql.generated.GlossaryTerm; +import com.linkedin.datahub.graphql.generated.Join; import com.linkedin.datahub.graphql.generated.MLFeature; import com.linkedin.datahub.graphql.generated.MLFeatureTable; import com.linkedin.datahub.graphql.generated.MLModel; @@ -145,6 +146,11 @@ public Entity apply(Urn input) { ((Domain) partialEntity).setUrn(input.toString()); ((Domain) partialEntity).setType(EntityType.DOMAIN); } + if (input.getEntityType().equals("join")) { + partialEntity = new Join(); + ((Join) partialEntity).setUrn(input.toString()); + ((Join) partialEntity).setType(EntityType.JOIN); + } if (input.getEntityType().equals("assertion")) { partialEntity = new Assertion(); ((Assertion) partialEntity).setUrn(input.toString()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java new file mode 100644 index 00000000000000..1164354966ee85 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java @@ -0,0 +1,242 @@ +package com.linkedin.datahub.graphql.types.join; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.JoinUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.AutoCompleteResults; +import com.linkedin.datahub.graphql.generated.BrowsePath; +import com.linkedin.datahub.graphql.generated.BrowseResults; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.FieldSortInput; +import com.linkedin.datahub.graphql.generated.Join; +import com.linkedin.datahub.graphql.generated.JoinUpdateInput; +import com.linkedin.datahub.graphql.generated.SearchResults; +import com.linkedin.datahub.graphql.generated.Sort; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.types.BrowsableEntityType; +import com.linkedin.datahub.graphql.types.MutableType; +import com.linkedin.datahub.graphql.types.SearchableEntityType; +import com.linkedin.datahub.graphql.types.join.mappers.JoinMapper; +import com.linkedin.datahub.graphql.types.join.mappers.JoinUpdateInputMapper; +import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; +import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; +import com.linkedin.datahub.graphql.types.mappers.BrowseResultMapper; +import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.browse.BrowseResult; +import com.linkedin.metadata.query.AutoCompleteResult; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import graphql.execution.DataFetcherResult; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static com.linkedin.datahub.graphql.Constants.*; +import static com.linkedin.metadata.Constants.*; + + +public class JoinType implements com.linkedin.datahub.graphql.types.EntityType, + BrowsableEntityType, SearchableEntityType, + MutableType { + + + static final Set ASPECTS_TO_RESOLVE = ImmutableSet.of( + JOIN_KEY_ASPECT_NAME, + JOIN_PROPERTIES_ASPECT_NAME, + EDITABLE_JOIN_PROPERTIES_ASPECT_NAME, + INSTITUTIONAL_MEMORY_ASPECT_NAME, + OWNERSHIP_ASPECT_NAME, + STATUS_ASPECT_NAME, + CONTAINER_ASPECT_NAME, + GLOBAL_TAGS_ASPECT_NAME, + GLOSSARY_TERMS_ASPECT_NAME, + BROWSE_PATHS_ASPECT_NAME + ); + + private static final Set FACET_FIELDS = ImmutableSet.of("name"); + private static final String ENTITY_NAME = "join"; + + private final EntityClient _entityClient; + + + public JoinType(final EntityClient entityClient) { + _entityClient = entityClient; + } + + @Override + public Class objectClass() { + return Join.class; + } + + + @Override + public Class inputClass() { + return JoinUpdateInput.class; + } + + @Override + public EntityType type() { + return EntityType.JOIN; + } + + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + + @Override + public List> batchLoad(@Nonnull final List urns, @Nonnull final QueryContext context) + throws Exception { + final List joinUrns = urns.stream() + .map(this::getUrn) + .collect(Collectors.toList()); + + try { + final Map entities = _entityClient.batchGetV2( + JOIN_ENTITY_NAME, + new HashSet<>(joinUrns), + ASPECTS_TO_RESOLVE, + context.getAuthentication()); + + final List gmsResults = new ArrayList<>(); + for (Urn urn : joinUrns) { + gmsResults.add(entities.getOrDefault(urn, null)); + } + return gmsResults.stream() + .map(gmsResult -> + gmsResult == null ? null : DataFetcherResult.newResult() + .data(JoinMapper.map(gmsResult)) + .build() + ) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to load join entity", e); + } + } + + @Nonnull + @Override + public BrowseResults browse(@Nonnull List path, @Nullable List filters, int start, + int count, @Nonnull QueryContext context) throws Exception { + final Map facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS); + final String pathStr = path.size() > 0 ? BROWSE_PATH_DELIMITER + String.join(BROWSE_PATH_DELIMITER, path) : ""; + final BrowseResult result = _entityClient.browse( + "join", + pathStr, + facetFilters, + start, + count, + context.getAuthentication()); + return BrowseResultMapper.map(result); + } + + private Urn getUrn(final String urnStr) { + try { + return Urn.createFromString(urnStr); + } catch (URISyntaxException e) { + throw new RuntimeException(String.format("Failed to convert urn string %s into Urn", urnStr)); + } + } + + + + @Nonnull + @Override + public List browsePaths(@Nonnull String urn, @Nonnull QueryContext context) throws Exception { + final StringArray result = _entityClient.getBrowsePaths(getJoinUrn(urn), context.getAuthentication()); + return BrowsePathsMapper.map(result); + } + + private com.linkedin.common.urn.JoinUrn getJoinUrn(String urnStr) { + try { + return JoinUrn.createFromString(urnStr); + } catch (URISyntaxException e) { + throw new RuntimeException(String.format("Failed to retrieve data product with urn %s, invalid urn", urnStr)); + } + } + + @Override + public SearchResults search(@Nonnull String query, @Nullable List filters, + @Nullable FieldSortInput sort, int start, int count, @Nonnull QueryContext context) throws Exception { + final Map facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS); + String sortField = sort != null ? sort.getField() : null; + SortOrder sortOrder = sort != null ? (sort.getSortOrder().equals(Sort.asc) ? SortOrder.ASCENDING : SortOrder.DESCENDING) : null; + SortCriterion sortCriterion = null; + if (sortField != null && sortOrder != null) { + sortCriterion = + new SortCriterion().setField(sortField).setOrder(sortOrder); + } + final SearchResult searchResult = _entityClient.search(ENTITY_NAME, query, facetFilters, sortCriterion, start, + count, context.getAuthentication()); + return UrnSearchResultsMapper.map(searchResult); + + } + + @Override + public AutoCompleteResults autoComplete(@Nonnull String query, @Nullable String field, + @Nullable List filters, int limit, @Nonnull QueryContext context) throws Exception { + final Map facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS); + final AutoCompleteResult result = _entityClient.autoComplete(ENTITY_NAME, query, facetFilters, limit, context.getAuthentication()); + return AutoCompleteResultsMapper.map(result); + } + + + + + @Override + public Join update(String urn, @Nonnull JoinUpdateInput input, @Nonnull QueryContext context) + throws Exception { + if (isAuthorized(urn, input, context)) { + final CorpuserUrn actor = CorpuserUrn.createFromString(context.getAuthentication().getActor().toUrnStr()); + + // Same routine used by create - hence this check + JoinUrn inputUrn = new JoinUrn(UUID.randomUUID().toString()); + if (urn != null) { + inputUrn = JoinUrn.createFromString(urn); + if ("new".equals(inputUrn.getJoinIdEntity())) { + inputUrn = new JoinUrn(UUID.randomUUID().toString()); + } + } else { + urn = inputUrn.toString(); + } + + final JoinUrn updatedUrn = inputUrn; + + final Collection proposals = JoinUpdateInputMapper.map(input, actor); + proposals.forEach(proposal -> proposal.setEntityUrn(updatedUrn)); + + try { + _entityClient.batchIngestProposals(proposals, context.getAuthentication(), false); + } catch (RemoteInvocationException e) { + throw new RuntimeException(String.format("Failed to write entity with urn %s", urn), e); + } + + return load(urn, context).getData(); + } + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + + private boolean isAuthorized(String urn, JoinUpdateInput input, QueryContext context) { + return true; + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java new file mode 100644 index 00000000000000..e0d4b2a54dbf01 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java @@ -0,0 +1,128 @@ +package com.linkedin.datahub.graphql.types.join.mappers; + +import com.linkedin.common.GlobalTags; +import com.linkedin.common.GlossaryTerms; +import com.linkedin.common.InstitutionalMemory; +import com.linkedin.common.Ownership; +import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.DataMap; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.datahub.graphql.generated.Container; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.Join; +import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; +import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; +import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; +import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.join.EditableJoinProperties; +import com.linkedin.join.JoinProperties; +import com.linkedin.metadata.key.JoinKey; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +import static com.linkedin.metadata.Constants.*; + +/** + * Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema. + * + * To be replaced by auto-generated mappers implementations + */ +public class JoinMapper implements ModelMapper { + + public static final JoinMapper INSTANCE = new JoinMapper(); + + public static Join map(@Nonnull final EntityResponse entityResponse) { + return INSTANCE.apply(entityResponse); + } + + public Join apply(final EntityResponse entityResponse) { + final Join result = new Join(); + final Urn entityUrn = entityResponse.getUrn(); + + result.setUrn(entityUrn.toString()); + result.setType(EntityType.JOIN); + + final EnvelopedAspectMap aspectMap = entityResponse.getAspects(); + MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); + mappingHelper.mapToResult(JOIN_KEY_ASPECT_NAME, this::mapJoinKey); + mappingHelper.mapToResult(JOIN_PROPERTIES_ASPECT_NAME, this::mapProperties); + mappingHelper.mapToResult(EDITABLE_JOIN_PROPERTIES_ASPECT_NAME, this::mapEditableProperties); + mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (join, dataMap) -> + join.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap)))); + mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (join, dataMap) -> + join.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); + mappingHelper.mapToResult(STATUS_ASPECT_NAME, (join, dataMap) -> + join.setStatus(StatusMapper.map(new Status(dataMap)))); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (join, dataMap) -> this.mapGlobalTags(join, dataMap, entityUrn)); + mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (join, dataMap) -> + join.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); + mappingHelper.mapToResult(CONTAINER_ASPECT_NAME, this::mapContainers); + return mappingHelper.getResult(); + } + + private void mapEditableProperties(@Nonnull Join join, @Nonnull DataMap dataMap) { + final EditableJoinProperties editableJoinProperties = new EditableJoinProperties(dataMap); + join.setEditableProperties( + com.linkedin.datahub.graphql.generated.JoinEditableProperties.builder() + .setDescription(editableJoinProperties.getDescription()) + .setName(editableJoinProperties.getName()) + .build() + ); + } + + + private void mapJoinKey(@Nonnull Join join, @Nonnull DataMap datamap) { + JoinKey joinKey = new JoinKey(datamap); + join.setJoinId(joinKey.getJoinId()); + } + + private void mapProperties(@Nonnull Join join, @Nonnull DataMap dataMap) { + final JoinProperties joinProperties = new JoinProperties(dataMap); + join.setProperties( + com.linkedin.datahub.graphql.generated.JoinProperties.builder() + .setName(joinProperties.getName()) + .setDatasetA(joinProperties.getDatasetA().toString()) + .setDatasetB(joinProperties.getDatasetB().toString()) + .setJoinFieldMappings(mapJoinFieldMappings(joinProperties)) + .build()); + } + + private com.linkedin.datahub.graphql.generated.JoinFieldMapping mapJoinFieldMappings(JoinProperties joinProperties) { + return com.linkedin.datahub.graphql.generated.JoinFieldMapping.builder() + .setDetails(joinProperties.getJoinFieldMappings().getDetails()) + .setFieldMapping(joinProperties.getJoinFieldMappings() + .getFieldMapping() + .stream() + .map(this::mapFieldMap) + .collect(Collectors.toList())) + .build(); + } + + private com.linkedin.datahub.graphql.generated.FieldMap mapFieldMap(com.linkedin.join.FieldMap fieldMap) { + return com.linkedin.datahub.graphql.generated.FieldMap.builder() + .setAfield(fieldMap.getAfield()) + .setBfield(fieldMap.getBfield()) + .build(); + } + + private void mapGlobalTags(@Nonnull Join join, @Nonnull DataMap dataMap, @Nonnull final Urn entityUrn) { + com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap), entityUrn); + join.setTags(globalTags); + } + + private void mapContainers(@Nonnull Join join, @Nonnull DataMap dataMap) { + final com.linkedin.container.Container gmsContainer = new com.linkedin.container.Container(dataMap); + join.setContainer(Container + .builder() + .setType(EntityType.CONTAINER) + .setUrn(gmsContainer.getContainer().toString()) + .build()); + } + +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java new file mode 100644 index 00000000000000..11b27849aa32ec --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java @@ -0,0 +1,119 @@ +package com.linkedin.datahub.graphql.types.join.mappers; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.GlobalTags; +import com.linkedin.common.TagAssociationArray; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.generated.JoinFieldMappingInput; +import com.linkedin.datahub.graphql.generated.JoinUpdateInput; +import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryUpdateMapper; +import com.linkedin.datahub.graphql.types.common.mappers.OwnershipUpdateMapper; +import com.linkedin.datahub.graphql.types.common.mappers.util.UpdateMappingHelper; +import com.linkedin.datahub.graphql.types.mappers.InputModelMapper; +import com.linkedin.datahub.graphql.types.tag.mappers.TagAssociationUpdateMapper; +import com.linkedin.join.EditableJoinProperties; +import com.linkedin.join.FieldMap; +import com.linkedin.join.FieldMapArray; +import com.linkedin.join.JoinFieldMapping; +import com.linkedin.mxe.MetadataChangeProposal; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +import static com.linkedin.metadata.Constants.*; + + +public class JoinUpdateInputMapper + implements InputModelMapper, Urn> { + public static final JoinUpdateInputMapper INSTANCE = new JoinUpdateInputMapper(); + + public static Collection map(@Nonnull final JoinUpdateInput joinUpdateInput, + @Nonnull final Urn actor) { + return INSTANCE.apply(joinUpdateInput, actor); + } + + @Override + public Collection apply(JoinUpdateInput input, Urn actor) { + final Collection proposals = new ArrayList<>(8); + final UpdateMappingHelper updateMappingHelper = new UpdateMappingHelper(JOIN_ENTITY_NAME); + final AuditStamp auditStamp = new AuditStamp(); + auditStamp.setActor(actor, SetMode.IGNORE_NULL); + auditStamp.setTime(System.currentTimeMillis()); + + if (input.getProperties() != null) { + com.linkedin.join.JoinProperties joinProperties = new com.linkedin.join.JoinProperties(); + if (input.getProperties().getName() != null) { + joinProperties.setName(input.getProperties().getName()); + } + try { + if (input.getProperties().getDataSetA() != null) { + joinProperties.setDatasetA(DatasetUrn.createFromString(input.getProperties().getDataSetA())); + } + if (input.getProperties().getDatasetB() != null) { + joinProperties.setDatasetB(DatasetUrn.createFromString(input.getProperties().getDatasetB())); + } + } catch (URISyntaxException e) { + e.printStackTrace(); + } + + if (input.getProperties().getJoinFieldmappings() != null) { + JoinFieldMappingInput joinFieldMapping = input.getProperties().getJoinFieldmappings(); + if (joinFieldMapping.getDetails() != null || (joinFieldMapping.getFieldMapping() != null && joinFieldMapping.getFieldMapping().size() > 0)) { + JoinFieldMapping joinFieldMapping1 = new JoinFieldMapping(); + if (joinFieldMapping.getDetails() != null) { + joinFieldMapping1.setDetails(joinFieldMapping.getDetails()); + } + + if (joinFieldMapping.getFieldMapping() != null && joinFieldMapping.getFieldMapping().size() > 0) { + com.linkedin.join.FieldMapArray fieldMapArray = new FieldMapArray(); + joinFieldMapping.getFieldMapping().forEach(fieldMappingInput -> { + FieldMap fieldMap = new FieldMap(); + if (fieldMappingInput.getAfield() != null) { + fieldMap.setAfield(fieldMappingInput.getAfield()); + } + if (fieldMappingInput.getBfield() != null) { + fieldMap.setBfield(fieldMappingInput.getBfield()); + } + fieldMapArray.add(fieldMap); + }); + joinFieldMapping1.setFieldMapping(fieldMapArray); + } + joinProperties.setJoinFieldMappings(joinFieldMapping1); + } + proposals.add(updateMappingHelper.aspectToProposal(joinProperties, JOIN_PROPERTIES_ASPECT_NAME)); + } + + if (input.getOwnership() != null) { + proposals.add(updateMappingHelper.aspectToProposal(OwnershipUpdateMapper.map(input.getOwnership(), actor), + OWNERSHIP_ASPECT_NAME)); + } + + if (input.getInstitutionalMemory() != null) { + proposals.add(updateMappingHelper.aspectToProposal(InstitutionalMemoryUpdateMapper.map(input.getInstitutionalMemory()), + INSTITUTIONAL_MEMORY_ASPECT_NAME)); + } + + if (input.getTags() != null) { + final GlobalTags globalTags = new GlobalTags(); + if (input.getTags() != null) { + globalTags.setTags(new TagAssociationArray( + input.getTags().getTags().stream().map(TagAssociationUpdateMapper::map).collect(Collectors.toList()))); + } + proposals.add(updateMappingHelper.aspectToProposal(globalTags, GLOBAL_TAGS_ASPECT_NAME)); + } + + if (input.getEditableProperties() != null) { + final EditableJoinProperties editableJoinProperties = new EditableJoinProperties(); + editableJoinProperties.setName(input.getEditableProperties().getName()); + editableJoinProperties.setDescription(input.getEditableProperties().getDescription()); + proposals.add(updateMappingHelper.aspectToProposal(editableJoinProperties, EDITABLE_JOIN_PROPERTIES_ASPECT_NAME)); + } + } + return proposals; + } +} + diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index ee5d6e5e278e36..279d015d363c6f 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -69,6 +69,11 @@ type Query { """ tag(urn: String!): Tag + """ + Fetch a Join by primary key (urn) + """ + join(urn: String!): Join + """ Fetch a Glossary Term by primary key (urn) """ @@ -184,6 +189,11 @@ type Query { """ getInviteToken(input: GetInviteTokenInput!): InviteToken + """ + List all Join + """ + listJoin(input: ListJoinInput!): ListJoinResult + """ List all Posts """ @@ -212,6 +222,200 @@ type Query { getQuickFilters(input: GetQuickFiltersInput!): GetQuickFiltersResult } +""" +Input provided when listing existing join +""" +input ListJoinInput { + """ + The starting offset of the result set returned + """ + start: Int + + """ + The maximum number of Roles to be returned in the result set + """ + count: Int + + """ + Optional search query + """ + query: String +} +""" +The result obtained when listing Join +""" +type ListJoinResult { + """ + The starting offset of the result set returned + """ + start: Int! + + """ + The number of Join in the returned result set + """ + count: Int! + + """ + The total number of Join in the result set + """ + total: Int! + + """ + The Join themselves + """ + join: [Join!]! +} + + +""" +A Join is a high-level abstraction that dictates what datasets fields are joined. +""" +type Join implements EntityWithRelationships & Entity & BrowsableEntity { + """ + The primary key of the role + """ + urn: String! + + """ + The standard Entity Type + """ + type: EntityType! + + """ + Unique id for the join + """ + joinId: String! + + """ + An additional set of read only properties + """ + properties: JoinProperties + + """ + An additional set of of read write properties + """ + editableProperties: JoinEditableProperties + + """ + References to internal resources related to the dataset + """ + institutionalMemory: InstitutionalMemory + + """ + Ownership metadata of the dataset + """ + ownership: Ownership + + """ + Status of the Dataset + """ + status: Status + + """ + The parent container in which the entity resides + """ + container: Container + + """ + Recursively get the lineage of containers for this entity + """ + parentContainers: ParentContainersResult + + """ + Tags used for searching dataset + """ + tags: GlobalTags + + """ + The structured glossary terms associated with the dataset + """ + glossaryTerms: GlossaryTerms + + """ + List of relationships between the source Entity and some destination entities with a given types + """ + relationships(input: RelationshipsInput!): EntityRelationshipsResult + """ + Edges extending from this entity grouped by direction in the lineage graph + """ + lineage(input: LineageInput!): EntityLineageResult + + """ + The browse paths corresponding to the dataset. If no Browse Paths have been generated before, this will be null. + """ + browsePaths: [BrowsePath!] +} + +""" +Additional properties about a Join +""" +type JoinEditableProperties { + + """ + Documentation of the Join + """ + description: String + """ + Display name of the Join + """ + name: String +} + +""" +Additional properties about a Join +""" +type JoinProperties { + + """ + The name of the Join used in display + """ + name: String! + """ + The urn of datasetA + """ + datasetA: String! + + """ + The urn of datasetB + """ + datasetB: String! + + """ + The joinFieldMappings + """ + joinFieldMappings: [JoinFieldMapping!] + +} + +""" +Join Field Mapping +""" +type JoinFieldMapping { + """ + The Array of FieldMap + """ + fieldMapping: [FieldMap!] + """ + Any transformation logic or notes pertaining to this specific join + """ + details: String! +} + +""" +Join FieldMap +""" +type FieldMap { + """ + afield + """ + afield: String! + """ + bfield + """ + bfield: String! +} + + """ Root type used for updating DataHub Metadata Coming soon createEntity, addOwner, removeOwner mutations @@ -441,6 +645,31 @@ type Mutation { """ unsetDomain(entityUrn: String!): Boolean + """ + Create a new DataHub View (Saved Filter) + """ + createJoin( + "Input required to create a new Join" + input: JoinUpdateInput!): Join + + """ + Update a Join + """ + updateJoin( + "The urn of the Join to update" + urn: String!, + "Input required to updat an existing DataHub View" + input: JoinUpdateInput!): Join + + """ + Delete a Join + """ + deleteJoin( + "The urn of the Join to delete" + urn: String!): Boolean + + + """ Sets the Deprecation status for a Metadata Entity. Requires the Edit Deprecation status privilege for an entity. """ @@ -677,6 +906,11 @@ enum EntityType { """ DATA_PLATFORM + """ + The Join Entity + """ + JOIN + """ The Dashboard Entity """ @@ -4097,6 +4331,21 @@ input DatasetEditablePropertiesUpdate { description: String! } +""" +Update to writable Dataset fields +""" +input JoinEditablePropertiesUpdate { + """ + Display name of the Join + """ + name: String + + """ + Writable description for Join + """ + description: String! +} + """ Update to writable Chart fields """ @@ -4215,6 +4464,82 @@ input CreateTagInput { description: String } +""" +Input required to create/update a new Join +""" +input JoinUpdateInput { + """ + Details about the Join + """ + properties: JoinPropertiesInput + """ + Update to editable properties + """ + editableProperties: JoinEditablePropertiesUpdate + """ + Update to institutional memory, ie documentation + """ + institutionalMemory: InstitutionalMemoryUpdate + """ + Update to ownership + """ + ownership: OwnershipUpdate + """ + Update to tags + """ + tags: GlobalTagsUpdate +} + +""" +Details about the Join +""" +input JoinPropertiesInput { + """ + Details about the Join + """ + name: String! + """ + Details about the Join + """ + dataSetA: String! + """ + Details about the Join + """ + datasetB: String! + """ + Details about the Join + """ + joinFieldmappings: JoinFieldMappingInput! + +} + +""" +Details about the Join +""" +input JoinFieldMappingInput { + """ + Details about the Join + """ + fieldMapping: [FieldMappingInput] + """ + Details about the Join + """ + details: String +} +""" +Details about the Join +""" +input FieldMappingInput { + """ + Details about the Join + """ + afield: String + """ + Details about the Join + """ + bfield: String +} + """ An update for the ownership information for a Metadata Entity """ diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index 0df857510fdb3b..a0799c473d35d5 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -907,3 +907,25 @@ fragment inputFieldsFields on InputFields { } } } + +fragment joinPropertiesFields on JoinProperties { + name + datasetA + datasetB + joinFieldMappings { + ...joinFieldMappingFields + } +} + +fragment joinFieldMappingFields on JoinFieldMapping { + fieldMapping { + afield + bfield + } + details +} + +fragment joinEditablePropertiesFields on JoinEditableProperties { + description + name +} \ No newline at end of file diff --git a/datahub-web-react/src/graphql/join.graphql b/datahub-web-react/src/graphql/join.graphql new file mode 100644 index 00000000000000..5b36778cb3d179 --- /dev/null +++ b/datahub-web-react/src/graphql/join.graphql @@ -0,0 +1,42 @@ +query getJoin($urn: String!) { + join(urn: $urn) { + urn + type + joinId + properties { + ...joinPropertiesFields + } + editableProperties { + ...joinEditablePropertiesFields + } + institutionalMemory { + ...institutionalMemoryFields + } + ownership { + ...ownershipFields + } + status { + removed + } + container { + ...entityContainer + } + parentContainers { + ...parentContainersFields + } + tags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + outgoing: relationships( + input: { types: ["JoinA", "JoinB"], direction: OUTGOING, start: 0, count: 100 } + ) { + ...fullRelationshipResults + } + browsePaths { + path + } + } +} diff --git a/datahub-web-react/src/graphql/lineage.graphql b/datahub-web-react/src/graphql/lineage.graphql index fb8d6daa028243..5416cd77fdf515 100644 --- a/datahub-web-react/src/graphql/lineage.graphql +++ b/datahub-web-react/src/graphql/lineage.graphql @@ -224,6 +224,17 @@ fragment lineageNodeProperties on EntityWithRelationships { ... on MLPrimaryKey { ...nonRecursiveMLPrimaryKey } + ... on Join { + urn + type + joinId + properties { + ...joinPropertiesFields + } + editableProperties { + ...joinEditablePropertiesFields + } + } } fragment lineageFields on EntityWithRelationships { diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index 3f944179859615..8cc09a1ffc5365 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -631,6 +631,17 @@ fragment searchResultFields on Entity { ...parentContainersFields } } + ... on Join { + urn + type + joinId + properties { + ...joinPropertiesFields + } + editableProperties { + ...joinEditablePropertiesFields + } + } ... on MLFeatureTable { name description @@ -707,6 +718,17 @@ fragment searchResultFields on Entity { ... on DataPlatform { ...nonConflictingPlatformFields } + ... on Join { + urn + type + joinId + properties { + ...joinPropertiesFields + } + editableProperties { + ...joinEditablePropertiesFields + } + } } fragment facetFields on FacetMetadata { @@ -802,6 +824,28 @@ fragment schemaFieldEntityFields on SchemaFieldEntity { } } +fragment searchResultsWithRelationships on SearchResults { + start + count + total + searchResults { + entity { + ...searchResultFieldsWithRelationships + } + matchedFields { + name + value + } + insights { + text + icon + } + } + facets { + ...facetFields + } +} + fragment searchAcrossRelationshipResults on SearchAcrossLineageResults { start count @@ -850,12 +894,23 @@ query getSearchResults($input: SearchInput!) { } } +query getSearchResultsWithRelationships($input: SearchInput!) { + search(input: $input) { + ...searchResultsWithRelationships + } +} + query getSearchResultsForMultiple($input: SearchAcrossEntitiesInput!) { searchAcrossEntities(input: $input) { ...searchResults } } +query getSearchResultsForMultipleWithRelationships($input: SearchAcrossEntitiesInput!) { + searchAcrossEntities(input: $input) { + ...searchResultsWithRelationships + } +} query searchAcrossLineage($input: SearchAcrossLineageInput!, $includeAssertions: Boolean = false) { searchAcrossLineage(input: $input) { ...searchAcrossRelationshipResults diff --git a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java index ed92bca555eeaa..6b7833ca99032b 100644 --- a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java +++ b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java @@ -11,12 +11,12 @@ public class JoinUrn extends Urn { private final String _joinId; - public JoinUrn(String name) { - super(ENTITY_TYPE, TupleKey.create(name)); - this._joinId = name; + public JoinUrn(String joinId) { + super(ENTITY_TYPE, TupleKey.create(joinId)); + this._joinId = joinId; } - public String getName() { + public String getJoinIdEntity() { return _joinId; } diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl index 294dbe70e76849..c979aade06e956 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl @@ -9,11 +9,21 @@ record FieldMap { /** * All fields from dataset A that are required for the join, maps to bFields 1:1 */ + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } afield: SchemaFieldPath /** * All fields from dataset B that are required for the join, maps to aFields 1:1 */ + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } bfield: SchemaFieldPath } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl index f875f36f420375..01517dcebebc25 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl @@ -6,6 +6,7 @@ import com.linkedin.dataset.SchemaFieldPath * Field Mapping about a join between two datasets */ record JoinFieldMapping { + /** * All fields from dataset A that are required for the join to dataset B */ @@ -14,5 +15,10 @@ record JoinFieldMapping { /** * Any transformation logic or notes pertaining to this specific join */ + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } details: string } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl index 0f7449d582c2b4..41bd7c843ce2e6 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl @@ -13,7 +13,7 @@ import com.linkedin.common.CustomProperties record JoinProperties includes CustomProperties { /** - * Display name of the Join + * Name of the Join */ @Searchable = { "fieldType": "TEXT_PARTIAL", @@ -29,6 +29,11 @@ record JoinProperties includes CustomProperties { "name": "joinA", "entityTypes": [ "dataset" ] } + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } datasetA: DatasetUrn /** @@ -38,12 +43,17 @@ record JoinProperties includes CustomProperties { "name": "joinB", "entityTypes": [ "dataset" ] } + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } datasetB: DatasetUrn /** - * Array of JoinFieldMapping + * JoinFieldMapping (in future we can make it an array) */ - joinFieldMappings: array[JoinFieldMapping] + joinFieldMappings: JoinFieldMapping /** * A timestamp documenting when the asset was created in the source Data Platform (not on DataHub) diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl index 9cf3e325967edb..374bdb6cd99069 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl @@ -25,5 +25,5 @@ typeref Snapshot = union[ DataHubPolicySnapshot, SchemaFieldSnapshot, DataHubRetentionSnapshot, - JoinSnapshot, + JoinSnapshot, ] From 89e878292ea5086910c29253fb7d82057b301d0e Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Wed, 31 May 2023 11:38:39 -0400 Subject: [PATCH 04/71] Graphql change and extraneous code removed --- .../datahub/graphql/types/join/JoinType.java | 27 +- .../src/main/resources/entity.graphql | 3 +- .../com.linkedin.entity.aspects.snapshot.json | 5 +- ...com.linkedin.entity.entities.snapshot.json | 247 +++++++++++++++++- .../com.linkedin.entity.runs.snapshot.json | 5 +- ...m.linkedin.platform.platform.snapshot.json | 243 ++++++++++++++++- 6 files changed, 496 insertions(+), 34 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java index 1164354966ee85..1659630f31a285 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java @@ -13,11 +13,9 @@ import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; -import com.linkedin.datahub.graphql.generated.FieldSortInput; import com.linkedin.datahub.graphql.generated.Join; import com.linkedin.datahub.graphql.generated.JoinUpdateInput; import com.linkedin.datahub.graphql.generated.SearchResults; -import com.linkedin.datahub.graphql.generated.Sort; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.BrowsableEntityType; import com.linkedin.datahub.graphql.types.MutableType; @@ -32,8 +30,8 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.filter.SortCriterion; -import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.search.SearchResult; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; @@ -177,32 +175,21 @@ private com.linkedin.common.urn.JoinUrn getJoinUrn(String urnStr) { @Override public SearchResults search(@Nonnull String query, @Nullable List filters, - @Nullable FieldSortInput sort, int start, int count, @Nonnull QueryContext context) throws Exception { + int start, int count, @Nonnull QueryContext context) throws Exception { final Map facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS); - String sortField = sort != null ? sort.getField() : null; - SortOrder sortOrder = sort != null ? (sort.getSortOrder().equals(Sort.asc) ? SortOrder.ASCENDING : SortOrder.DESCENDING) : null; - SortCriterion sortCriterion = null; - if (sortField != null && sortOrder != null) { - sortCriterion = - new SortCriterion().setField(sortField).setOrder(sortOrder); - } - final SearchResult searchResult = _entityClient.search(ENTITY_NAME, query, facetFilters, sortCriterion, start, - count, context.getAuthentication()); + final SearchResult searchResult = _entityClient.search(ENTITY_NAME, query, facetFilters, start, + count, context.getAuthentication(), new SearchFlags().setFulltext(true)); return UrnSearchResultsMapper.map(searchResult); } @Override public AutoCompleteResults autoComplete(@Nonnull String query, @Nullable String field, - @Nullable List filters, int limit, @Nonnull QueryContext context) throws Exception { - final Map facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS); - final AutoCompleteResult result = _entityClient.autoComplete(ENTITY_NAME, query, facetFilters, limit, context.getAuthentication()); + @Nullable Filter filters, int limit, @Nonnull QueryContext context) throws Exception { + final AutoCompleteResult result = _entityClient.autoComplete(ENTITY_NAME, query, filters, limit, context.getAuthentication()); return AutoCompleteResultsMapper.map(result); } - - - @Override public Join update(String urn, @Nonnull JoinUpdateInput input, @Nonnull QueryContext context) throws Exception { diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 279d015d363c6f..7afb3b9c2283a5 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -383,7 +383,7 @@ type JoinProperties { """ The joinFieldMappings """ - joinFieldMappings: [JoinFieldMapping!] + joinFieldMappings: JoinFieldMapping! } @@ -415,7 +415,6 @@ type FieldMap { bfield: String! } - """ Root type used for updating DataHub Metadata Coming soon createEntity, addOwner, removeOwner mutations diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json index 67de18770786bf..f6f84ebe940354 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json @@ -1777,7 +1777,7 @@ } ] } }, - "doc" : "Fine-grained column-level lineages", + "doc" : "Fine-grained column-level lineages\nNot currently supported in the UI\nUse UpstreamLineage aspect for datasets to express Column Level Lineage for the UI", "optional" : true } ], "Aspect" : { @@ -2743,7 +2743,8 @@ "Searchable" : { "boostScore" : 5.0, "fieldName" : "fieldPaths", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "queryByDefault" : "true" } }, { "name" : "jsonPath", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 4918985a200e1d..e0f51a5bf1de38 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -1006,6 +1006,34 @@ "name" : "institutionalMemory" } }, "com.linkedin.common.InstitutionalMemoryMetadata", { + "type" : "typeref", + "name" : "JoinUrn", + "namespace" : "com.linkedin.common", + "doc" : "Standardized join identifier.", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.JoinUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Standardized Join identifier.", + "entityType" : "join", + "fields" : [ { + "doc" : "Join native name e.g. .
, /dir/subdir/, or ", + "maxLength" : 284, + "name" : "joinId", + "type" : "string" + } ], + "maxLength" : 284, + "name" : "Join", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:datahub" + } + } + }, { "type" : "enum", "name" : "MLFeatureDataType", "namespace" : "com.linkedin.common", @@ -1257,6 +1285,30 @@ "optional" : true } ] }, "com.linkedin.common.fieldtransformer.TransformationType", "com.linkedin.common.fieldtransformer.UDFTransformer", { + "type" : "record", + "name" : "Container", + "namespace" : "com.linkedin.container", + "doc" : "Link from an asset to its parent container", + "fields" : [ { + "name" : "container", + "type" : "com.linkedin.common.Urn", + "doc" : "The parent container of an asset", + "Relationship" : { + "entityTypes" : [ "container" ], + "name" : "IsPartOf" + }, + "Searchable" : { + "addToFilters" : true, + "fieldName" : "container", + "fieldType" : "URN", + "filterNameOverride" : "Container", + "hasValuesFieldName" : "hasContainer" + } + } ], + "Aspect" : { + "name" : "container" + } + }, { "type" : "record", "name" : "DashboardInfo", "namespace" : "com.linkedin.dashboard", @@ -1806,7 +1858,7 @@ } ] } }, - "doc" : "Fine-grained column-level lineages", + "doc" : "Fine-grained column-level lineages\nNot currently supported in the UI\nUse UpstreamLineage aspect for datasets to express Column Level Lineage for the UI", "optional" : true } ], "Aspect" : { @@ -3103,7 +3155,8 @@ "Searchable" : { "boostScore" : 5.0, "fieldName" : "fieldPaths", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "queryByDefault" : "true" } }, { "name" : "jsonPath", @@ -5401,10 +5454,194 @@ "keyAspect" : "dataHubRetentionKey", "name" : "dataHubRetention" } + }, { + "type" : "record", + "name" : "JoinSnapshot", + "doc" : "A metadata snapshot for a specific join entity.", + "fields" : [ { + "name" : "urn", + "type" : "com.linkedin.common.JoinUrn", + "doc" : "URN for the entity the metadata snapshot is associated with." + }, { + "name" : "aspects", + "type" : { + "type" : "array", + "items" : { + "type" : "typeref", + "name" : "JoinAspect", + "namespace" : "com.linkedin.metadata.aspect", + "doc" : "A union of all supported metadata aspects for a Join", + "ref" : [ { + "type" : "record", + "name" : "JoinKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a Join", + "fields" : [ { + "name" : "joinId", + "type" : "string", + "Searchable" : { + "fieldType" : "TEXT" + } + } ], + "Aspect" : { + "name" : "joinKey" + } + }, { + "type" : "record", + "name" : "JoinProperties", + "namespace" : "com.linkedin.join", + "doc" : "Properties associated with a Join", + "include" : [ "com.linkedin.common.CustomProperties" ], + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the Join", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "datasetA", + "type" : "com.linkedin.common.DatasetUrn", + "doc" : "First dataset in the join (no directionality)", + "Relationship" : { + "entityTypes" : [ "dataset" ], + "name" : "joinA" + }, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "datasetB", + "type" : "com.linkedin.common.DatasetUrn", + "doc" : "Second dataset in the join (no directionality)", + "Relationship" : { + "entityTypes" : [ "dataset" ], + "name" : "joinB" + }, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "joinFieldMappings", + "type" : { + "type" : "record", + "name" : "JoinFieldMapping", + "doc" : "Field Mapping about a join between two datasets", + "fields" : [ { + "name" : "fieldMapping", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "FieldMap", + "doc" : "Field Mapping of 1:1 field", + "fields" : [ { + "name" : "afield", + "type" : "com.linkedin.dataset.SchemaFieldPath", + "doc" : "All fields from dataset A that are required for the join, maps to bFields 1:1", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "bfield", + "type" : "com.linkedin.dataset.SchemaFieldPath", + "doc" : "All fields from dataset B that are required for the join, maps to aFields 1:1", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + } ] + } + }, + "doc" : "All fields from dataset A that are required for the join to dataset B" + }, { + "name" : "details", + "type" : "string", + "doc" : " Any transformation logic or notes pertaining to this specific join", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + } ] + }, + "doc" : "JoinFieldMapping (in future we can make it an array)" + }, { + "name" : "created", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "A timestamp documenting when the asset was created in the source Data Platform (not on DataHub)", + "optional" : true, + "Searchable" : { + "/time" : { + "fieldName" : "createdAt", + "fieldType" : "DATETIME" + } + } + }, { + "name" : "lastModified", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "A timestamp documenting when the asset was last modified in the source Data Platform (not on DataHub)", + "optional" : true, + "Searchable" : { + "/time" : { + "fieldName" : "lastModifiedAt", + "fieldType" : "DATETIME" + } + } + } ], + "Aspect" : { + "name" : "joinProperties" + } + }, { + "type" : "record", + "name" : "EditableJoinProperties", + "namespace" : "com.linkedin.join", + "doc" : "EditableJoinProperties stores editable changes made to join properties. This separates changes made from\ningestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines", + "include" : [ "com.linkedin.common.ChangeAuditStamps" ], + "fields" : [ { + "name" : "description", + "type" : "string", + "doc" : "Documentation of the join", + "optional" : true, + "Searchable" : { + "fieldName" : "editedDescription", + "fieldType" : "TEXT" + } + }, { + "name" : "name", + "type" : "string", + "doc" : "Display name of the Join", + "optional" : true, + "Searchable" : { + "fieldName" : "editedName", + "fieldType" : "TEXT_PARTIAL" + } + } ], + "Aspect" : { + "name" : "editableJoinProperties" + } + }, "com.linkedin.common.InstitutionalMemory", "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.container.Container", "com.linkedin.common.GlobalTags", "com.linkedin.common.GlossaryTerms", "com.linkedin.common.BrowsePaths" ] + } + }, + "doc" : "The list of metadata aspects associated with the Join. Depending on the use case, this can either be all, or a selection, of supported aspects." + } ], + "Entity" : { + "keyAspect" : "joinKey", + "name" : "join" + } } ] } } ] - }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryRelatedTerms", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserStatus", "com.linkedin.identity.GroupMembership", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataHubPolicyAspect", "com.linkedin.metadata.aspect.DataHubRetentionAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.SchemaFieldAspect", "com.linkedin.metadata.aspect.TagAspect", { + }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryRelatedTerms", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserStatus", "com.linkedin.identity.GroupMembership", "com.linkedin.join.EditableJoinProperties", "com.linkedin.join.FieldMap", "com.linkedin.join.JoinFieldMapping", "com.linkedin.join.JoinProperties", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataHubPolicyAspect", "com.linkedin.metadata.aspect.DataHubRetentionAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.JoinAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.SchemaFieldAspect", "com.linkedin.metadata.aspect.TagAspect", { "type" : "record", "name" : "BrowseResult", "namespace" : "com.linkedin.metadata.browse", @@ -5488,7 +5725,7 @@ "type" : "int", "doc" : "The total number of elements (entities + groups) directly under queried path" } ] - }, "com.linkedin.metadata.browse.BrowseResultEntity", "com.linkedin.metadata.browse.BrowseResultGroup", "com.linkedin.metadata.browse.BrowseResultMetadata", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataHubPolicyKey", "com.linkedin.metadata.key.DataHubRetentionKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.SchemaFieldKey", "com.linkedin.metadata.key.TagKey", { + }, "com.linkedin.metadata.browse.BrowseResultEntity", "com.linkedin.metadata.browse.BrowseResultGroup", "com.linkedin.metadata.browse.BrowseResultMetadata", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataHubPolicyKey", "com.linkedin.metadata.key.DataHubRetentionKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.JoinKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.SchemaFieldKey", "com.linkedin.metadata.key.TagKey", { "type" : "record", "name" : "AnyResult", "namespace" : "com.linkedin.metadata.query", @@ -6108,7 +6345,7 @@ "type" : "int", "doc" : "The total number of entities directly under searched path" } ] - }, "com.linkedin.metadata.search.SearchResultMetadata", "com.linkedin.metadata.snapshot.ChartSnapshot", "com.linkedin.metadata.snapshot.CorpGroupSnapshot", "com.linkedin.metadata.snapshot.CorpUserSnapshot", "com.linkedin.metadata.snapshot.DashboardSnapshot", "com.linkedin.metadata.snapshot.DataFlowSnapshot", "com.linkedin.metadata.snapshot.DataHubPolicySnapshot", "com.linkedin.metadata.snapshot.DataHubRetentionSnapshot", "com.linkedin.metadata.snapshot.DataJobSnapshot", "com.linkedin.metadata.snapshot.DataPlatformSnapshot", "com.linkedin.metadata.snapshot.DataProcessSnapshot", "com.linkedin.metadata.snapshot.DatasetSnapshot", "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot", "com.linkedin.metadata.snapshot.GlossaryTermSnapshot", "com.linkedin.metadata.snapshot.MLFeatureSnapshot", "com.linkedin.metadata.snapshot.MLFeatureTableSnapshot", "com.linkedin.metadata.snapshot.MLModelDeploymentSnapshot", "com.linkedin.metadata.snapshot.MLModelGroupSnapshot", "com.linkedin.metadata.snapshot.MLModelSnapshot", "com.linkedin.metadata.snapshot.MLPrimaryKeySnapshot", "com.linkedin.metadata.snapshot.SchemaFieldSnapshot", "com.linkedin.metadata.snapshot.Snapshot", "com.linkedin.metadata.snapshot.TagSnapshot", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.DeploymentStatus", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLFeatureTableProperties", "com.linkedin.ml.metadata.MLHyperParam", "com.linkedin.ml.metadata.MLMetric", "com.linkedin.ml.metadata.MLModelDeploymentProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelGroupProperties", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.MLPrimaryKeyProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", { + }, "com.linkedin.metadata.search.SearchResultMetadata", "com.linkedin.metadata.snapshot.ChartSnapshot", "com.linkedin.metadata.snapshot.CorpGroupSnapshot", "com.linkedin.metadata.snapshot.CorpUserSnapshot", "com.linkedin.metadata.snapshot.DashboardSnapshot", "com.linkedin.metadata.snapshot.DataFlowSnapshot", "com.linkedin.metadata.snapshot.DataHubPolicySnapshot", "com.linkedin.metadata.snapshot.DataHubRetentionSnapshot", "com.linkedin.metadata.snapshot.DataJobSnapshot", "com.linkedin.metadata.snapshot.DataPlatformSnapshot", "com.linkedin.metadata.snapshot.DataProcessSnapshot", "com.linkedin.metadata.snapshot.DatasetSnapshot", "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot", "com.linkedin.metadata.snapshot.GlossaryTermSnapshot", "com.linkedin.metadata.snapshot.JoinSnapshot", "com.linkedin.metadata.snapshot.MLFeatureSnapshot", "com.linkedin.metadata.snapshot.MLFeatureTableSnapshot", "com.linkedin.metadata.snapshot.MLModelDeploymentSnapshot", "com.linkedin.metadata.snapshot.MLModelGroupSnapshot", "com.linkedin.metadata.snapshot.MLModelSnapshot", "com.linkedin.metadata.snapshot.MLPrimaryKeySnapshot", "com.linkedin.metadata.snapshot.SchemaFieldSnapshot", "com.linkedin.metadata.snapshot.Snapshot", "com.linkedin.metadata.snapshot.TagSnapshot", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.DeploymentStatus", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLFeatureTableProperties", "com.linkedin.ml.metadata.MLHyperParam", "com.linkedin.ml.metadata.MLMetric", "com.linkedin.ml.metadata.MLModelDeploymentProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelGroupProperties", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.MLPrimaryKeyProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", { "type" : "record", "name" : "SystemMetadata", "namespace" : "com.linkedin.mxe", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json index 252ddde6081f8f..915b7cd64bfde8 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json @@ -1530,7 +1530,7 @@ } ] } }, - "doc" : "Fine-grained column-level lineages", + "doc" : "Fine-grained column-level lineages\nNot currently supported in the UI\nUse UpstreamLineage aspect for datasets to express Column Level Lineage for the UI", "optional" : true } ], "Aspect" : { @@ -2488,7 +2488,8 @@ "Searchable" : { "boostScore" : 5.0, "fieldName" : "fieldPaths", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "queryByDefault" : "true" } }, { "name" : "jsonPath", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json index c2163171311126..c16ea02d007c5e 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json @@ -1006,6 +1006,34 @@ "name" : "institutionalMemory" } }, "com.linkedin.common.InstitutionalMemoryMetadata", { + "type" : "typeref", + "name" : "JoinUrn", + "namespace" : "com.linkedin.common", + "doc" : "Standardized join identifier.", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.JoinUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Standardized Join identifier.", + "entityType" : "join", + "fields" : [ { + "doc" : "Join native name e.g. .
, /dir/subdir/, or ", + "maxLength" : 284, + "name" : "joinId", + "type" : "string" + } ], + "maxLength" : 284, + "name" : "Join", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:datahub" + } + } + }, { "type" : "enum", "name" : "MLFeatureDataType", "namespace" : "com.linkedin.common", @@ -1257,6 +1285,30 @@ "optional" : true } ] }, "com.linkedin.common.fieldtransformer.TransformationType", "com.linkedin.common.fieldtransformer.UDFTransformer", { + "type" : "record", + "name" : "Container", + "namespace" : "com.linkedin.container", + "doc" : "Link from an asset to its parent container", + "fields" : [ { + "name" : "container", + "type" : "com.linkedin.common.Urn", + "doc" : "The parent container of an asset", + "Relationship" : { + "entityTypes" : [ "container" ], + "name" : "IsPartOf" + }, + "Searchable" : { + "addToFilters" : true, + "fieldName" : "container", + "fieldType" : "URN", + "filterNameOverride" : "Container", + "hasValuesFieldName" : "hasContainer" + } + } ], + "Aspect" : { + "name" : "container" + } + }, { "type" : "record", "name" : "DashboardInfo", "namespace" : "com.linkedin.dashboard", @@ -1806,7 +1858,7 @@ } ] } }, - "doc" : "Fine-grained column-level lineages", + "doc" : "Fine-grained column-level lineages\nNot currently supported in the UI\nUse UpstreamLineage aspect for datasets to express Column Level Lineage for the UI", "optional" : true } ], "Aspect" : { @@ -3097,7 +3149,8 @@ "Searchable" : { "boostScore" : 5.0, "fieldName" : "fieldPaths", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "queryByDefault" : "true" } }, { "name" : "jsonPath", @@ -5395,10 +5448,194 @@ "keyAspect" : "dataHubRetentionKey", "name" : "dataHubRetention" } + }, { + "type" : "record", + "name" : "JoinSnapshot", + "doc" : "A metadata snapshot for a specific join entity.", + "fields" : [ { + "name" : "urn", + "type" : "com.linkedin.common.JoinUrn", + "doc" : "URN for the entity the metadata snapshot is associated with." + }, { + "name" : "aspects", + "type" : { + "type" : "array", + "items" : { + "type" : "typeref", + "name" : "JoinAspect", + "namespace" : "com.linkedin.metadata.aspect", + "doc" : "A union of all supported metadata aspects for a Join", + "ref" : [ { + "type" : "record", + "name" : "JoinKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a Join", + "fields" : [ { + "name" : "joinId", + "type" : "string", + "Searchable" : { + "fieldType" : "TEXT" + } + } ], + "Aspect" : { + "name" : "joinKey" + } + }, { + "type" : "record", + "name" : "JoinProperties", + "namespace" : "com.linkedin.join", + "doc" : "Properties associated with a Join", + "include" : [ "com.linkedin.common.CustomProperties" ], + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the Join", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "datasetA", + "type" : "com.linkedin.common.DatasetUrn", + "doc" : "First dataset in the join (no directionality)", + "Relationship" : { + "entityTypes" : [ "dataset" ], + "name" : "joinA" + }, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "datasetB", + "type" : "com.linkedin.common.DatasetUrn", + "doc" : "Second dataset in the join (no directionality)", + "Relationship" : { + "entityTypes" : [ "dataset" ], + "name" : "joinB" + }, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "joinFieldMappings", + "type" : { + "type" : "record", + "name" : "JoinFieldMapping", + "doc" : "Field Mapping about a join between two datasets", + "fields" : [ { + "name" : "fieldMapping", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "FieldMap", + "doc" : "Field Mapping of 1:1 field", + "fields" : [ { + "name" : "afield", + "type" : "com.linkedin.dataset.SchemaFieldPath", + "doc" : "All fields from dataset A that are required for the join, maps to bFields 1:1", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "bfield", + "type" : "com.linkedin.dataset.SchemaFieldPath", + "doc" : "All fields from dataset B that are required for the join, maps to aFields 1:1", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + } ] + } + }, + "doc" : "All fields from dataset A that are required for the join to dataset B" + }, { + "name" : "details", + "type" : "string", + "doc" : " Any transformation logic or notes pertaining to this specific join", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + } ] + }, + "doc" : "JoinFieldMapping (in future we can make it an array)" + }, { + "name" : "created", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "A timestamp documenting when the asset was created in the source Data Platform (not on DataHub)", + "optional" : true, + "Searchable" : { + "/time" : { + "fieldName" : "createdAt", + "fieldType" : "DATETIME" + } + } + }, { + "name" : "lastModified", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "A timestamp documenting when the asset was last modified in the source Data Platform (not on DataHub)", + "optional" : true, + "Searchable" : { + "/time" : { + "fieldName" : "lastModifiedAt", + "fieldType" : "DATETIME" + } + } + } ], + "Aspect" : { + "name" : "joinProperties" + } + }, { + "type" : "record", + "name" : "EditableJoinProperties", + "namespace" : "com.linkedin.join", + "doc" : "EditableJoinProperties stores editable changes made to join properties. This separates changes made from\ningestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines", + "include" : [ "com.linkedin.common.ChangeAuditStamps" ], + "fields" : [ { + "name" : "description", + "type" : "string", + "doc" : "Documentation of the join", + "optional" : true, + "Searchable" : { + "fieldName" : "editedDescription", + "fieldType" : "TEXT" + } + }, { + "name" : "name", + "type" : "string", + "doc" : "Display name of the Join", + "optional" : true, + "Searchable" : { + "fieldName" : "editedName", + "fieldType" : "TEXT_PARTIAL" + } + } ], + "Aspect" : { + "name" : "editableJoinProperties" + } + }, "com.linkedin.common.InstitutionalMemory", "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.container.Container", "com.linkedin.common.GlobalTags", "com.linkedin.common.GlossaryTerms", "com.linkedin.common.BrowsePaths" ] + } + }, + "doc" : "The list of metadata aspects associated with the Join. Depending on the use case, this can either be all, or a selection, of supported aspects." + } ], + "Entity" : { + "keyAspect" : "joinKey", + "name" : "join" + } } ] } } ] - }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryRelatedTerms", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserStatus", "com.linkedin.identity.GroupMembership", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataHubPolicyAspect", "com.linkedin.metadata.aspect.DataHubRetentionAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.SchemaFieldAspect", "com.linkedin.metadata.aspect.TagAspect", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataHubPolicyKey", "com.linkedin.metadata.key.DataHubRetentionKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.SchemaFieldKey", "com.linkedin.metadata.key.TagKey", "com.linkedin.metadata.snapshot.ChartSnapshot", "com.linkedin.metadata.snapshot.CorpGroupSnapshot", "com.linkedin.metadata.snapshot.CorpUserSnapshot", "com.linkedin.metadata.snapshot.DashboardSnapshot", "com.linkedin.metadata.snapshot.DataFlowSnapshot", "com.linkedin.metadata.snapshot.DataHubPolicySnapshot", "com.linkedin.metadata.snapshot.DataHubRetentionSnapshot", "com.linkedin.metadata.snapshot.DataJobSnapshot", "com.linkedin.metadata.snapshot.DataPlatformSnapshot", "com.linkedin.metadata.snapshot.DataProcessSnapshot", "com.linkedin.metadata.snapshot.DatasetSnapshot", "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot", "com.linkedin.metadata.snapshot.GlossaryTermSnapshot", "com.linkedin.metadata.snapshot.MLFeatureSnapshot", "com.linkedin.metadata.snapshot.MLFeatureTableSnapshot", "com.linkedin.metadata.snapshot.MLModelDeploymentSnapshot", "com.linkedin.metadata.snapshot.MLModelGroupSnapshot", "com.linkedin.metadata.snapshot.MLModelSnapshot", "com.linkedin.metadata.snapshot.MLPrimaryKeySnapshot", "com.linkedin.metadata.snapshot.SchemaFieldSnapshot", "com.linkedin.metadata.snapshot.Snapshot", "com.linkedin.metadata.snapshot.TagSnapshot", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.DeploymentStatus", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLFeatureTableProperties", "com.linkedin.ml.metadata.MLHyperParam", "com.linkedin.ml.metadata.MLMetric", "com.linkedin.ml.metadata.MLModelDeploymentProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelGroupProperties", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.MLPrimaryKeyProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", { + }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryRelatedTerms", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserStatus", "com.linkedin.identity.GroupMembership", "com.linkedin.join.EditableJoinProperties", "com.linkedin.join.FieldMap", "com.linkedin.join.JoinFieldMapping", "com.linkedin.join.JoinProperties", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataHubPolicyAspect", "com.linkedin.metadata.aspect.DataHubRetentionAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.JoinAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.SchemaFieldAspect", "com.linkedin.metadata.aspect.TagAspect", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataHubPolicyKey", "com.linkedin.metadata.key.DataHubRetentionKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.JoinKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.SchemaFieldKey", "com.linkedin.metadata.key.TagKey", "com.linkedin.metadata.snapshot.ChartSnapshot", "com.linkedin.metadata.snapshot.CorpGroupSnapshot", "com.linkedin.metadata.snapshot.CorpUserSnapshot", "com.linkedin.metadata.snapshot.DashboardSnapshot", "com.linkedin.metadata.snapshot.DataFlowSnapshot", "com.linkedin.metadata.snapshot.DataHubPolicySnapshot", "com.linkedin.metadata.snapshot.DataHubRetentionSnapshot", "com.linkedin.metadata.snapshot.DataJobSnapshot", "com.linkedin.metadata.snapshot.DataPlatformSnapshot", "com.linkedin.metadata.snapshot.DataProcessSnapshot", "com.linkedin.metadata.snapshot.DatasetSnapshot", "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot", "com.linkedin.metadata.snapshot.GlossaryTermSnapshot", "com.linkedin.metadata.snapshot.JoinSnapshot", "com.linkedin.metadata.snapshot.MLFeatureSnapshot", "com.linkedin.metadata.snapshot.MLFeatureTableSnapshot", "com.linkedin.metadata.snapshot.MLModelDeploymentSnapshot", "com.linkedin.metadata.snapshot.MLModelGroupSnapshot", "com.linkedin.metadata.snapshot.MLModelSnapshot", "com.linkedin.metadata.snapshot.MLPrimaryKeySnapshot", "com.linkedin.metadata.snapshot.SchemaFieldSnapshot", "com.linkedin.metadata.snapshot.Snapshot", "com.linkedin.metadata.snapshot.TagSnapshot", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.DeploymentStatus", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLFeatureTableProperties", "com.linkedin.ml.metadata.MLHyperParam", "com.linkedin.ml.metadata.MLMetric", "com.linkedin.ml.metadata.MLModelDeploymentProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelGroupProperties", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.MLPrimaryKeyProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", { "type" : "record", "name" : "GenericPayload", "namespace" : "com.linkedin.mxe", From 35ea6ca39105987f5acd09357a93f74c16423f2e Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Thu, 15 Jun 2023 17:11:12 -0400 Subject: [PATCH 05/71] Remove Snapshot and Aspect --- .../datahub/graphql/GmsGraphQLEngine.java | 16 ++++++++++ .../join/mappers/JoinUpdateInputMapper.java | 25 +++++++++------- .../src/main/resources/entity.graphql | 4 +-- .../main/pegasus/com/linkedin/join/Joins.pdl | 15 ---------- .../linkedin/metadata/aspect/JoinAspect.pdl | 30 ------------------- .../metadata/snapshot/JoinSnapshot.pdl | 24 --------------- .../linkedin/metadata/snapshot/Snapshot.pdl | 1 - 7 files changed, 32 insertions(+), 83 deletions(-) delete mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl delete mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl delete mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index d5252473b06524..f16a6ecc74025d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -60,6 +60,7 @@ import com.linkedin.datahub.graphql.generated.IngestionSource; import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata; import com.linkedin.datahub.graphql.generated.Join; +import com.linkedin.datahub.graphql.generated.JoinProperties; import com.linkedin.datahub.graphql.generated.LineageRelationship; import com.linkedin.datahub.graphql.generated.ListAccessTokenResult; import com.linkedin.datahub.graphql.generated.ListOwnershipTypesResult; @@ -1410,6 +1411,7 @@ private void configureTypeExtensions(final RuntimeWiring.Builder builder) { private void configureJoinResolvers(final RuntimeWiring.Builder builder) { builder .type("Join", typeWiring -> typeWiring + .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient)) .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.joinType)) .dataFetcher("container", @@ -1419,6 +1421,20 @@ private void configureJoinResolvers(final RuntimeWiring.Builder builder) { return join.getContainer() != null ? join.getContainer().getUrn() : null; }) )) + .type("JoinProperties", typeWiring -> typeWiring + .dataFetcher("datasetA", + new LoadableTypeResolver<>(datasetType, + (env) -> { + final JoinProperties joinProperties = env.getSource(); + return joinProperties.getDatasetA() != null ? joinProperties.getDatasetA().getUrn() : null; + })) + .dataFetcher("datasetB", + new LoadableTypeResolver<>(datasetType, + (env) -> { + final JoinProperties joinProperties = env.getSource(); + return joinProperties.getDatasetB() != null ? joinProperties.getDatasetB().getUrn() : null; + })) + ) .type("Owner", typeWiring -> typeWiring .dataFetcher("owner", new OwnerTypeResolver<>(ownerTypes, (env) -> ((Owner) env.getSource()).getOwner())) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java index 11b27849aa32ec..d074447e894b24 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java @@ -1,8 +1,8 @@ package com.linkedin.datahub.graphql.types.join.mappers; -import com.linkedin.common.AuditStamp; -import com.linkedin.common.GlobalTags; import com.linkedin.common.TagAssociationArray; +import com.linkedin.common.TimeStamp; +import com.linkedin.common.GlobalTags; import com.linkedin.common.urn.DatasetUrn; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.SetMode; @@ -40,10 +40,10 @@ public static Collection map(@Nonnull final JoinUpdateIn public Collection apply(JoinUpdateInput input, Urn actor) { final Collection proposals = new ArrayList<>(8); final UpdateMappingHelper updateMappingHelper = new UpdateMappingHelper(JOIN_ENTITY_NAME); - final AuditStamp auditStamp = new AuditStamp(); - auditStamp.setActor(actor, SetMode.IGNORE_NULL); - auditStamp.setTime(System.currentTimeMillis()); - + final long currentTime = System.currentTimeMillis(); + final TimeStamp timestamp = new TimeStamp(); + timestamp.setActor(actor, SetMode.IGNORE_NULL); + timestamp.setTime(currentTime); if (input.getProperties() != null) { com.linkedin.join.JoinProperties joinProperties = new com.linkedin.join.JoinProperties(); if (input.getProperties().getName() != null) { @@ -84,9 +84,10 @@ public Collection apply(JoinUpdateInput input, Urn actor } joinProperties.setJoinFieldMappings(joinFieldMapping1); } + joinProperties.setLastModified(timestamp); proposals.add(updateMappingHelper.aspectToProposal(joinProperties, JOIN_PROPERTIES_ASPECT_NAME)); } - + } if (input.getOwnership() != null) { proposals.add(updateMappingHelper.aspectToProposal(OwnershipUpdateMapper.map(input.getOwnership(), actor), OWNERSHIP_ASPECT_NAME)); @@ -105,14 +106,16 @@ public Collection apply(JoinUpdateInput input, Urn actor } proposals.add(updateMappingHelper.aspectToProposal(globalTags, GLOBAL_TAGS_ASPECT_NAME)); } - if (input.getEditableProperties() != null) { final EditableJoinProperties editableJoinProperties = new EditableJoinProperties(); - editableJoinProperties.setName(input.getEditableProperties().getName()); - editableJoinProperties.setDescription(input.getEditableProperties().getDescription()); + if (input.getEditableProperties().getName().trim().length() > 0) { + editableJoinProperties.setName(input.getEditableProperties().getName()); + } + if (input.getEditableProperties().getDescription().trim().length() > 0) { + editableJoinProperties.setDescription(input.getEditableProperties().getDescription()); + } proposals.add(updateMappingHelper.aspectToProposal(editableJoinProperties, EDITABLE_JOIN_PROPERTIES_ASPECT_NAME)); } - } return proposals; } } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 54c63fd5e546dd..f14a603b231da4 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -385,12 +385,12 @@ type JoinProperties { """ The urn of datasetA """ - datasetA: String! + datasetA: Dataset! """ The urn of datasetB """ - datasetB: String! + datasetB: Dataset! """ The joinFieldMappings diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl deleted file mode 100644 index 2e339d15a2a2db..00000000000000 --- a/metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl +++ /dev/null @@ -1,15 +0,0 @@ -namespace com.linkedin.join -import com.linkedin.common.JoinUrn - -/** - * Joins information of an entity. - */ -@Aspect = { - "name": "joins" -} -record Joins { - /** - * Join - */ - joins: array[JoinUrn] -} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl deleted file mode 100644 index ef8e48ad3252fa..00000000000000 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl +++ /dev/null @@ -1,30 +0,0 @@ -namespace com.linkedin.metadata.aspect - -import com.linkedin.metadata.key.JoinKey -import com.linkedin.join.JoinProperties -import com.linkedin.join.EditableJoinProperties - -import com.linkedin.common.InstitutionalMemory -import com.linkedin.common.Ownership -import com.linkedin.common.Status -import com.linkedin.container.Container -import com.linkedin.common.GlobalTags -import com.linkedin.common.GlossaryTerms -import com.linkedin.common.BrowsePaths - - -/** - * A union of all supported metadata aspects for a Join - */ -typeref JoinAspect = union[ - JoinKey, - JoinProperties, - EditableJoinProperties, - InstitutionalMemory, - Ownership, - Status, - Container, - GlobalTags, - GlossaryTerms, - BrowsePaths -] diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl deleted file mode 100644 index 58bc2c59569ab3..00000000000000 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl +++ /dev/null @@ -1,24 +0,0 @@ -namespace com.linkedin.metadata.snapshot - -import com.linkedin.common.JoinUrn -import com.linkedin.metadata.aspect.JoinAspect - -/** - * A metadata snapshot for a specific join entity. - */ -@Entity = { - "name": "join", - "keyAspect": "joinKey" -} -record JoinSnapshot { - - /** - * URN for the entity the metadata snapshot is associated with. - */ - urn: JoinUrn - - /** - * The list of metadata aspects associated with the Join. Depending on the use case, this can either be all, or a selection, of supported aspects. - */ - aspects: array[JoinAspect] -} diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl index 374bdb6cd99069..91993724afbadc 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl @@ -25,5 +25,4 @@ typeref Snapshot = union[ DataHubPolicySnapshot, SchemaFieldSnapshot, DataHubRetentionSnapshot, - JoinSnapshot, ] From a819bfad828e0c479c06374a4a003010666e11c7 Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Fri, 23 Jun 2023 13:45:33 -0400 Subject: [PATCH 06/71] Join resolver updates --- .../types/join/mappers/JoinMapper.java | 26 +- .../join/mappers/JoinUpdateInputMapper.java | 14 + .../src/main/resources/entity.graphql | 26 +- datahub-web-react/src/graphql/dataset.graphql | 5 + .../src/graphql/fragments.graphql | 27 +- datahub-web-react/src/graphql/join.graphql | 9 +- datahub-web-react/src/graphql/search.graphql | 13 + ...com.linkedin.entity.entities.snapshot.json | 244 +----------------- ...m.linkedin.platform.platform.snapshot.json | 238 +---------------- 9 files changed, 114 insertions(+), 488 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java index e0d4b2a54dbf01..84ac2014847a2a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java @@ -9,6 +9,7 @@ import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.datahub.graphql.generated.Container; +import com.linkedin.datahub.graphql.generated.Dataset; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.Join; import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; @@ -84,23 +85,34 @@ private void mapJoinKey(@Nonnull Join join, @Nonnull DataMap datamap) { private void mapProperties(@Nonnull Join join, @Nonnull DataMap dataMap) { final JoinProperties joinProperties = new JoinProperties(dataMap); - join.setProperties( - com.linkedin.datahub.graphql.generated.JoinProperties.builder() - .setName(joinProperties.getName()) - .setDatasetA(joinProperties.getDatasetA().toString()) - .setDatasetB(joinProperties.getDatasetB().toString()) + join.setProperties(com.linkedin.datahub.graphql.generated.JoinProperties.builder() + .setName(joinProperties.getName()) + .setDatasetA(createPartialDataset(joinProperties.getDatasetA())) + .setDatasetB(createPartialDataset(joinProperties.getDatasetB())) .setJoinFieldMappings(mapJoinFieldMappings(joinProperties)) + .setCreatedActor(joinProperties.hasCreated() && joinProperties.getCreated().getActor().toString().length() > 0 + ? joinProperties.getCreated().getActor().toString() : "") + .setCreatedTime(joinProperties.hasCreated() && joinProperties.getCreated().getTime() > 0 + ? joinProperties.getCreated().getTime() : 0) .build()); } + private Dataset createPartialDataset(@Nonnull Urn datasetUrn) { + Dataset partialDataset = new Dataset(); + + partialDataset.setUrn(datasetUrn.toString()); + + return partialDataset; + + } private com.linkedin.datahub.graphql.generated.JoinFieldMapping mapJoinFieldMappings(JoinProperties joinProperties) { return com.linkedin.datahub.graphql.generated.JoinFieldMapping.builder() .setDetails(joinProperties.getJoinFieldMappings().getDetails()) .setFieldMapping(joinProperties.getJoinFieldMappings() .getFieldMapping() - .stream() + .stream() .map(this::mapFieldMap) - .collect(Collectors.toList())) + .collect(Collectors.toList())) .build(); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java index d074447e894b24..269647d034b707 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java @@ -84,6 +84,20 @@ public Collection apply(JoinUpdateInput input, Urn actor } joinProperties.setJoinFieldMappings(joinFieldMapping1); } + if (input.getProperties().getCreated() != null && input.getProperties().getCreated()) { + joinProperties.setCreated(timestamp); + } else { + if (input.getProperties().getCreatedBy().trim().length() > 0 && input.getProperties().getCreatedAt() != 0) { + final TimeStamp timestampEdit = new TimeStamp(); + try { + timestampEdit.setActor(Urn.createFromString(input.getProperties().getCreatedBy())); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + timestampEdit.setTime(input.getProperties().getCreatedAt()); + joinProperties.setCreated(timestampEdit); + } + } joinProperties.setLastModified(timestamp); proposals.add(updateMappingHelper.aspectToProposal(joinProperties, JOIN_PROPERTIES_ASPECT_NAME)); } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 6627ff8461d2e6..89b3a6f3e05da3 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -356,6 +356,10 @@ type Join implements EntityWithRelationships & Entity & BrowsableEntity { The browse paths corresponding to the dataset. If no Browse Paths have been generated before, this will be null. """ browsePaths: [BrowsePath!] + """ + Privileges given to a user relevant to this entity + """ + privileges: EntityPrivileges } """ @@ -397,6 +401,15 @@ type JoinProperties { """ joinFieldMappings: JoinFieldMapping! + """ + Created timestamp millis associated with the Join + """ + createdTime: Long + + """ + Actor associated with the Join's created timestamp + """ + createdActor: String } """ @@ -4599,7 +4612,18 @@ input JoinPropertiesInput { Details about the Join """ joinFieldmappings: JoinFieldMappingInput! - + """ + optional flag about the Join is getting create + """ + created: Boolean + """ + optional field to prevent created time while the Join is getting update + """ + createdAt: Long + """ + optional field to prevent create actor while the Join is getting update + """ + createdBy: String } """ diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index 0a3292057545d4..319ae3f3f63bc4 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -110,6 +110,11 @@ fragment nonSiblingDatasetFields on Dataset { timestampMillis lastUpdatedTimestamp } + incoming: relationships( + input: { types: ["joinA", "joinB"], direction: INCOMING, start: 0, count: 100 } + ) { + ...fullRelationshipResults + } ...viewProperties autoRenderAspects: aspects(input: { autoRenderOnly: true }) { aspectName diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index d93ec7405dbe8a..0e98107c91e275 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -973,11 +973,17 @@ fragment inputFieldsFields on InputFields { fragment joinPropertiesFields on JoinProperties { name - datasetA - datasetB + datasetA { + ...datasetJoinFields + } + datasetB { + ...datasetJoinFields + } joinFieldMappings { ...joinFieldMappingFields } + createdTime + createdActor } fragment joinFieldMappingFields on JoinFieldMapping { @@ -992,6 +998,23 @@ fragment joinEditablePropertiesFields on JoinEditableProperties { description name } + +fragment datasetJoinFields on Dataset { + urn + name + properties { + name + description + } + editableProperties { + name + description + } + schemaMetadata { + ...schemaMetadataFields + } +} + fragment browsePathV2Fields on BrowsePathV2 { path { name diff --git a/datahub-web-react/src/graphql/join.graphql b/datahub-web-react/src/graphql/join.graphql index 5b36778cb3d179..b7ea6e5df2e0bc 100644 --- a/datahub-web-react/src/graphql/join.graphql +++ b/datahub-web-react/src/graphql/join.graphql @@ -31,7 +31,7 @@ query getJoin($urn: String!) { ...glossaryTerms } outgoing: relationships( - input: { types: ["JoinA", "JoinB"], direction: OUTGOING, start: 0, count: 100 } + input: { types: ["joinA", "joinB"], direction: OUTGOING, start: 0, count: 100 } ) { ...fullRelationshipResults } @@ -40,3 +40,10 @@ query getJoin($urn: String!) { } } } + +mutation createJoin($input: JoinUpdateInput!) { + createJoin(input: $input) { + urn + joinId + } +} \ No newline at end of file diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index b035e76246f07f..04ce3f199d3eee 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -654,6 +654,18 @@ fragment searchResultFields on Entity { editableProperties { ...joinEditablePropertiesFields } + ownership { + ...ownershipFields + } + tags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + browsePaths { + path + } } ... on MLFeatureTable { name @@ -741,6 +753,7 @@ fragment searchResultFields on Entity { editableProperties { ...joinEditablePropertiesFields } + } ... on DataProduct { ...dataProductSearchFields } diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 93e448335c0903..089ff16a0102b5 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -1015,34 +1015,6 @@ "name" : "institutionalMemory" } }, "com.linkedin.common.InstitutionalMemoryMetadata", { - "type" : "typeref", - "name" : "JoinUrn", - "namespace" : "com.linkedin.common", - "doc" : "Standardized join identifier.", - "ref" : "string", - "java" : { - "class" : "com.linkedin.common.urn.JoinUrn" - }, - "validate" : { - "com.linkedin.common.validator.TypedUrnValidator" : { - "accessible" : true, - "constructable" : true, - "doc" : "Standardized Join identifier.", - "entityType" : "join", - "fields" : [ { - "doc" : "Join native name e.g. .
, /dir/subdir/, or ", - "maxLength" : 284, - "name" : "joinId", - "type" : "string" - } ], - "maxLength" : 284, - "name" : "Join", - "namespace" : "li", - "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], - "owningTeam" : "urn:li:internalTeam:datahub" - } - } - }, { "type" : "enum", "name" : "MLFeatureDataType", "namespace" : "com.linkedin.common", @@ -1305,30 +1277,6 @@ "optional" : true } ] }, "com.linkedin.common.fieldtransformer.TransformationType", "com.linkedin.common.fieldtransformer.UDFTransformer", { - "type" : "record", - "name" : "Container", - "namespace" : "com.linkedin.container", - "doc" : "Link from an asset to its parent container", - "fields" : [ { - "name" : "container", - "type" : "com.linkedin.common.Urn", - "doc" : "The parent container of an asset", - "Relationship" : { - "entityTypes" : [ "container" ], - "name" : "IsPartOf" - }, - "Searchable" : { - "addToFilters" : true, - "fieldName" : "container", - "fieldType" : "URN", - "filterNameOverride" : "Container", - "hasValuesFieldName" : "hasContainer" - } - } ], - "Aspect" : { - "name" : "container" - } - }, { "type" : "record", "name" : "DashboardInfo", "namespace" : "com.linkedin.dashboard", @@ -5487,194 +5435,10 @@ "keyAspect" : "dataHubRetentionKey", "name" : "dataHubRetention" } - }, { - "type" : "record", - "name" : "JoinSnapshot", - "doc" : "A metadata snapshot for a specific join entity.", - "fields" : [ { - "name" : "urn", - "type" : "com.linkedin.common.JoinUrn", - "doc" : "URN for the entity the metadata snapshot is associated with." - }, { - "name" : "aspects", - "type" : { - "type" : "array", - "items" : { - "type" : "typeref", - "name" : "JoinAspect", - "namespace" : "com.linkedin.metadata.aspect", - "doc" : "A union of all supported metadata aspects for a Join", - "ref" : [ { - "type" : "record", - "name" : "JoinKey", - "namespace" : "com.linkedin.metadata.key", - "doc" : "Key for a Join", - "fields" : [ { - "name" : "joinId", - "type" : "string", - "Searchable" : { - "fieldType" : "TEXT" - } - } ], - "Aspect" : { - "name" : "joinKey" - } - }, { - "type" : "record", - "name" : "JoinProperties", - "namespace" : "com.linkedin.join", - "doc" : "Properties associated with a Join", - "include" : [ "com.linkedin.common.CustomProperties" ], - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the Join", - "Searchable" : { - "boostScore" : 10.0, - "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL" - } - }, { - "name" : "datasetA", - "type" : "com.linkedin.common.DatasetUrn", - "doc" : "First dataset in the join (no directionality)", - "Relationship" : { - "entityTypes" : [ "dataset" ], - "name" : "joinA" - }, - "Searchable" : { - "boostScore" : 10.0, - "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL" - } - }, { - "name" : "datasetB", - "type" : "com.linkedin.common.DatasetUrn", - "doc" : "Second dataset in the join (no directionality)", - "Relationship" : { - "entityTypes" : [ "dataset" ], - "name" : "joinB" - }, - "Searchable" : { - "boostScore" : 10.0, - "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL" - } - }, { - "name" : "joinFieldMappings", - "type" : { - "type" : "record", - "name" : "JoinFieldMapping", - "doc" : "Field Mapping about a join between two datasets", - "fields" : [ { - "name" : "fieldMapping", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "FieldMap", - "doc" : "Field Mapping of 1:1 field", - "fields" : [ { - "name" : "afield", - "type" : "com.linkedin.dataset.SchemaFieldPath", - "doc" : "All fields from dataset A that are required for the join, maps to bFields 1:1", - "Searchable" : { - "boostScore" : 10.0, - "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL" - } - }, { - "name" : "bfield", - "type" : "com.linkedin.dataset.SchemaFieldPath", - "doc" : "All fields from dataset B that are required for the join, maps to aFields 1:1", - "Searchable" : { - "boostScore" : 10.0, - "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL" - } - } ] - } - }, - "doc" : "All fields from dataset A that are required for the join to dataset B" - }, { - "name" : "details", - "type" : "string", - "doc" : " Any transformation logic or notes pertaining to this specific join", - "Searchable" : { - "boostScore" : 10.0, - "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL" - } - } ] - }, - "doc" : "JoinFieldMapping (in future we can make it an array)" - }, { - "name" : "created", - "type" : "com.linkedin.common.TimeStamp", - "doc" : "A timestamp documenting when the asset was created in the source Data Platform (not on DataHub)", - "optional" : true, - "Searchable" : { - "/time" : { - "fieldName" : "createdAt", - "fieldType" : "DATETIME" - } - } - }, { - "name" : "lastModified", - "type" : "com.linkedin.common.TimeStamp", - "doc" : "A timestamp documenting when the asset was last modified in the source Data Platform (not on DataHub)", - "optional" : true, - "Searchable" : { - "/time" : { - "fieldName" : "lastModifiedAt", - "fieldType" : "DATETIME" - } - } - } ], - "Aspect" : { - "name" : "joinProperties" - } - }, { - "type" : "record", - "name" : "EditableJoinProperties", - "namespace" : "com.linkedin.join", - "doc" : "EditableJoinProperties stores editable changes made to join properties. This separates changes made from\ningestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines", - "include" : [ "com.linkedin.common.ChangeAuditStamps" ], - "fields" : [ { - "name" : "description", - "type" : "string", - "doc" : "Documentation of the join", - "optional" : true, - "Searchable" : { - "fieldName" : "editedDescription", - "fieldType" : "TEXT" - } - }, { - "name" : "name", - "type" : "string", - "doc" : "Display name of the Join", - "optional" : true, - "Searchable" : { - "fieldName" : "editedName", - "fieldType" : "TEXT_PARTIAL" - } - } ], - "Aspect" : { - "name" : "editableJoinProperties" - } - }, "com.linkedin.common.InstitutionalMemory", "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.container.Container", "com.linkedin.common.GlobalTags", "com.linkedin.common.GlossaryTerms", "com.linkedin.common.BrowsePaths" ] - } - }, - "doc" : "The list of metadata aspects associated with the Join. Depending on the use case, this can either be all, or a selection, of supported aspects." - } ], - "Entity" : { - "keyAspect" : "joinKey", - "name" : "join" - } } ] } } ] - }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryRelatedTerms", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserStatus", "com.linkedin.identity.GroupMembership", "com.linkedin.join.EditableJoinProperties", "com.linkedin.join.FieldMap", "com.linkedin.join.JoinFieldMapping", "com.linkedin.join.JoinProperties", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataHubPolicyAspect", "com.linkedin.metadata.aspect.DataHubRetentionAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.JoinAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.SchemaFieldAspect", "com.linkedin.metadata.aspect.TagAspect", { + }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryRelatedTerms", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserStatus", "com.linkedin.identity.GroupMembership", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataHubPolicyAspect", "com.linkedin.metadata.aspect.DataHubRetentionAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.SchemaFieldAspect", "com.linkedin.metadata.aspect.TagAspect", { "type" : "record", "name" : "BrowseResult", "namespace" : "com.linkedin.metadata.browse", @@ -5758,7 +5522,7 @@ "type" : "int", "doc" : "The total number of elements (entities + groups) directly under queried path" } ] - }, "com.linkedin.metadata.browse.BrowseResultEntity", "com.linkedin.metadata.browse.BrowseResultGroup", "com.linkedin.metadata.browse.BrowseResultMetadata", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataHubPolicyKey", "com.linkedin.metadata.key.DataHubRetentionKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.JoinKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.SchemaFieldKey", "com.linkedin.metadata.key.TagKey", { + }, "com.linkedin.metadata.browse.BrowseResultEntity", "com.linkedin.metadata.browse.BrowseResultGroup", "com.linkedin.metadata.browse.BrowseResultMetadata", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataHubPolicyKey", "com.linkedin.metadata.key.DataHubRetentionKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.SchemaFieldKey", "com.linkedin.metadata.key.TagKey", { "type" : "record", "name" : "AnyResult", "namespace" : "com.linkedin.metadata.query", @@ -5882,7 +5646,7 @@ }, { "name" : "maxAggValues", "type" : "int", - "doc" : "The maximum number of values in an facet aggregation", + "doc" : "The maximum number of values in a facet aggregation", "default" : 20 }, { "name" : "fulltext", @@ -6378,7 +6142,7 @@ "type" : "int", "doc" : "The total number of entities directly under searched path" } ] - }, "com.linkedin.metadata.search.SearchResultMetadata", "com.linkedin.metadata.snapshot.ChartSnapshot", "com.linkedin.metadata.snapshot.CorpGroupSnapshot", "com.linkedin.metadata.snapshot.CorpUserSnapshot", "com.linkedin.metadata.snapshot.DashboardSnapshot", "com.linkedin.metadata.snapshot.DataFlowSnapshot", "com.linkedin.metadata.snapshot.DataHubPolicySnapshot", "com.linkedin.metadata.snapshot.DataHubRetentionSnapshot", "com.linkedin.metadata.snapshot.DataJobSnapshot", "com.linkedin.metadata.snapshot.DataPlatformSnapshot", "com.linkedin.metadata.snapshot.DataProcessSnapshot", "com.linkedin.metadata.snapshot.DatasetSnapshot", "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot", "com.linkedin.metadata.snapshot.GlossaryTermSnapshot", "com.linkedin.metadata.snapshot.JoinSnapshot", "com.linkedin.metadata.snapshot.MLFeatureSnapshot", "com.linkedin.metadata.snapshot.MLFeatureTableSnapshot", "com.linkedin.metadata.snapshot.MLModelDeploymentSnapshot", "com.linkedin.metadata.snapshot.MLModelGroupSnapshot", "com.linkedin.metadata.snapshot.MLModelSnapshot", "com.linkedin.metadata.snapshot.MLPrimaryKeySnapshot", "com.linkedin.metadata.snapshot.SchemaFieldSnapshot", "com.linkedin.metadata.snapshot.Snapshot", "com.linkedin.metadata.snapshot.TagSnapshot", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.DeploymentStatus", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLFeatureTableProperties", "com.linkedin.ml.metadata.MLHyperParam", "com.linkedin.ml.metadata.MLMetric", "com.linkedin.ml.metadata.MLModelDeploymentProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelGroupProperties", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.MLPrimaryKeyProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", { + }, "com.linkedin.metadata.search.SearchResultMetadata", "com.linkedin.metadata.snapshot.ChartSnapshot", "com.linkedin.metadata.snapshot.CorpGroupSnapshot", "com.linkedin.metadata.snapshot.CorpUserSnapshot", "com.linkedin.metadata.snapshot.DashboardSnapshot", "com.linkedin.metadata.snapshot.DataFlowSnapshot", "com.linkedin.metadata.snapshot.DataHubPolicySnapshot", "com.linkedin.metadata.snapshot.DataHubRetentionSnapshot", "com.linkedin.metadata.snapshot.DataJobSnapshot", "com.linkedin.metadata.snapshot.DataPlatformSnapshot", "com.linkedin.metadata.snapshot.DataProcessSnapshot", "com.linkedin.metadata.snapshot.DatasetSnapshot", "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot", "com.linkedin.metadata.snapshot.GlossaryTermSnapshot", "com.linkedin.metadata.snapshot.MLFeatureSnapshot", "com.linkedin.metadata.snapshot.MLFeatureTableSnapshot", "com.linkedin.metadata.snapshot.MLModelDeploymentSnapshot", "com.linkedin.metadata.snapshot.MLModelGroupSnapshot", "com.linkedin.metadata.snapshot.MLModelSnapshot", "com.linkedin.metadata.snapshot.MLPrimaryKeySnapshot", "com.linkedin.metadata.snapshot.SchemaFieldSnapshot", "com.linkedin.metadata.snapshot.Snapshot", "com.linkedin.metadata.snapshot.TagSnapshot", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.DeploymentStatus", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLFeatureTableProperties", "com.linkedin.ml.metadata.MLHyperParam", "com.linkedin.ml.metadata.MLMetric", "com.linkedin.ml.metadata.MLModelDeploymentProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelGroupProperties", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.MLPrimaryKeyProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", { "type" : "record", "name" : "SystemMetadata", "namespace" : "com.linkedin.mxe", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json index f72aa088e56fb5..6a8c8f054779f9 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json @@ -1015,34 +1015,6 @@ "name" : "institutionalMemory" } }, "com.linkedin.common.InstitutionalMemoryMetadata", { - "type" : "typeref", - "name" : "JoinUrn", - "namespace" : "com.linkedin.common", - "doc" : "Standardized join identifier.", - "ref" : "string", - "java" : { - "class" : "com.linkedin.common.urn.JoinUrn" - }, - "validate" : { - "com.linkedin.common.validator.TypedUrnValidator" : { - "accessible" : true, - "constructable" : true, - "doc" : "Standardized Join identifier.", - "entityType" : "join", - "fields" : [ { - "doc" : "Join native name e.g. .
, /dir/subdir/, or ", - "maxLength" : 284, - "name" : "joinId", - "type" : "string" - } ], - "maxLength" : 284, - "name" : "Join", - "namespace" : "li", - "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], - "owningTeam" : "urn:li:internalTeam:datahub" - } - } - }, { "type" : "enum", "name" : "MLFeatureDataType", "namespace" : "com.linkedin.common", @@ -1305,30 +1277,6 @@ "optional" : true } ] }, "com.linkedin.common.fieldtransformer.TransformationType", "com.linkedin.common.fieldtransformer.UDFTransformer", { - "type" : "record", - "name" : "Container", - "namespace" : "com.linkedin.container", - "doc" : "Link from an asset to its parent container", - "fields" : [ { - "name" : "container", - "type" : "com.linkedin.common.Urn", - "doc" : "The parent container of an asset", - "Relationship" : { - "entityTypes" : [ "container" ], - "name" : "IsPartOf" - }, - "Searchable" : { - "addToFilters" : true, - "fieldName" : "container", - "fieldType" : "URN", - "filterNameOverride" : "Container", - "hasValuesFieldName" : "hasContainer" - } - } ], - "Aspect" : { - "name" : "container" - } - }, { "type" : "record", "name" : "DashboardInfo", "namespace" : "com.linkedin.dashboard", @@ -5481,194 +5429,10 @@ "keyAspect" : "dataHubRetentionKey", "name" : "dataHubRetention" } - }, { - "type" : "record", - "name" : "JoinSnapshot", - "doc" : "A metadata snapshot for a specific join entity.", - "fields" : [ { - "name" : "urn", - "type" : "com.linkedin.common.JoinUrn", - "doc" : "URN for the entity the metadata snapshot is associated with." - }, { - "name" : "aspects", - "type" : { - "type" : "array", - "items" : { - "type" : "typeref", - "name" : "JoinAspect", - "namespace" : "com.linkedin.metadata.aspect", - "doc" : "A union of all supported metadata aspects for a Join", - "ref" : [ { - "type" : "record", - "name" : "JoinKey", - "namespace" : "com.linkedin.metadata.key", - "doc" : "Key for a Join", - "fields" : [ { - "name" : "joinId", - "type" : "string", - "Searchable" : { - "fieldType" : "TEXT" - } - } ], - "Aspect" : { - "name" : "joinKey" - } - }, { - "type" : "record", - "name" : "JoinProperties", - "namespace" : "com.linkedin.join", - "doc" : "Properties associated with a Join", - "include" : [ "com.linkedin.common.CustomProperties" ], - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the Join", - "Searchable" : { - "boostScore" : 10.0, - "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL" - } - }, { - "name" : "datasetA", - "type" : "com.linkedin.common.DatasetUrn", - "doc" : "First dataset in the join (no directionality)", - "Relationship" : { - "entityTypes" : [ "dataset" ], - "name" : "joinA" - }, - "Searchable" : { - "boostScore" : 10.0, - "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL" - } - }, { - "name" : "datasetB", - "type" : "com.linkedin.common.DatasetUrn", - "doc" : "Second dataset in the join (no directionality)", - "Relationship" : { - "entityTypes" : [ "dataset" ], - "name" : "joinB" - }, - "Searchable" : { - "boostScore" : 10.0, - "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL" - } - }, { - "name" : "joinFieldMappings", - "type" : { - "type" : "record", - "name" : "JoinFieldMapping", - "doc" : "Field Mapping about a join between two datasets", - "fields" : [ { - "name" : "fieldMapping", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "FieldMap", - "doc" : "Field Mapping of 1:1 field", - "fields" : [ { - "name" : "afield", - "type" : "com.linkedin.dataset.SchemaFieldPath", - "doc" : "All fields from dataset A that are required for the join, maps to bFields 1:1", - "Searchable" : { - "boostScore" : 10.0, - "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL" - } - }, { - "name" : "bfield", - "type" : "com.linkedin.dataset.SchemaFieldPath", - "doc" : "All fields from dataset B that are required for the join, maps to aFields 1:1", - "Searchable" : { - "boostScore" : 10.0, - "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL" - } - } ] - } - }, - "doc" : "All fields from dataset A that are required for the join to dataset B" - }, { - "name" : "details", - "type" : "string", - "doc" : " Any transformation logic or notes pertaining to this specific join", - "Searchable" : { - "boostScore" : 10.0, - "enableAutocomplete" : true, - "fieldType" : "TEXT_PARTIAL" - } - } ] - }, - "doc" : "JoinFieldMapping (in future we can make it an array)" - }, { - "name" : "created", - "type" : "com.linkedin.common.TimeStamp", - "doc" : "A timestamp documenting when the asset was created in the source Data Platform (not on DataHub)", - "optional" : true, - "Searchable" : { - "/time" : { - "fieldName" : "createdAt", - "fieldType" : "DATETIME" - } - } - }, { - "name" : "lastModified", - "type" : "com.linkedin.common.TimeStamp", - "doc" : "A timestamp documenting when the asset was last modified in the source Data Platform (not on DataHub)", - "optional" : true, - "Searchable" : { - "/time" : { - "fieldName" : "lastModifiedAt", - "fieldType" : "DATETIME" - } - } - } ], - "Aspect" : { - "name" : "joinProperties" - } - }, { - "type" : "record", - "name" : "EditableJoinProperties", - "namespace" : "com.linkedin.join", - "doc" : "EditableJoinProperties stores editable changes made to join properties. This separates changes made from\ningestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines", - "include" : [ "com.linkedin.common.ChangeAuditStamps" ], - "fields" : [ { - "name" : "description", - "type" : "string", - "doc" : "Documentation of the join", - "optional" : true, - "Searchable" : { - "fieldName" : "editedDescription", - "fieldType" : "TEXT" - } - }, { - "name" : "name", - "type" : "string", - "doc" : "Display name of the Join", - "optional" : true, - "Searchable" : { - "fieldName" : "editedName", - "fieldType" : "TEXT_PARTIAL" - } - } ], - "Aspect" : { - "name" : "editableJoinProperties" - } - }, "com.linkedin.common.InstitutionalMemory", "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.container.Container", "com.linkedin.common.GlobalTags", "com.linkedin.common.GlossaryTerms", "com.linkedin.common.BrowsePaths" ] - } - }, - "doc" : "The list of metadata aspects associated with the Join. Depending on the use case, this can either be all, or a selection, of supported aspects." - } ], - "Entity" : { - "keyAspect" : "joinKey", - "name" : "join" - } } ] } } ] - }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryRelatedTerms", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserStatus", "com.linkedin.identity.GroupMembership", "com.linkedin.join.EditableJoinProperties", "com.linkedin.join.FieldMap", "com.linkedin.join.JoinFieldMapping", "com.linkedin.join.JoinProperties", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataHubPolicyAspect", "com.linkedin.metadata.aspect.DataHubRetentionAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.JoinAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.SchemaFieldAspect", "com.linkedin.metadata.aspect.TagAspect", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataHubPolicyKey", "com.linkedin.metadata.key.DataHubRetentionKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.JoinKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.SchemaFieldKey", "com.linkedin.metadata.key.TagKey", "com.linkedin.metadata.snapshot.ChartSnapshot", "com.linkedin.metadata.snapshot.CorpGroupSnapshot", "com.linkedin.metadata.snapshot.CorpUserSnapshot", "com.linkedin.metadata.snapshot.DashboardSnapshot", "com.linkedin.metadata.snapshot.DataFlowSnapshot", "com.linkedin.metadata.snapshot.DataHubPolicySnapshot", "com.linkedin.metadata.snapshot.DataHubRetentionSnapshot", "com.linkedin.metadata.snapshot.DataJobSnapshot", "com.linkedin.metadata.snapshot.DataPlatformSnapshot", "com.linkedin.metadata.snapshot.DataProcessSnapshot", "com.linkedin.metadata.snapshot.DatasetSnapshot", "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot", "com.linkedin.metadata.snapshot.GlossaryTermSnapshot", "com.linkedin.metadata.snapshot.JoinSnapshot", "com.linkedin.metadata.snapshot.MLFeatureSnapshot", "com.linkedin.metadata.snapshot.MLFeatureTableSnapshot", "com.linkedin.metadata.snapshot.MLModelDeploymentSnapshot", "com.linkedin.metadata.snapshot.MLModelGroupSnapshot", "com.linkedin.metadata.snapshot.MLModelSnapshot", "com.linkedin.metadata.snapshot.MLPrimaryKeySnapshot", "com.linkedin.metadata.snapshot.SchemaFieldSnapshot", "com.linkedin.metadata.snapshot.Snapshot", "com.linkedin.metadata.snapshot.TagSnapshot", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.DeploymentStatus", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLFeatureTableProperties", "com.linkedin.ml.metadata.MLHyperParam", "com.linkedin.ml.metadata.MLMetric", "com.linkedin.ml.metadata.MLModelDeploymentProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelGroupProperties", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.MLPrimaryKeyProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", { + }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryRelatedTerms", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserStatus", "com.linkedin.identity.GroupMembership", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataHubPolicyAspect", "com.linkedin.metadata.aspect.DataHubRetentionAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.SchemaFieldAspect", "com.linkedin.metadata.aspect.TagAspect", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataHubPolicyKey", "com.linkedin.metadata.key.DataHubRetentionKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.SchemaFieldKey", "com.linkedin.metadata.key.TagKey", "com.linkedin.metadata.snapshot.ChartSnapshot", "com.linkedin.metadata.snapshot.CorpGroupSnapshot", "com.linkedin.metadata.snapshot.CorpUserSnapshot", "com.linkedin.metadata.snapshot.DashboardSnapshot", "com.linkedin.metadata.snapshot.DataFlowSnapshot", "com.linkedin.metadata.snapshot.DataHubPolicySnapshot", "com.linkedin.metadata.snapshot.DataHubRetentionSnapshot", "com.linkedin.metadata.snapshot.DataJobSnapshot", "com.linkedin.metadata.snapshot.DataPlatformSnapshot", "com.linkedin.metadata.snapshot.DataProcessSnapshot", "com.linkedin.metadata.snapshot.DatasetSnapshot", "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot", "com.linkedin.metadata.snapshot.GlossaryTermSnapshot", "com.linkedin.metadata.snapshot.MLFeatureSnapshot", "com.linkedin.metadata.snapshot.MLFeatureTableSnapshot", "com.linkedin.metadata.snapshot.MLModelDeploymentSnapshot", "com.linkedin.metadata.snapshot.MLModelGroupSnapshot", "com.linkedin.metadata.snapshot.MLModelSnapshot", "com.linkedin.metadata.snapshot.MLPrimaryKeySnapshot", "com.linkedin.metadata.snapshot.SchemaFieldSnapshot", "com.linkedin.metadata.snapshot.Snapshot", "com.linkedin.metadata.snapshot.TagSnapshot", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.DeploymentStatus", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLFeatureTableProperties", "com.linkedin.ml.metadata.MLHyperParam", "com.linkedin.ml.metadata.MLMetric", "com.linkedin.ml.metadata.MLModelDeploymentProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelGroupProperties", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.MLPrimaryKeyProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", { "type" : "record", "name" : "GenericPayload", "namespace" : "com.linkedin.mxe", From 0cd63236bbc955ef0d83ac9e8ef6d084759b44bb Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Sun, 25 Jun 2023 13:29:27 -0400 Subject: [PATCH 07/71] Additional changes to graphql files --- .../src/main/resources/entity.graphql | 5 + datahub-web-react/src/graphql/lineage.graphql | 264 ++++++++++++ .../src/graphql/relationships.graphql | 33 ++ datahub-web-react/src/graphql/search.graphql | 386 +++++++++++++++++- 4 files changed, 685 insertions(+), 3 deletions(-) diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 89b3a6f3e05da3..a1c5fd45d894ff 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -3152,6 +3152,11 @@ Dataset properties that are editable via the UI This represents logical metadata as opposed to technical metadata """ type DatasetEditableProperties { + """ + Display name of the Dataset + """ + name: String + """ Description of the Dataset """ diff --git a/datahub-web-react/src/graphql/lineage.graphql b/datahub-web-react/src/graphql/lineage.graphql index 54f704499c1205..9c4251c3b451c1 100644 --- a/datahub-web-react/src/graphql/lineage.graphql +++ b/datahub-web-react/src/graphql/lineage.graphql @@ -162,6 +162,7 @@ fragment lineageNodeProperties on EntityWithRelationships { qualifiedName } editableProperties { + name description } platform { @@ -238,6 +239,269 @@ fragment lineageNodeProperties on EntityWithRelationships { editableProperties { ...joinEditablePropertiesFields } + ownership { + ...ownershipFields + } + tags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + browsePaths { + path + } + } +} + +fragment lineageNodePropertiesOptimized on EntityWithRelationships { + type + ... on DataJob { + urn + type + dataFlow { + ...nonRecursiveDataFlowFields + } + jobId + ownership { + ...ownershipFields + } + properties { + name + description + externalUrl + customProperties { + key + value + } + } + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + domain { + ...entityDomain + } + deprecation { + ...deprecationFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + editableProperties { + description + } + status { + removed + } + } + ... on DataFlow { + orchestrator + flowId + cluster + properties { + name + description + project + } + ownership { + ...ownershipFields + } + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + editableProperties { + description + } + platform { + ...platformFields + } + domain { + ...entityDomain + } + status { + removed + } + } + ... on Dashboard { + urn + type + tool + dashboardId + properties { + name + description + externalUrl + lastRefreshed + created { + time + } + lastModified { + time + } + } + ownership { + ...ownershipFields + } + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + platform { + ...platformFields + } + domain { + ...entityDomain + } + parentContainers { + ...parentContainersFields + } + status { + removed + } + deprecation { + ...deprecationFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + editableProperties { + description + } + status { + removed + } + subTypes { + typeNames + } + } + ... on Chart { + tool + chartId + properties { + name + description + } + editableProperties { + description + } + ownership { + ...ownershipFields + } + platform { + ...platformFields + } + domain { + ...entityDomain + } + status { + removed + } + } + ... on Dataset { + name + properties { + name + description + qualifiedName + } + editableProperties { + name + description + } + platform { + ...platformFields + } + ownership { + ...ownershipFields + } + subTypes { + typeNames + } + status { + removed + } + fineGrainedLineages { + upstreams { + urn + path + } + downstreams { + urn + path + } + } + } + ... on MLModelGroup { + urn + type + name + description + origin + platform { + ...platformFields + } + ownership { + ...ownershipFields + } + status { + removed + } + } + ... on MLModel { + urn + type + name + description + origin + platform { + ...platformFields + } + ownership { + ...ownershipFields + } + status { + removed + } + } + ... on MLFeatureTable { + ...nonRecursiveMLFeatureTable + } + ... on MLFeature { + ...nonRecursiveMLFeature + } + ... on MLPrimaryKey { + ...nonRecursiveMLPrimaryKey + } + ... on Join { + urn + type + joinId + properties { + ...joinPropertiesFields + } + editableProperties { + ...joinEditablePropertiesFields + } + ownership { + ...ownershipFields + } + tags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + browsePaths { + path + } } } diff --git a/datahub-web-react/src/graphql/relationships.graphql b/datahub-web-react/src/graphql/relationships.graphql index e2afff5dfa6d27..f224e981052b41 100644 --- a/datahub-web-react/src/graphql/relationships.graphql +++ b/datahub-web-react/src/graphql/relationships.graphql @@ -11,6 +11,19 @@ fragment fullRelationshipResults on EntityRelationshipsResult { } } +fragment fullRelationshipResultsOptimized on EntityRelationshipsResult { + start + count + total + relationships { + type + direction + entity { + ...relationshipFieldsOptimized + } + } +} + fragment leafRelationshipResults on EntityRelationshipsResult { start count @@ -43,3 +56,23 @@ fragment relationshipFields on EntityWithRelationships { ...leafLineageResults } } + +fragment relationshipFieldsOptimized on EntityWithRelationships { + ...lineageNodePropertiesOptimized + ... on Dataset { + siblings { + isPrimary + siblings { + urn + type + ...lineageNodePropertiesOptimized + } + } + } + upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) { + ...leafLineageResults + } + downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) { + ...leafLineageResults + } +} \ No newline at end of file diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index 04ce3f199d3eee..b14b631e403be7 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -264,6 +264,7 @@ fragment searchResultFields on Entity { ...dataPlatformInstanceFields } editableProperties { + name description } platformNativeType @@ -644,6 +645,82 @@ fragment searchResultFields on Entity { ...parentContainersFields } } + ... on MLFeatureTable { + name + description + featureTableProperties { + description + mlFeatures { + urn + } + mlPrimaryKeys { + urn + } + } + ownership { + ...ownershipFields + } + platform { + ...platformFields + } + deprecation { + ...deprecationFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } + ... on MLFeature { + ...nonRecursiveMLFeature + } + ... on MLPrimaryKey { + ...nonRecursiveMLPrimaryKey + } + ... on MLModel { + name + description + origin + ownership { + ...ownershipFields + } + platform { + ...platformFields + } + deprecation { + ...deprecationFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } + ... on MLModelGroup { + name + origin + description + ownership { + ...ownershipFields + } + platform { + ...platformFields + } + deprecation { + ...deprecationFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + } + ... on Tag { + name + properties { + name + colorHex + } + description + } + ... on DataPlatform { + ...nonConflictingPlatformFields + } ... on Join { urn type @@ -667,6 +744,300 @@ fragment searchResultFields on Entity { path } } +} + + + +fragment searchResultFieldsWithRelationships on Entity { + urn + type + ... on Dataset { + name + origin + uri + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + editableProperties { + name + description + } + platformNativeType + properties { + name + description + qualifiedName + customProperties { + key + value + } + externalUrl + } + ownership { + ...ownershipFields + } + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + subTypes { + typeNames + } + incoming: relationships( + input: { types: ["DownstreamOf", "Consumes", "Produces", "Subscribes", "Registers"], direction: INCOMING, start: 0, count: 100 } + ) { + ...fullRelationshipResultsOptimized + } + domain { + ...entityDomain + } + container { + ...entityContainer + } + } + ... on CorpUser { + username + properties { + active + displayName + title + firstName + lastName + fullName + email + } + info { + active + displayName + title + firstName + lastName + fullName + email + } + editableProperties { + displayName + title + pictureLink + } + } + ... on CorpGroup { + name + info { + displayName + description + } + memberCount: relationships(input: { types: ["IsMemberOfGroup"], direction: INCOMING, start: 0, count: 1 }) { + total + } + } + ... on Dashboard { + urn + type + tool + dashboardId + properties { + name + description + externalUrl + access + lastModified { + time + } + } + ownership { + ...ownershipFields + } + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + editableProperties { + description + } + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + domain { + ...entityDomain + } + container { + ...entityContainer + } + } + ... on Chart { + urn + type + tool + chartId + properties { + name + description + externalUrl + type + access + lastModified { + time + } + } + ownership { + ...ownershipFields + } + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + editableProperties { + description + } + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + domain { + ...entityDomain + } + container { + ...entityContainer + } + } + ... on DataFlow { + urn + type + orchestrator + flowId + cluster + properties { + name + description + project + } + ownership { + ...ownershipFields + } + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + editableProperties { + description + } + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + domain { + ...entityDomain + } + } + ... on DataJob { + urn + type + dataFlow { + ...nonRecursiveDataFlowFields + } + jobId + ownership { + ...ownershipFields + } + properties { + name + description + } + globalTags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + editableProperties { + description + } + domain { + ...entityDomain + } + } + ... on GlossaryTerm { + name + hierarchicalName + properties { + name + description + termSource + sourceRef + sourceUrl + rawSchema + customProperties { + key + value + } + } + } + ... on Domain { + urn + properties { + name + description + } + ownership { + ...ownershipFields + } + } + ... on Container { + urn + properties { + name + description + } + platform { + ...platformFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + editableProperties { + description + } + ownership { + ...ownershipFields + } + tags { + ...globalTagsFields + } + institutionalMemory { + ...institutionalMemoryFields + } + glossaryTerms { + ...glossaryTerms + } + subTypes { + typeNames + } + entities(input: {}) { + total + } + container { + ...entityContainer + } + } ... on MLFeatureTable { name description @@ -753,9 +1124,18 @@ fragment searchResultFields on Entity { editableProperties { ...joinEditablePropertiesFields } - } - ... on DataProduct { - ...dataProductSearchFields + ownership { + ...ownershipFields + } + tags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + browsePaths { + path + } } } From f84a576531ec9b72ef49078e62b7c4cd0ffcb5aa Mon Sep 17 00:00:00 2001 From: plashkar Date: Tue, 27 Jun 2023 20:53:47 +0530 Subject: [PATCH 08/71] join ui changes --- datahub-web-react/src/App.tsx | 2 + .../src/app/entity/dataset/DatasetEntity.tsx | 5 + .../src/app/entity/joins/JoinEntity.tsx | 142 ++++++ .../app/entity/joins/preview/JoinAction.less | 12 + .../entity/joins/preview/JoinPreviewCard.tsx | 54 ++ .../styled/Join/CreateJoinModal.less | 273 ++++++++++ .../styled/Join/CreateJoinModal.tsx | 475 ++++++++++++++++++ .../components/styled/Join/JoinPreview.less | 217 ++++++++ .../components/styled/Join/JoinPreview.tsx | 158 ++++++ .../search/EmbeddedListSearchResults.tsx | 3 + .../components/styled/search/SearchSelect.tsx | 32 +- .../styled/search/SearchSelectModal.tsx | 3 + .../Dataset/Relationship/RelationTab.less | 84 ++++ .../tabs/Dataset/Relationship/RelationTab.tsx | 325 ++++++++++++ .../app/entity/shared/tabs/Join/JoinTab.less | 44 ++ .../app/entity/shared/tabs/Join/JoinTab.tsx | 55 ++ .../renderer/component/EntityNameList.tsx | 45 +- datahub-web-react/src/graphql/join.graphql | 7 + datahub-web-react/src/graphql/lineage.graphql | 23 - datahub-web-react/src/images/Arrow.svg | 3 + datahub-web-react/src/images/close_dark.svg | 3 + .../src/images/editIconBlack.svg | 3 + datahub-web-react/src/images/joinIcon.svg | 3 + 23 files changed, 1927 insertions(+), 44 deletions(-) create mode 100644 datahub-web-react/src/app/entity/joins/JoinEntity.tsx create mode 100644 datahub-web-react/src/app/entity/joins/preview/JoinAction.less create mode 100644 datahub-web-react/src/app/entity/joins/preview/JoinPreviewCard.tsx create mode 100644 datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.less create mode 100644 datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.tsx create mode 100644 datahub-web-react/src/app/entity/shared/components/styled/Join/JoinPreview.less create mode 100644 datahub-web-react/src/app/entity/shared/components/styled/Join/JoinPreview.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Relationship/RelationTab.less create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Relationship/RelationTab.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Join/JoinTab.less create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Join/JoinTab.tsx create mode 100644 datahub-web-react/src/images/Arrow.svg create mode 100644 datahub-web-react/src/images/close_dark.svg create mode 100644 datahub-web-react/src/images/editIconBlack.svg create mode 100644 datahub-web-react/src/images/joinIcon.svg diff --git a/datahub-web-react/src/App.tsx b/datahub-web-react/src/App.tsx index 7b580d3c14643a..4500d0a98c8969 100644 --- a/datahub-web-react/src/App.tsx +++ b/datahub-web-react/src/App.tsx @@ -34,6 +34,7 @@ import { ContainerEntity } from './app/entity/container/ContainerEntity'; import GlossaryNodeEntity from './app/entity/glossaryNode/GlossaryNodeEntity'; import { DataPlatformEntity } from './app/entity/dataPlatform/DataPlatformEntity'; import { DataProductEntity } from './app/entity/dataProduct/DataProductEntity'; +import { JoinEntity } from './app/entity/joins/JoinEntity'; /* Construct Apollo Client @@ -116,6 +117,7 @@ const App: React.VFC = () => { register.register(new GlossaryNodeEntity()); register.register(new DataPlatformEntity()); register.register(new DataProductEntity()); + register.register(new JoinEntity()); return register; }, []); diff --git a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx index cf9e70643aea7f..8169c2188c5913 100644 --- a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx +++ b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx @@ -30,6 +30,7 @@ import { EmbedTab } from '../shared/tabs/Embed/EmbedTab'; import EmbeddedProfile from '../shared/embed/EmbeddedProfile'; import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection'; import { getDataProduct } from '../shared/utils'; +import { RelationTab } from '../shared/tabs/Dataset/Relationship/RelationTab'; const SUBTYPES = { VIEW: 'view', @@ -96,6 +97,10 @@ export class DatasetEntity implements Entity { name: 'Schema', component: SchemaTab, }, + { + name: 'Relationships', + component: RelationTab, + }, { name: 'View Definition', component: ViewDefinitionTab, diff --git a/datahub-web-react/src/app/entity/joins/JoinEntity.tsx b/datahub-web-react/src/app/entity/joins/JoinEntity.tsx new file mode 100644 index 00000000000000..3f034b1ac6d501 --- /dev/null +++ b/datahub-web-react/src/app/entity/joins/JoinEntity.tsx @@ -0,0 +1,142 @@ +import * as React from 'react'; +import { DatabaseOutlined, DatabaseFilled } from '@ant-design/icons'; +import { EntityType, Join, OwnershipType, SearchResult } from '../../../types.generated'; +import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { getDataForEntityType } from '../shared/containers/profile/utils'; +import { GenericEntityProperties } from '../shared/types'; +import { JoinPreviewCard } from './preview/JoinPreviewCard'; +import joinIcon from '../../../images/joinIcon.svg'; +import { JoinTab } from '../shared/tabs/Join/JoinTab'; +import { useGetJoinQuery, useUpdateJoinMutation } from '../../../graphql/join.generated'; +import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'; +import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab'; +import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection'; +import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection'; +import { EntityProfile } from '../shared/containers/profile/EntityProfile'; +import './preview/JoinAction.less'; +import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection'; + +/** + * Definition of the DataHub Join entity. + */ + +export class JoinEntity implements Entity { + type: EntityType = EntityType.Join; + + icon = (fontSize: number, styleType: IconStyleType) => { + if (styleType === IconStyleType.TAB_VIEW) { + return ; + } + + if (styleType === IconStyleType.HIGHLIGHT) { + return ; + } + + if (styleType === IconStyleType.SVG) { + return ( + + ); + } + + return ; + }; + + isSearchEnabled = () => true; + + isBrowseEnabled = () => false; + + isLineageEnabled = () => false; + + getAutoCompleteFieldName = () => 'name'; + + getPathName = () => 'join'; + + getCollectionName = () => ''; + + getEntityName = () => 'Join'; + + renderProfile = (urn: string) => ( + + ); + + getOverridePropertiesFromEntity = (_join?: Join | null): GenericEntityProperties => { + return {}; + }; + + renderPreview = (_: PreviewType, data: Join) => { + return ( + <> + {data.properties?.name || data.editableProperties?.name || ''} + } + description={data?.editableProperties?.description || ''} + owners={data.ownership?.owners} + glossaryTerms={data?.glossaryTerms || undefined} + globalTags={data?.tags} + /> + + ); + }; + + renderSearch = (result: SearchResult) => { + return this.renderPreview(PreviewType.SEARCH, result.entity as Join); + }; + + displayName = (data: Join) => { + return data.properties?.name || data.editableProperties?.name || data.urn; + }; + + getGenericEntityProperties = (data: Join) => { + return getDataForEntityType({ + data, + entityType: this.type, + getOverrideProperties: this.getOverridePropertiesFromEntity, + }); + }; + + supportedCapabilities = () => { + return new Set([]); + }; +} diff --git a/datahub-web-react/src/app/entity/joins/preview/JoinAction.less b/datahub-web-react/src/app/entity/joins/preview/JoinAction.less new file mode 100644 index 00000000000000..7ac539d7a6a1e2 --- /dev/null +++ b/datahub-web-react/src/app/entity/joins/preview/JoinAction.less @@ -0,0 +1,12 @@ +@import "../../../../../node_modules/antd/dist/antd.less"; + +.joinName { + width: 385px; + height: 24px; + font-style: normal; + font-weight: 700; + font-size: 16px; + line-height: 24px; + align-items: center; + color: #262626; +} \ No newline at end of file diff --git a/datahub-web-react/src/app/entity/joins/preview/JoinPreviewCard.tsx b/datahub-web-react/src/app/entity/joins/preview/JoinPreviewCard.tsx new file mode 100644 index 00000000000000..eac2b9f2d333a0 --- /dev/null +++ b/datahub-web-react/src/app/entity/joins/preview/JoinPreviewCard.tsx @@ -0,0 +1,54 @@ +import React from 'react'; +import { Card, Collapse } from 'antd'; +import joinIcon from '../../../../images/joinIcon.svg'; +import { EntityType, Owner, GlobalTags, GlossaryTerms } from '../../../../types.generated'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; +import { IconStyleType } from '../../Entity'; + +const { Panel } = Collapse; + +export const JoinPreviewCard = ({ + urn, + name, + owners, + description, + globalTags, + glossaryTerms, +}: { + urn: string; + name: string | any; + description: string | any; + globalTags?: GlobalTags | null; + glossaryTerms?: GlossaryTerms | null; + owners?: Array | null; +}): JSX.Element => { + const entityRegistry = useEntityRegistry(); + const getJoinHeader = (): JSX.Element => { + return ( +
+ } + tags={globalTags || undefined} + glossaryTerms={glossaryTerms || undefined} + owners={owners} + type="Join" + typeIcon={entityRegistry.getIcon(EntityType.Join, 14, IconStyleType.ACCENT)} + titleSizePx={18} + /> +
+ ); + }; + + return ( + <> + + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.less b/datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.less new file mode 100644 index 00000000000000..877dcf65f801ef --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.less @@ -0,0 +1,273 @@ +@import "../../../../../../../node_modules/antd/dist/antd.less"; + +.CreateJoinModal { + .join-name { + padding: 8px 16px; + width: 948.5px !important; + height: 40px !important; + background: #FFFFFF; + border: 1px solid #D9D9D9; + box-shadow: 0px 2px 0px rgba(0, 0, 0, 0.016); + border-radius: 2px; + align-items: center; + } + .ant-select-single.ant-select-lg:not(.ant-select-customize-input) .ant-select-selector { + align-items: center; + padding: 8px 16px; + gap: 8px; + max-width: 370px; + min-width: 370px; + height: 38px; + background: #FFFFFF; + border: 1px solid #D9D9D9; + box-shadow: 0px 2px 0px rgba(0, 0, 0, 0.016); + border-radius: 2px; + } + .ant-modal-content { + box-sizing: border-box; + width: 1000px; + height: 765px; + background: #FFFFFF; + border: 1px solid #ADC0D7; + box-shadow: 0px 0px 10px rgba(0, 0, 0, 0.15); + border-radius: 8px; + left: -215px; + top: -55px; + } + .inner-div { + max-width: 950px; + min-width: 950px; + height: 640px; + overflow-y: scroll; + margin-top: -10px; + overflow-x: hidden; + } + .ant-modal-header { + padding-top: 32px; + border-bottom: 0px !important; + } + + .join-title { + width: 300px !important; + height: 22px; + font-family: 'Arial'; + font-style: normal; + font-weight: 700; + font-size: 20px; + line-height: 22px; + color: #000000; + padding-top: 4px; + } + .all-content-heading{ + width: 380px; + height: 16px; + margin-top: 16px; + margin-bottom: 8px; + font-family: 'Arial'; + font-style: normal; + font-weight: 700; + font-size: 14px; + line-height: 16px; + color: #1B2F41; + flex: none; + } + .all-table-heading{ + width: 380px; + height: 16px; + margin-bottom: 8px; + font-family: 'Arial'; + font-style: normal; + font-weight: 700; + font-size: 14px; + line-height: 16px; + color: #1B2F41; + flex: none; + } + + .field-heading{ + height: 16px; + margin-top: 32px; + margin-bottom: 8px; + font-family: 'Arial'; + font-style: normal; + font-weight: 700; + font-size: 14px; + line-height: 16px; + color: #1B2F41; + } + .all-information{ + width: 680px; + height: 24px; + font-family: 'Arial'; + font-style: normal; + font-weight: 400; + font-size: 16px; + line-height: 24px; + color: #1B2F41; + } + .instructions-list { + width: 774px; + height: 220px; + font-family: 'Arial'; + font-style: normal; + font-weight: 400; + font-size: 14px; + line-height: 150%; + color: #556573; + flex: none; + } + .ant-modal-footer { + padding-top: 0px; + padding-bottom: 10px; + padding-right: 25px; + border-top: 0px; + } + + .ant-btn-link { + padding-left: 0px !important; + padding-right: 1px !important; + font-family: 'Arial' !important; + font-style: normal !important; + font-weight: 400 !important; + font-size: 14px !important; + color: #1890FF !important; + } + + .add-btn-link { + padding-left: 865px !important; + padding-right: 8px !important; + padding-top: 16px !important; + height: 20px; + font-family: 'Arial' !important; + font-style: normal !important; + font-weight: 700 !important; + font-size: 12px !important; + color: #1890FF !important; + line-height: 20px; + } + + .cancel-btn { + box-sizing: border-box; + margin-left: 440px; + width: 85px; + height: 40px !important; + background: #FFFFFF; + border: 1px solid #D9D9D9 !important; + border-radius: 5px; + color: #262626; + } + + .submit-btn, .submit-btn:hover { + margin-left: 28px; + margin-top: 6px; + width: 86px; + height: 40px; + background: #1890FF; + border: none; + color: #FFFFFF; + } + .footer-parent-div { + padding-left: 8px; + display: flex; + } + .join-select-selector { + align-items: center; + width: 300px !important; + height: 38px !important; + border: none; + max-width: 373px !important; + min-width: 373px !important; + } + .join-details-ta { + height: 100px; + width: 720px; + font-style: normal; + font-weight: 400; + font-size: 14px; + line-height: 22px; + color: rgba(0, 0, 0, 0.85); + } + .JoinTable { + .icon-image { + box-sizing: border-box; + width: 16px; + height: 0px; + border: 1px solid #000000; + } + .ant-table-thead > tr th { + font-style: normal; + font-weight: 500; + font-size: 14px; + line-height: 22px; + color: #1B2F41; + align-items: center; + padding: 16px; + gap: 4px; + isolation: isolate; + height: 56px !important; + background: #FFFFFF; + border-color: rgba(0, 0, 0, 0.12); + } + .ant-table-tbody > tr td { + letter-spacing: 0.3px; + margin-left: 0px; + background: white; + font-style: normal; + font-weight: 400; + font-size: 14px; + line-height: 22px; + color: rgba(0, 0, 0, 0.85); + border-color: rgba(0, 0, 0, 0.12); + } + td:nth-child(1), td:nth-child(3){ + max-width: 360px !important; + } + th:nth-child(1), th:nth-child(3) { + .titleNameDisplay{ + max-width: 360px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + width: fit-content; + display: inline-block; + font-size: 14px; + padding: 4px 0; + } + .firstRow{ + display: flex; + justify-content: left; + } + + .editableNameDisplay { + display: block; + overflow-wrap: break-word; + white-space: nowrap; + max-width: 360px; + overflow: hidden; + text-overflow: ellipsis; + height: 16px; + font-family: 'Arial'; + font-style: normal; + font-weight: 400; + font-size: 14px; + line-height: 16px; + color: #595959; + } + } + td:nth-child(2), th:nth-child(2){ + min-width: 44px !important; + max-width: 44px !important; + } + td:nth-child(4), th:nth-child(4){ + min-width: 75px !important; + max-width: 75px !important; + } + table { + border-radius: 0.375rem; + border-collapse: collapse; + } + .SelectedRow { + background-color: #ECF2F8; + } + } +} \ No newline at end of file diff --git a/datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.tsx new file mode 100644 index 00000000000000..f25cbf16a78ea4 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/components/styled/Join/CreateJoinModal.tsx @@ -0,0 +1,475 @@ +import React, { useContext, useState } from 'react'; +import { Button, Form, FormInstance, Input, message, Modal, Select, Table } from 'antd'; +import TextArea from 'antd/lib/input/TextArea'; +import { PlusOutlined } from '@ant-design/icons'; +import { useApolloClient } from '@apollo/client'; +import arrow from '../../../../../../images/Arrow.svg'; +import './CreateJoinModal.less'; +import { Dataset, EntityType, Join, OwnershipType } from '../../../../../../types.generated'; +import { GetSearchResultsDocument } from '../../../../../../graphql/search.generated'; +import { useCreateJoinMutation, useUpdateJoinMutation } from '../../../../../../graphql/join.generated'; +import { useUserContext } from '../../../../../context/useUserContext'; + +type Props = { + table1?: any; + table1Schema?: any; + table2?: any; + table2Schema?: any; + visible: boolean; + setModalVisible?: any; + onCancel: () => void; + editJoin?: Join; + editFlag?: boolean; +}; +const EditableContext = React.createContext | null>(null); + +interface JoinRecord { + key: string; + field1Name: string; + field2Name: string; +} +interface JoinDataType { + key: React.Key; + field1Name: string; + field2Name: string; +} +interface EditableRowProps { + index: number; +} +const EditableRow: React.FC = ({ ...propsAt }) => { + const [form] = Form.useForm(); + return ( +
+ + + + + ); +}; +interface EditableCellProps { + editable: boolean; + children: React.ReactNode; + dataIndex: keyof JoinRecord; + record: JoinRecord; + tableRecord?: Dataset; + value?: any; + handleSave: (record: JoinRecord) => void; +} +const EditableCell = ({ + editable, + children, + dataIndex, + record, + tableRecord, + value, + handleSave, + ...restProps +}: EditableCellProps) => { + const form = useContext(EditableContext)!; + const save = async () => { + try { + const values = await form.validateFields(); + handleSave({ ...record, ...values }); + } catch (errInfo) { + console.log('Save failed:', errInfo); + } + }; + + let childNode = children; + if (editable) { + childNode = ( + + ; +}; + +type EditableTableProps = Parameters[0]; +type ColumnTypes = Exclude; + +export const CreateJoinModal = ({ + table1, + table1Schema, + table2, + table2Schema, + visible, + setModalVisible, + onCancel, + editJoin, + editFlag, +}: Props) => { + const client = useApolloClient(); + const [form] = Form.useForm(); + const { user } = useUserContext(); + const table1Dataset = editJoin?.properties?.datasetA || table1?.dataset; + const table1DatasetSchema = editJoin?.properties?.datasetA || table1Schema; + const table2Dataset = editJoin?.properties?.datasetB || table2?.dataset; + const table2DatasetSchema = editJoin?.properties?.datasetB || table2Schema?.dataset; + + const [details, setDetails] = useState(editJoin?.properties?.joinFieldMappings?.details || ''); + const [joinName, setJoinName] = useState(editJoin?.properties?.name || editJoin?.joinId || ''); + const [tableData, setTableData] = useState( + editJoin?.properties?.joinFieldMappings?.fieldMapping?.map((item, index) => { + return { + key: index, + field1Name: item.afield, + field2Name: item.bfield, + }; + }) || [ + { key: '0', field1Name: '', field2Name: '' }, + { key: '1', field1Name: '', field2Name: '' }, + ], + ); + const [count, setCount] = useState(editJoin?.properties?.joinFieldMappings?.fieldMapping?.length || 2); + const [createMutation] = useCreateJoinMutation(); + const [updateMutation] = useUpdateJoinMutation(); + const handleDelete = (record) => { + const newData = tableData.filter((item) => item.key !== record.key); + setTableData(newData); + }; + const onCancelSelect = () => { + Modal.confirm({ + title: `Exit`, + content: `Are you sure you want to exit? The changes made to the join will not be applied.`, + onOk() { + onCancel?.(); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + }; + const checkDuplicateJoin = async (name): Promise => { + const { data } = await client.query({ + query: GetSearchResultsDocument, + variables: { + input: { + type: EntityType.Join, + query: '', + orFilters: [ + { + and: [ + { + field: 'name', + values: [name], + }, + ], + }, + ], + start: 0, + count: 1000, + }, + }, + }); + return data && data.search && data.search.total > 0; + }; + const validateTableData = (fieldMappingData: JoinDataType) => { + if (fieldMappingData.field1Name !== '' && fieldMappingData.field2Name !== '') { + return true; + } + return false; + }; + const validateJoin = async (nameField: string, tableSchema: JoinDataType[]) => { + const errors: string[] = []; + const bDuplicateName = await checkDuplicateJoin(nameField?.trim()).then((result) => result); + if (nameField === '') { + errors.push('Join name is required.'); + } + if (bDuplicateName && !editFlag) { + errors.push('This join name already exists. A unique name for each join is required.'); + } + const faultyRows = tableSchema.filter((item) => validateTableData(item) !== true); + if (faultyRows.length > 0) { + errors.push('Fields should be selected'); + } + console.log(faultyRows); + return errors; + }; + const onSubmit = async () => { + const errors = validateJoin(joinName, tableData); + if ((await errors).length > 0) { + const errorHtml = (await errors).join(`
`); + message.error({ content:

}); + return; + } + if (editFlag) { + updateMutation({ + variables: { + urn: editJoin?.urn || '', + input: { + properties: { + dataSetA: table1Dataset?.urn || '', + datasetB: table2Dataset?.urn || '', + name: joinName, + createdBy: editJoin?.properties?.createdActor || '', + createdAt: editJoin?.properties?.createdTime || 0, + joinFieldmappings: { + details, + fieldMapping: tableData.map((r) => { + return { + afield: r.field1Name, + bfield: r.field2Name, + }; + }), + }, + }, + }, + }, + }); + } else { + createMutation({ + variables: { + input: { + properties: { + dataSetA: table1Dataset?.urn || '', + datasetB: table2Dataset?.urn || '', + name: joinName, + joinFieldmappings: { + details, + fieldMapping: tableData.map((r) => { + return { + afield: r.field1Name, + bfield: r.field2Name, + }; + }), + }, + created: true, + }, + ownership: { + owners: [ + { + owner: user?.urn || '', + type: OwnershipType.TechnicalOwner, + }, + ], + }, + }, + }, + }); + } + setModalVisible(false); + window.location.reload(); + }; + function getDatasetName(datainput: any): string { + return datainput?.editableProperties?.name || datainput?.properties?.name || datainput?.name || datainput?.urn; + } + const table1NameBusiness = getDatasetName(table1Dataset); + const table1NameTech = table1Dataset?.name || table1Dataset?.urn.split(',').at(1) || ''; + const table2NameBusiness = getDatasetName(table2Dataset); + const table2NameTech = table2Dataset?.name || table2Dataset?.urn.split(',').at(1) || ''; + + const handleAdd = () => { + const newData: JoinDataType = { + key: count, + field1Name: '', + field2Name: '', + }; + setTableData([...tableData, newData]); + setCount(count + 1); + }; + const defaultColumns: (ColumnTypes[number] & { editable?: boolean; dataIndex: string; tableRecord?: any })[] = [ + { + title: ( +

+

+ {table1NameBusiness || table1NameTech} +
+
{table1NameTech !== table1NameBusiness && table1NameTech}
+

+ ), + dataIndex: 'field1Name', + // width: '40%', + tableRecord: table1DatasetSchema || {}, + editable: true, + }, + { + title: '', + dataIndex: '', + // width: '8%', + editable: false, + render: () => , + }, + { + title: ( +

+

+ {table2NameBusiness || table2NameTech} +
+
{table2NameTech !== table2NameBusiness && table2NameTech}
+

+ ), + dataIndex: 'field2Name', + // width: '40%', + tableRecord: table2DatasetSchema || {}, + editable: true, + }, + { + title: 'Action', + dataIndex: '', + // width: '8%', + editable: false, + render: (record) => + tableData.length > 1 ? ( + + ) : null, + }, + ]; + const handleSave = (row: JoinDataType) => { + const newData = [...tableData]; + const index = newData.findIndex((item) => row.key === item.key); + const item = newData[index]; + newData.splice(index, 1, { + ...item, + ...row, + }); + setTableData(newData); + }; + const components = { + body: { + row: EditableRow, + cell: EditableCell, + }, + }; + + const columns = defaultColumns.map((col) => { + if (!col.editable) { + return col; + } + return { + ...col, + onCell: (record: JoinDataType) => ({ + record, + editable: col.editable, + dataIndex: col.dataIndex, + tableRecord: col.tableRecord, + title: col.title, + handleSave, + }), + }; + }); + return ( + +

Join parameters

+
+ +
+
+ +
+ + } + visible={visible} + closable={false} + className="CreateJoinModal" + okButtonProps={{ hidden: true }} + cancelButtonProps={{ hidden: true }} + onCancel={onCancelSelect} + > +
+

Table 1

+

{table1NameBusiness}

+
{table1NameTech !== table1NameBusiness && table1NameTech}
+

Table 2

+

{table2NameBusiness}

+
{table2NameTech !== table2NameBusiness && table2NameTech}
+

Join name

+
+ + checkDuplicateJoin(value?.trim()).then((result) => { + return result === true + ? Promise.reject( + new Error( + 'This join name already exists. A unique name for each join is required.', + ), + ) + : Promise.resolve(); + }), + }, + ]} + > + setJoinName(e.target.value)} + /> + +

Fields

+
{childNode}
+ +

Join details

+ +