feat(openapi-v3): entity-registry openapi spec (#9550)

Co-authored-by: Ajoy Majumdar <[email protected]>

Adds support for custom aspects in the openapi api

build.gradle

@@ -172,6 +172,7 @@ project.ext.externalDependency = [
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.9',
     'json': 'org.json:json:20231013',
+    'jsonSchemaValidator': 'com.github.java-json-tools:json-schema-validator:2.2.14',
     'junit': 'junit:junit:4.13.2',
     'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",

buildSrc/build.gradle

@@ -14,10 +14,11 @@ dependencies {
    * Other companies are also separately maintaining forks (like: https://github.com/java-json-tools/json-schema-avro).
    * We have built several customizations on top of it for various bug fixes, especially around union scheams
    */
-  implementation('io.acryl:json-schema-avro:0.2.2') {
+  implementation('io.acryl:json-schema-avro:0.2.3') {
     exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
     exclude group: 'com.google.guava', module: 'guava'
   }
+  implementation 'com.github.java-json-tools:json-schema-validator:2.2.14'
   implementation 'com.google.guava:guava:32.1.2-jre'
   implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.5'
   implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5'

docs/dev-guides/timeline.md

@@ -81,7 +81,7 @@ Took 1.077 seconds to hard delete 6 rows for 1 entities
 Update succeeded with status 200
 Update succeeded with status 200
 Update succeeded with status 200
-http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=TECHNICAL_SCHEMA&start=1644874316591&end=2682397800000
+http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=TECHNICAL_SCHEMA&start=1644874316591&end=2682397800000
 2022-02-24 15:31:53 - 0.0.0-computed
 	ADD TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:property_id): A forwards & backwards compatible change due to the newly added field 'property_id'.
 	ADD TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:service): A forwards & backwards compatible change due to the newly added field 'service'.
@@ -115,7 +115,7 @@ Took 1.087 seconds to hard delete 6 rows for 1 entities
 Update succeeded with status 200
 Update succeeded with status 200
 Update succeeded with status 200
-http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=OWNER&start=1644874829027&end=2682397800000
+http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=OWNER&start=1644874829027&end=2682397800000
 2022-02-24 15:40:26 - 0.0.0-computed
 	ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:datahub): A new owner 'datahub' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added.
 	ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:jdoe): A new owner 'jdoe' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added.
@@ -127,7 +127,7 @@ http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3Adat
 Update succeeded with status 200
 Update succeeded with status 200
 Update succeeded with status 200
-http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=OWNER&start=1644874831456&end=2682397800000
+http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=OWNER&start=1644874831456&end=2682397800000
 2022-02-24 15:40:26 - 0.0.0-computed
 	ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:datahub): A new owner 'datahub' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added.
 	ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:jdoe): A new owner 'jdoe' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added.
@@ -161,7 +161,7 @@ Took 0.626 seconds to hard delete 9 rows for 1 entities
 Update succeeded with status 200
 Update succeeded with status 200
 Update succeeded with status 200
-http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=TAG&start=1644875047911&end=2682397800000
+http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=TAG&start=1644875047911&end=2682397800000
 2022-02-24 15:44:05 - 0.0.0-computed
 	ADD TAG dataset:hive:testTimelineDataset (urn:li:tag:Legacy): A new tag 'Legacy' for the entity 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added.
 2022-02-24 15:44:06 - 0.1.0-computed
@@ -189,7 +189,7 @@ Took 0.578 seconds to hard delete 6 rows for 1 entities
 Update succeeded with status 200
 Update succeeded with status 200
 Update succeeded with status 200
-http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=DOCUMENTATION&start=1644875157616&end=2682397800000
+http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=DOCUMENTATION&start=1644875157616&end=2682397800000
 2022-02-24 15:45:55 - 0.0.0-computed
 	ADD DOCUMENTATION dataset:hive:testTimelineDataset (https://www.linkedin.com): The institutionalMemory 'https://www.linkedin.com' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added.
 2022-02-24 15:45:56 - 0.1.0-computed
@@ -218,7 +218,7 @@ Took 0.443 seconds to hard delete 6 rows for 1 entities
 Update succeeded with status 200
 Update succeeded with status 200
 Update succeeded with status 200
-http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=GLOSSARY_TERM&start=1644875100605&end=2682397800000
+http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=GLOSSARY_TERM&start=1644875100605&end=2682397800000
 1969-12-31 18:00:00 - 0.0.0-computed
 	None None  : java.lang.NullPointerException:null
 2022-02-24 15:44:58 - 0.1.0-computed

entity-registry/build.gradle

@@ -15,7 +15,7 @@ dependencies {
   implementation externalDependency.jacksonDataBind
   implementation externalDependency.jacksonDataFormatYaml
   implementation externalDependency.reflections
-  implementation externalDependency.jsonPatch
+  api externalDependency.jsonPatch
   constraints {
     implementation(externalDependency.snakeYaml) {
       because("previous versions are vulnerable to CVE-2022-25857")

entity-registry/src/test/java/com/linkedin/metadata/models/OpenApiSpecBuilderTest.java

@@ -1,6 +1,6 @@
 package com.linkedin.metadata.models;
 
-import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
 
 import com.datahub.test.TestEntityProfile;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -42,7 +42,7 @@
 
 @SuppressWarnings({"rawtypes", "unchecked"})
 public class OpenApiSpecBuilderTest {
-  private static final String MODEL_VERSION = "_test";
+  private static final String MODEL_VERSION = "_v3";
   private static final String TYPE_OBJECT = "object";
   private static final String TYPE_BOOLEAN = "boolean";
   private static final String TYPE_STRING = "string";
@@ -98,7 +98,7 @@ public void testOpenApiSpecBuilder() throws Exception {
         new ConfigEntityRegistry(
             TestEntityProfile.class.getClassLoader().getResourceAsStream("entity-registry.yml"));
     MergedEntityRegistry er = new MergedEntityRegistry(configEntityRegistry);
-    new PluginEntityRegistryLoader(TestConstants.BASE_DIRECTORY, 60)
+    new PluginEntityRegistryLoader(TestConstants.BASE_DIRECTORY, 1)
         .withBaseRegistry(er)
         .start(true);
 
@@ -108,9 +108,9 @@ public void testOpenApiSpecBuilder() throws Exception {
         Path.of(getClass().getResource("/").getPath(), "open-api.yaml"),
         openapiYaml.getBytes(StandardCharsets.UTF_8));
 
-    assertEquals(openAPI.getComponents().getSchemas().size(), 930);
-    assertEquals(openAPI.getComponents().getParameters().size(), 57);
-    assertEquals(openAPI.getPaths().size(), 104);
+    assertTrue(openAPI.getComponents().getSchemas().size() >= 882);
+    assertTrue(openAPI.getComponents().getParameters().size() >= 54);
+    assertTrue(openAPI.getPaths().size() >= 98);
   }
 
   private OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) {

metadata-ingestion/src/datahub/cli/timeline_cli.py

@@ -79,7 +79,7 @@ def get_timeline(
     diff_param: str = f"&raw={diff}" if diff else ""
     endpoint: str = (
         host
-        + f"/openapi/timeline/v1/{encoded_urn}?categories={categories}{start_time_param}{end_time_param}{diff_param}"
+        + f"/openapi/v2/timeline/v1/{encoded_urn}?categories={categories}{start_time_param}{end_time_param}{diff_param}"
     )
     click.echo(endpoint)
 

metadata-io/src/main/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtil.java

@@ -332,7 +332,7 @@ private static DataPlatformInfo getDataPlatformInfo(
     return null;
   }
 
-  private static MetadataChangeProposal getProposalFromAspectForDefault(
+  public static MetadataChangeProposal getProposalFromAspectForDefault(
       String aspectName,
       RecordTemplate aspect,
       RecordTemplate entityKeyAspect,

metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java

@@ -1166,9 +1166,7 @@ private Stream<IngestResult> ingestProposalSync(
     if (!unsupported.isEmpty()) {
       throw new UnsupportedOperationException(
           "ChangeType not supported: "
-              + unsupported.stream()
-                  .map(item -> item.getMetadataChangeProposal().getChangeType())
-                  .collect(Collectors.toSet()));
+              + unsupported.stream().map(item -> item.getChangeType()).collect(Collectors.toSet()));
     }
 
     List<UpdateAspectResult> upsertResults = ingestAspects(opContext, nonTimeseries, true, true);

metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java

@@ -308,7 +308,7 @@ public static <T extends RecordTemplate> MetadataChangeProposal buildMCP(
     return proposal;
   }
 
-  static SystemMetadata parseSystemMetadata(String jsonSystemMetadata) {
+  public static SystemMetadata parseSystemMetadata(String jsonSystemMetadata) {
     if (jsonSystemMetadata == null || jsonSystemMetadata.equals("")) {
       SystemMetadata response = new SystemMetadata();
       response.setRunId(DEFAULT_RUN_ID);

metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java

@@ -100,6 +100,18 @@ public SystemAspect getSystemAspect(@Nullable Long version) {
         .build(getEntitySpec(), getAspectSpec(), entityAspect);
   }
 
+  @Nonnull
+  public MetadataChangeProposal getMetadataChangeProposal() {
+    final MetadataChangeProposal mcp = new MetadataChangeProposal();
+    mcp.setEntityUrn(getUrn());
+    mcp.setChangeType(getChangeType());
+    mcp.setEntityType(getEntitySpec().getName());
+    mcp.setAspectName(getAspectName());
+    mcp.setAspect(GenericRecordUtils.serializeAspect(getRecordTemplate()));
+    mcp.setSystemMetadata(getSystemMetadata());
+    return mcp;
+  }
+
   public static class ChangeItemImplBuilder {
 
     // Ensure use of other builders

metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java

@@ -727,6 +727,7 @@ public void testAsyncProposalVersioned() throws Exception {
     gmce.setChangeType(ChangeType.UPSERT);
     gmce.setEntityType("dataset");
     gmce.setAspectName("datasetProperties");
+    gmce.setSystemMetadata(new SystemMetadata());
     JacksonDataTemplateCodec dataTemplateCodec = new JacksonDataTemplateCodec();
     byte[] datasetPropertiesSerialized = dataTemplateCodec.dataTemplateToBytes(datasetProperties);
     GenericAspect genericAspect = new GenericAspect();

metadata-service/openapi-analytics-servlet/build.gradle

@@ -58,7 +58,7 @@ task openApiGenerate(type: GenerateSwaggerCode) {
             'java11'             : "true",
             'modelPropertyNaming': "original",
             'modelPackage'       : "io.datahubproject.openapi.generated",
-            'apiPackage'         : "io.datahubproject.openapi.generated.controller",
+            'apiPackage'         : "io.datahubproject.openapi.v2.generated.controller",
             'delegatePattern'    : "true"
     ]
 }

metadata-service/openapi-analytics-servlet/src/main/java/io/datahubproject/openapi/config/OpenapiAnalyticsConfig.java

@@ -1,7 +1,7 @@
 package io.datahubproject.openapi.config;
 
 import io.datahubproject.openapi.delegates.DatahubUsageEventsImpl;
-import io.datahubproject.openapi.generated.controller.DatahubUsageEventsApiDelegate;
+import io.datahubproject.openapi.v2.generated.controller.DatahubUsageEventsApiDelegate;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 

metadata-service/openapi-analytics-servlet/src/main/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImpl.java

@@ -11,7 +11,7 @@
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.metadata.context.RequestContext;
 import io.datahubproject.openapi.exception.UnauthorizedException;
-import io.datahubproject.openapi.generated.controller.DatahubUsageEventsApiDelegate;
+import io.datahubproject.openapi.v2.generated.controller.DatahubUsageEventsApiDelegate;
 import java.util.List;
 import java.util.Objects;
 import org.springframework.beans.factory.annotation.Autowired;

metadata-service/openapi-analytics-servlet/src/main/resources/JavaSpring/apiController.mustache

@@ -62,7 +62,7 @@ import java.util.concurrent.Callable;
 {{/isJava8or11}}
 {{>generatedAnnotation}}
 @Controller
-@RequestMapping("/v1/analytics")
+@RequestMapping("/v2/analytics")
 {{#operations}}
 public class {{classname}}Controller implements {{classname}} {
 

metadata-service/openapi-analytics-servlet/src/main/resources/open-api.yaml

@@ -2,7 +2,7 @@ openapi: "3.0.0"
 info:
   title: Analytics API
   description: This is a service for DataHub Analytics.
-  version: v1
+  version: v2
 
 paths:
   /datahub_usage_events/_search:

metadata-service/openapi-analytics-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIAnalyticsTestConfiguration.java

@@ -11,6 +11,7 @@
 import com.datahub.authentication.AuthenticationContext;
 import com.datahub.authorization.AuthorizationResult;
 import com.datahub.authorization.AuthorizerChain;
+import com.linkedin.metadata.models.registry.*;
 import com.linkedin.metadata.search.elasticsearch.ElasticSearchService;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.test.metadata.context.TestOperationContexts;
@@ -53,4 +54,13 @@ public AuthorizerChain authorizerChain() {
 
     return authorizerChain;
   }
+
+  @Bean("entityRegistry")
+  @Primary
+  public EntityRegistry entityRegistry() throws EntityRegistryException, InterruptedException {
+    return new ConfigEntityRegistry(
+        OpenAPIAnalyticsTestConfiguration.class
+            .getClassLoader()
+            .getResourceAsStream("entity-registry.yml"));
+  }
 }

metadata-service/openapi-analytics-servlet/src/test/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImplTest.java

@@ -5,7 +5,7 @@
 
 import io.datahubproject.openapi.config.OpenAPIAnalyticsTestConfiguration;
 import io.datahubproject.openapi.config.SpringWebConfig;
-import io.datahubproject.openapi.generated.controller.DatahubUsageEventsApiController;
+import io.datahubproject.openapi.v2.generated.controller.DatahubUsageEventsApiController;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.context.annotation.ComponentScan;
@@ -16,7 +16,7 @@
 import org.testng.annotations.Test;
 
 @SpringBootTest(classes = {SpringWebConfig.class})
-@ComponentScan(basePackages = {"io.datahubproject.openapi.generated.controller"})
+@ComponentScan(basePackages = {"io.datahubproject.openapi.v2.generated.controller"})
 @Import({DatahubUsageEventsImpl.class, OpenAPIAnalyticsTestConfiguration.class})
 public class DatahubUsageEventsImplTest extends AbstractTestNGSpringContextTests {
 

metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java

@@ -21,7 +21,6 @@
 import io.datahubproject.metadata.context.RequestContext;
 import io.datahubproject.openapi.dto.UpsertAspectRequest;
 import io.datahubproject.openapi.dto.UrnResponseMap;
-import io.datahubproject.openapi.entities.EntitiesController;
 import io.datahubproject.openapi.exception.UnauthorizedException;
 import io.datahubproject.openapi.generated.BrowsePathsV2AspectRequestV2;
 import io.datahubproject.openapi.generated.BrowsePathsV2AspectResponseV2;
@@ -59,6 +58,7 @@
 import io.datahubproject.openapi.generated.StatusAspectRequestV2;
 import io.datahubproject.openapi.generated.StatusAspectResponseV2;
 import io.datahubproject.openapi.util.OpenApiEntitiesUtil;
+import io.datahubproject.openapi.v1.entities.EntitiesController;
 import java.net.URISyntaxException;
 import java.util.List;
 import java.util.Map;

metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache

@@ -4,7 +4,7 @@ import io.datahubproject.openapi.v2.delegates.EntityApiDelegateImpl;
 import io.datahubproject.metadata.context.OperationContext;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.search.SearchService;
-import io.datahubproject.openapi.entities.EntitiesController;
+import io.datahubproject.openapi.v1.entities.EntitiesController;
 import com.datahub.authorization.AuthorizerChain;
 
 import org.springframework.web.bind.annotation.RequestMapping;

metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/config/OpenAPIEntityTestConfiguration.java

@@ -28,10 +28,10 @@
 import com.linkedin.metadata.timeline.TimelineService;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.openapi.dto.UrnResponseMap;
-import io.datahubproject.openapi.entities.EntitiesController;
 import io.datahubproject.openapi.generated.EntityResponse;
-import io.datahubproject.openapi.relationships.RelationshipsController;
-import io.datahubproject.openapi.timeline.TimelineController;
+import io.datahubproject.openapi.v1.entities.EntitiesController;
+import io.datahubproject.openapi.v1.relationships.RelationshipsController;
+import io.datahubproject.openapi.v2.controller.TimelineController;
 import io.datahubproject.test.metadata.context.TestOperationContexts;
 import java.util.Arrays;
 import java.util.Map;

metadata-service/openapi-servlet/build.gradle

@@ -26,6 +26,9 @@ dependencies {
   implementation externalDependency.antlr4Runtime
   implementation externalDependency.antlr4
   implementation externalDependency.swaggerAnnotations
+  implementation externalDependency.guava
+  implementation('io.acryl:json-schema-avro:0.2.3')
+  implementation externalDependency.jsonSchemaValidator
 
   annotationProcessor externalDependency.lombok