misc(updates): rebased and fixes

* json schema validator update * tests fixes, timeline, counts
datahub-project · Apr 11, 2024 · be9d7b0 · be9d7b0
1 parent 99525b2
commit be9d7b0
Show file tree

Hide file tree

Showing 13 changed files with 31 additions and 33 deletions.
diff --git a/build.gradle b/build.gradle
@@ -169,6 +169,7 @@ project.ext.externalDependency = [
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.9',
     'json': 'org.json:json:20231013',
+    'jsonSchemaValidator': 'com.github.java-json-tools:json-schema-validator:2.2.14',
     'junit': 'junit:junit:4.13.2',
     'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -14,10 +14,11 @@ dependencies {
    * Other companies are also separately maintaining forks (like: https://github.com/java-json-tools/json-schema-avro).
    * We have built several customizations on top of it for various bug fixes, especially around union scheams
    */
-  implementation('io.acryl:json-schema-avro:0.2.2') {
+  implementation('io.acryl:json-schema-avro:0.2.3') {
     exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
     exclude group: 'com.google.guava', module: 'guava'
   }
+  implementation 'com.github.java-json-tools:json-schema-validator:2.2.14'
   implementation 'com.google.guava:guava:32.1.2-jre'
   implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.5'
   implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5'

diff --git a/docs/dev-guides/timeline.md b/docs/dev-guides/timeline.md
@@ -81,7 +81,7 @@ Took 1.077 seconds to hard delete 6 rows for 1 entities
 Update succeeded with status 200
 Update succeeded with status 200
 Update succeeded with status 200
-http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=TECHNICAL_SCHEMA&start=1644874316591&end=2682397800000
+http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=TECHNICAL_SCHEMA&start=1644874316591&end=2682397800000
 2022-02-24 15:31:53 - 0.0.0-computed
 	ADD TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:property_id): A forwards & backwards compatible change due to the newly added field 'property_id'.
 	ADD TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:service): A forwards & backwards compatible change due to the newly added field 'service'.
@@ -115,7 +115,7 @@ Took 1.087 seconds to hard delete 6 rows for 1 entities
 Update succeeded with status 200
 Update succeeded with status 200
 Update succeeded with status 200
-http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=OWNER&start=1644874829027&end=2682397800000
+http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=OWNER&start=1644874829027&end=2682397800000
 2022-02-24 15:40:26 - 0.0.0-computed
 	ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:datahub): A new owner 'datahub' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added.
 	ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:jdoe): A new owner 'jdoe' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added.
@@ -127,7 +127,7 @@ http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3Adat
 Update succeeded with status 200
 Update succeeded with status 200
 Update succeeded with status 200
-http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=OWNER&start=1644874831456&end=2682397800000
+http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=OWNER&start=1644874831456&end=2682397800000
 2022-02-24 15:40:26 - 0.0.0-computed
 	ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:datahub): A new owner 'datahub' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added.
 	ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:jdoe): A new owner 'jdoe' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added.
@@ -161,7 +161,7 @@ Took 0.626 seconds to hard delete 9 rows for 1 entities
 Update succeeded with status 200
 Update succeeded with status 200
 Update succeeded with status 200
-http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=TAG&start=1644875047911&end=2682397800000
+http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=TAG&start=1644875047911&end=2682397800000
 2022-02-24 15:44:05 - 0.0.0-computed
 	ADD TAG dataset:hive:testTimelineDataset (urn:li:tag:Legacy): A new tag 'Legacy' for the entity 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added.
 2022-02-24 15:44:06 - 0.1.0-computed
@@ -189,7 +189,7 @@ Took 0.578 seconds to hard delete 6 rows for 1 entities
 Update succeeded with status 200
 Update succeeded with status 200
 Update succeeded with status 200
-http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=DOCUMENTATION&start=1644875157616&end=2682397800000
+http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=DOCUMENTATION&start=1644875157616&end=2682397800000
 2022-02-24 15:45:55 - 0.0.0-computed
 	ADD DOCUMENTATION dataset:hive:testTimelineDataset (https://www.linkedin.com): The institutionalMemory 'https://www.linkedin.com' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added.
 2022-02-24 15:45:56 - 0.1.0-computed
@@ -218,7 +218,7 @@ Took 0.443 seconds to hard delete 6 rows for 1 entities
 Update succeeded with status 200
 Update succeeded with status 200
 Update succeeded with status 200
-http://localhost:8080/openapi/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=GLOSSARY_TERM&start=1644875100605&end=2682397800000
+http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=GLOSSARY_TERM&start=1644875100605&end=2682397800000
 1969-12-31 18:00:00 - 0.0.0-computed
 	None None  : java.lang.NullPointerException:null
 2022-02-24 15:44:58 - 0.1.0-computed

diff --git a/entity-registry/src/test/java/com/linkedin/metadata/models/OpenApiSpecBuilderTest.java b/entity-registry/src/test/java/com/linkedin/metadata/models/OpenApiSpecBuilderTest.java
@@ -98,7 +98,9 @@ public void testOpenApiSpecBuilder() throws Exception {
         new ConfigEntityRegistry(
             TestEntityProfile.class.getClassLoader().getResourceAsStream("entity-registry.yml"));
     MergedEntityRegistry er = new MergedEntityRegistry(configEntityRegistry);
-    new PluginEntityRegistryLoader(TestConstants.BASE_DIRECTORY).withBaseRegistry(er).start(true);
+    new PluginEntityRegistryLoader(TestConstants.BASE_DIRECTORY, 1)
+        .withBaseRegistry(er)
+        .start(true);
 
     OpenAPI openAPI = generateOpenApiSpec(er);
     String openapiYaml = Yaml.pretty(openAPI);

diff --git a/metadata-ingestion/src/datahub/cli/timeline_cli.py b/metadata-ingestion/src/datahub/cli/timeline_cli.py
@@ -79,7 +79,7 @@ def get_timeline(
     diff_param: str = f"&raw={diff}" if diff else ""
     endpoint: str = (
         host
-        + f"/openapi/timeline/v1/{encoded_urn}?categories={categories}{start_time_param}{end_time_param}{diff_param}"
+        + f"/openapi/v2/timeline/v1/{encoded_urn}?categories={categories}{start_time_param}{end_time_param}{diff_param}"
     )
     click.echo(endpoint)
 

diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java
@@ -313,7 +313,7 @@ public static <T extends RecordTemplate> MetadataChangeProposal buildMCP(
     return proposal;
   }
 
-  static SystemMetadata parseSystemMetadata(String jsonSystemMetadata) {
+  public static SystemMetadata parseSystemMetadata(String jsonSystemMetadata) {
     if (jsonSystemMetadata == null || jsonSystemMetadata.equals("")) {
       SystemMetadata response = new SystemMetadata();
       response.setRunId(DEFAULT_RUN_ID);

diff --git a/...servlet/src/test/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImplTest.java b/...servlet/src/test/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImplTest.java
@@ -6,7 +6,7 @@
 import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
 import io.datahubproject.openapi.config.OpenAPIAnalyticsTestConfiguration;
 import io.datahubproject.openapi.config.SpringWebConfig;
-import io.datahubproject.openapi.generated.controller.DatahubUsageEventsApiController;
+import io.datahubproject.openapi.v2.generated.controller.DatahubUsageEventsApiController;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.context.annotation.ComponentScan;
@@ -18,7 +18,7 @@
 import org.testng.annotations.Test;
 
 @SpringBootTest(classes = {SpringWebConfig.class})
-@ComponentScan(basePackages = {"io.datahubproject.openapi.generated.controller"})
+@ComponentScan(basePackages = {"io.datahubproject.openapi.v2.generated.controller"})
 @Import({DatahubUsageEventsImpl.class, OpenAPIAnalyticsTestConfiguration.class})
 public class DatahubUsageEventsImplTest extends AbstractTestNGSpringContextTests {
   @BeforeTest

diff --git a/metadata-service/openapi-servlet/build.gradle b/metadata-service/openapi-servlet/build.gradle
@@ -27,7 +27,8 @@ dependencies {
   implementation externalDependency.antlr4
   implementation externalDependency.swaggerAnnotations
   implementation externalDependency.guava
-  implementation 'io.acryl:json-schema-avro:0.2.2'
+  implementation('io.acryl:json-schema-avro:0.2.3')
+  implementation externalDependency.jsonSchemaValidator
 
   annotationProcessor externalDependency.lombok
 

diff --git a/...enapi/v1/entities/EntitiesController.java → ...pi/v1/v1/entities/EntitiesController.java b/...enapi/v1/entities/EntitiesController.java → ...pi/v1/v1/entities/EntitiesController.java
diff --git a/...elationships/RelationshipsController.java → ...elationships/RelationshipsController.java b/...elationships/RelationshipsController.java → ...elationships/RelationshipsController.java
diff --git a/...enapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java b/...enapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java
@@ -21,6 +21,8 @@
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.events.metadata.ChangeType;
+import com.linkedin.metadata.aspect.batch.AspectsBatch;
+import com.linkedin.metadata.aspect.batch.BatchItem;
 import com.linkedin.metadata.aspect.batch.ChangeMCP;
 import com.linkedin.metadata.aspect.patch.GenericJsonPatch;
 import com.linkedin.metadata.aspect.patch.template.common.GenericPatchTemplate;
@@ -255,9 +257,7 @@ public ResponseEntity<Object> getAspect(
     }
 
     return ResponseEntity.of(
-        toRecordTemplates(
-                List.of(urn), Set.of(aspectName), withSystemMetadata)
-            .stream()
+        toRecordTemplates(List.of(urn), Set.of(aspectName), withSystemMetadata).stream()
             .findFirst()
             .flatMap(
                 e ->
@@ -319,20 +319,13 @@ public ResponseEntity<List<GenericEntity>> createEntity(
       @RequestBody @Nonnull String jsonEntityList)
       throws URISyntaxException, JsonProcessingException {
 
-    EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName);
     Authentication authentication = AuthenticationContext.getAuthentication();
-
     AspectsBatch batch = toBatch(jsonEntityList, authentication.getActor());
 
-    if (restApiAuthorizationEnabled) {
-      for (BatchItem item : batch.getItems()) {
-        checkAuthorized(
-            authorizationChain,
-            authentication.getActor(),
-            entitySpec,
-            item.getUrn().toString(),
-            ImmutableList.of(PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType()));
-      }
+    if (!AuthUtil.isAPIAuthorizedEntityType(
+        authentication, authorizationChain, CREATE, entityName)) {
+      throw new UnauthorizedException(
+          authentication.getActor().toUrnStr() + " is unauthorized to " + CREATE + " entities.");
     }
 
     Set<IngestResult> results = entityService.ingestProposal(batch, async);
@@ -620,8 +613,8 @@ private AspectsBatch toBatch(String entityArrayList, Actor actor)
           AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey());
 
           if (aspectSpec != null) {
-            MCPUpsertBatchItem.MCPUpsertBatchItemBuilder builder =
-                MCPUpsertBatchItem.builder()
+            ChangeItemImpl.ChangeItemImplBuilder builder =
+                ChangeItemImpl.builder()
                     .urn(entityUrn)
                     .aspectName(aspectSpec.getName())
                     .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr()))

diff --git a/...api/v2/controller/TimelineController.java → ...roller/controller/TimelineController.java b/...api/v2/controller/TimelineController.java → ...roller/controller/TimelineController.java
diff --git a/...ce/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/OpenAPIV3GeneratorTest.java b/...ce/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/OpenAPIV3GeneratorTest.java
@@ -1,6 +1,6 @@
 package io.datahubproject.openapi.v3;
 
-import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
 
 import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
 import com.linkedin.metadata.models.registry.ConfigEntityRegistry;
@@ -37,8 +37,8 @@ public void testOpenApiSpecBuilder() throws Exception {
         Path.of(getClass().getResource("/").getPath(), "open-api.yaml"),
         openapiYaml.getBytes(StandardCharsets.UTF_8));
 
-    assertEquals(openAPI.getComponents().getSchemas().size(), 818);
-    assertEquals(openAPI.getComponents().getParameters().size(), 50);
-    assertEquals(openAPI.getPaths().size(), 418);
+    assertTrue(openAPI.getComponents().getSchemas().size() > 900);
+    assertTrue(openAPI.getComponents().getParameters().size() > 50);
+    assertTrue(openAPI.getPaths().size() > 500);
   }
 }