v3 OpenAPI fixes

datahub-project · Feb 15, 2024 · dc16ed8 · dc16ed8
1 parent 2a8e317
commit dc16ed8
Show file tree

Hide file tree

Showing 19 changed files with 363 additions and 42 deletions.
diff --git a/metadata-io/src/main/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtil.java b/metadata-io/src/main/java/com/linkedin/metadata/aspect/utils/DefaultAspectsUtil.java
@@ -276,7 +276,7 @@ private static DataPlatformInfo getDataPlatformInfo(Urn urn, EntityService<?> en
     return null;
   }
 
-  private static MetadataChangeProposal getProposalFromAspect(
+  public static MetadataChangeProposal getProposalFromAspect(
       String aspectName,
       RecordTemplate aspect,
       RecordTemplate entityKeyAspect,

diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java
@@ -1114,15 +1114,13 @@ private Stream<IngestResult> ingestProposalSync(AspectsBatch aspectsBatch) {
         nonTimeseries.getMCPItems().stream()
             .filter(
                 item ->
-                    item.getMetadataChangeProposal().getChangeType() != ChangeType.PATCH
-                        && item.getMetadataChangeProposal().getChangeType() != ChangeType.UPSERT)
+                    item.getChangeType() != ChangeType.PATCH
+                        && item.getChangeType() != ChangeType.UPSERT)
             .collect(Collectors.toList());
     if (!unsupported.isEmpty()) {
       throw new UnsupportedOperationException(
           "ChangeType not supported: "
-              + unsupported.stream()
-                  .map(item -> item.getMetadataChangeProposal().getChangeType())
-                  .collect(Collectors.toSet()));
+              + unsupported.stream().map(item -> item.getChangeType()).collect(Collectors.toSet()));
     }
 
     List<UpdateAspectResult> upsertResults = ingestAspects(nonTimeseries, true, true);

diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCPUpsertBatchItem.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCPUpsertBatchItem.java
@@ -16,6 +16,7 @@
 import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator;
 import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever;
 import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException;
+import com.linkedin.metadata.aspect.utils.DefaultAspectsUtil;
 import com.linkedin.metadata.entity.EntityAspect;
 import com.linkedin.metadata.entity.EntityUtils;
 import com.linkedin.metadata.entity.validation.ValidationUtils;
@@ -90,6 +91,17 @@ public ChangeType getChangeType() {
     return ChangeType.UPSERT;
   }
 
+  @Override
+  @Nonnull
+  public MetadataChangeProposal getMetadataChangeProposal() {
+    if (metadataChangeProposal != null) {
+      return metadataChangeProposal;
+    } else {
+      return DefaultAspectsUtil.getProposalFromAspect(
+          getAspectName(), getRecordTemplate(), null, this);
+    }
+  }
+
   public void applyMutationHooks(
       @Nullable RecordTemplate oldAspectValue,
       @Nullable SystemMetadata oldSystemMetadata,

diff --git a/metadata-service/openapi-analytics-servlet/build.gradle b/metadata-service/openapi-analytics-servlet/build.gradle
@@ -58,7 +58,7 @@ task openApiGenerate(type: GenerateSwaggerCode) {
             'java11'             : "true",
             'modelPropertyNaming': "original",
             'modelPackage'       : "io.datahubproject.openapi.generated",
-            'apiPackage'         : "io.datahubproject.openapi.generated.controller",
+            'apiPackage'         : "io.datahubproject.openapi.v2.generated.controller",
             'delegatePattern'    : "true"
     ]
 }

diff --git a/...lytics-servlet/src/main/java/io/datahubproject/openapi/config/OpenapiAnalyticsConfig.java b/...lytics-servlet/src/main/java/io/datahubproject/openapi/config/OpenapiAnalyticsConfig.java
@@ -1,7 +1,7 @@
 package io.datahubproject.openapi.config;
 
 import io.datahubproject.openapi.delegates.DatahubUsageEventsImpl;
-import io.datahubproject.openapi.generated.controller.DatahubUsageEventsApiDelegate;
+import io.datahubproject.openapi.v2.generated.controller.DatahubUsageEventsApiDelegate;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 

diff --git a/...ics-servlet/src/main/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImpl.java b/...ics-servlet/src/main/java/io/datahubproject/openapi/delegates/DatahubUsageEventsImpl.java
@@ -10,7 +10,7 @@
 import com.linkedin.metadata.authorization.PoliciesConfig;
 import com.linkedin.metadata.search.elasticsearch.ElasticSearchService;
 import io.datahubproject.openapi.exception.UnauthorizedException;
-import io.datahubproject.openapi.generated.controller.DatahubUsageEventsApiDelegate;
+import io.datahubproject.openapi.v2.generated.controller.DatahubUsageEventsApiDelegate;
 import java.util.Objects;
 import java.util.Optional;
 import org.springframework.beans.factory.annotation.Autowired;

diff --git a/...ta-service/openapi-analytics-servlet/src/main/resources/JavaSpring/apiController.mustache b/...ta-service/openapi-analytics-servlet/src/main/resources/JavaSpring/apiController.mustache
@@ -62,7 +62,7 @@ import java.util.concurrent.Callable;
 {{/isJava8or11}}
 {{>generatedAnnotation}}
 @Controller
-@RequestMapping("/v1/analytics")
+@RequestMapping("/v2/analytics")
 {{#operations}}
 public class {{classname}}Controller implements {{classname}} {
 

diff --git a/metadata-service/openapi-analytics-servlet/src/main/resources/open-api.yaml b/metadata-service/openapi-analytics-servlet/src/main/resources/open-api.yaml
@@ -2,7 +2,7 @@ openapi: "3.0.0"
 info:
   title: Analytics API
   description: This is a service for DataHub Analytics.
-  version: v1
+  version: v2
 
 paths:
   /datahub_usage_events/_search:

diff --git a/...y-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java b/...y-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java
@@ -21,7 +21,6 @@
 import com.linkedin.metadata.search.SearchService;
 import io.datahubproject.openapi.dto.UpsertAspectRequest;
 import io.datahubproject.openapi.dto.UrnResponseMap;
-import io.datahubproject.openapi.entities.EntitiesController;
 import io.datahubproject.openapi.exception.UnauthorizedException;
 import io.datahubproject.openapi.generated.BrowsePathsV2AspectRequestV2;
 import io.datahubproject.openapi.generated.BrowsePathsV2AspectResponseV2;
@@ -57,6 +56,7 @@
 import io.datahubproject.openapi.generated.StatusAspectRequestV2;
 import io.datahubproject.openapi.generated.StatusAspectResponseV2;
 import io.datahubproject.openapi.util.OpenApiEntitiesUtil;
+import io.datahubproject.openapi.v1.entities.EntitiesController;
 import java.net.URISyntaxException;
 import java.util.List;
 import java.util.Map;

diff --git a/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache b/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache
@@ -3,7 +3,7 @@ package {{package}};
 import io.datahubproject.openapi.v2.delegates.EntityApiDelegateImpl;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.search.SearchService;
-import io.datahubproject.openapi.entities.EntitiesController;
+import io.datahubproject.openapi.v1.entities.EntitiesController;
 import com.datahub.authorization.AuthorizerChain;
 
 import org.springframework.web.bind.annotation.RequestMapping;

diff --git a/...ervlet/src/test/java/io/datahubproject/openapi/config/OpenAPIEntityTestConfiguration.java b/...ervlet/src/test/java/io/datahubproject/openapi/config/OpenAPIEntityTestConfiguration.java
@@ -28,10 +28,10 @@
 import com.linkedin.metadata.systemmetadata.SystemMetadataService;
 import com.linkedin.metadata.timeline.TimelineService;
 import io.datahubproject.openapi.dto.UrnResponseMap;
-import io.datahubproject.openapi.entities.EntitiesController;
 import io.datahubproject.openapi.generated.EntityResponse;
-import io.datahubproject.openapi.relationships.RelationshipsController;
-import io.datahubproject.openapi.timeline.TimelineController;
+import io.datahubproject.openapi.v1.entities.EntitiesController;
+import io.datahubproject.openapi.v1.relationships.RelationshipsController;
+import io.datahubproject.openapi.v2.controller.TimelineController;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.stream.Collectors;

diff --git a/...rvice/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java b/...rvice/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java
@@ -31,6 +31,7 @@
 public class SpringWebConfig implements WebMvcConfigurer {
   private static final Set<String> OPERATIONS_PACKAGES =
       Set.of("io.datahubproject.openapi.operations", "io.datahubproject.openapi.health");
+  private static final Set<String> V1_PACKAGES = Set.of("io.datahubproject.openapi.v1");
   private static final Set<String> V2_PACKAGES = Set.of("io.datahubproject.openapi.v2");
   private static final Set<String> V3_PACKAGES = Set.of("io.datahubproject.openapi.v3");
   private static final Set<String> SCHEMA_REGISTRY_PACKAGES =
@@ -41,8 +42,9 @@ public class SpringWebConfig implements WebMvcConfigurer {
   static {
     NONDEFAULT_OPENAPI_PACKAGES = new HashSet<>();
     NONDEFAULT_OPENAPI_PACKAGES.addAll(OPERATIONS_PACKAGES);
-    NONDEFAULT_OPENAPI_PACKAGES.addAll(V2_PACKAGES);
     NONDEFAULT_OPENAPI_PACKAGES.addAll(SCHEMA_REGISTRY_PACKAGES);
+    NONDEFAULT_OPENAPI_PACKAGES.addAll(V1_PACKAGES);
+    NONDEFAULT_OPENAPI_PACKAGES.addAll(V2_PACKAGES);
     NONDEFAULT_OPENAPI_PACKAGES.addAll(V3_PACKAGES);
   }
 

diff --git a/.../openapi/entities/EntitiesController.java → ...enapi/v1/entities/EntitiesController.java b/.../openapi/entities/EntitiesController.java → ...enapi/v1/entities/EntitiesController.java
@@ -1,4 +1,4 @@
-package io.datahubproject.openapi.entities;
+package io.datahubproject.openapi.v1.entities;
 
 import static com.linkedin.metadata.utils.PegasusUtils.urnToEntityName;
 

diff --git a/...elationships/RelationshipsController.java → ...elationships/RelationshipsController.java b/...elationships/RelationshipsController.java → ...elationships/RelationshipsController.java
@@ -1,4 +1,4 @@
-package io.datahubproject.openapi.relationships;
+package io.datahubproject.openapi.v1.relationships;
 
 import static com.linkedin.metadata.search.utils.QueryUtils.*;
 

diff --git a/...enapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java b/...enapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java
@@ -7,18 +7,24 @@
 import com.datahub.authentication.AuthenticationContext;
 import com.datahub.authorization.AuthorizerChain;
 import com.datahub.util.RecordUtils;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.ByteString;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.entity.EnvelopedAspect;
+import com.linkedin.metadata.aspect.batch.AspectsBatch;
+import com.linkedin.metadata.aspect.batch.BatchItem;
 import com.linkedin.metadata.aspect.batch.UpsertItem;
 import com.linkedin.metadata.aspect.patch.GenericJsonPatch;
 import com.linkedin.metadata.aspect.patch.template.common.GenericPatchTemplate;
 import com.linkedin.metadata.authorization.PoliciesConfig;
 import com.linkedin.metadata.entity.EntityService;
+import com.linkedin.metadata.entity.EntityUtils;
+import com.linkedin.metadata.entity.IngestResult;
 import com.linkedin.metadata.entity.UpdateAspectResult;
 import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl;
 import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem;
@@ -45,6 +51,8 @@
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -190,7 +198,9 @@ public ResponseEntity<Object> headEntity(
   public ResponseEntity<Object> getAspect(
       @PathVariable("entityName") String entityName,
       @PathVariable("entityUrn") String entityUrn,
-      @PathVariable("aspectName") String aspectName)
+      @PathVariable("aspectName") String aspectName,
+      @RequestParam(value = "systemMetadata", required = false, defaultValue = "false")
+          Boolean withSystemMetadata)
       throws URISyntaxException {
 
     if (restApiAuthorizationEnabled) {
@@ -205,9 +215,16 @@ public ResponseEntity<Object> getAspect(
     }
 
     return ResponseEntity.of(
-        toRecordTemplates(List.of(UrnUtils.getUrn(entityUrn)), Set.of(aspectName), true).stream()
+        toRecordTemplates(
+                List.of(UrnUtils.getUrn(entityUrn)), Set.of(aspectName), withSystemMetadata)
+            .stream()
             .findFirst()
-            .flatMap(e -> e.getAspects().values().stream().findFirst()));
+            .flatMap(
+                e ->
+                    e.getAspects().entrySet().stream()
+                        .filter(entry -> entry.getKey().equals(aspectName))
+                        .map(Map.Entry::getValue)
+                        .findFirst()));
   }
 
   @Tag(name = "Generic Aspects")
@@ -257,6 +274,42 @@ public void deleteEntity(
     entityService.deleteAspect(entityUrn, entitySpec.getKeyAspectName(), Map.of(), true);
   }
 
+  @Tag(name = "Generic Entities")
+  @PostMapping(value = "/{entityName}", produces = MediaType.APPLICATION_JSON_VALUE)
+  @Operation(summary = "Create a batch of entities.")
+  public ResponseEntity<List<GenericEntity>> createEntity(
+      @PathVariable("entityName") String entityName,
+      @RequestParam(value = "async", required = false, defaultValue = "true") Boolean async,
+      @RequestParam(value = "systemMetadata", required = false, defaultValue = "false")
+          Boolean withSystemMetadata,
+      @RequestBody @Nonnull String jsonEntityList)
+      throws URISyntaxException, JsonProcessingException {
+
+    EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName);
+    Authentication authentication = AuthenticationContext.getAuthentication();
+
+    AspectsBatch batch = toBatch(jsonEntityList, authentication.getActor());
+
+    if (restApiAuthorizationEnabled) {
+      for (BatchItem item : batch.getItems()) {
+        checkAuthorized(
+            authorizationChain,
+            authentication.getActor(),
+            entitySpec,
+            item.getUrn().toString(),
+            ImmutableList.of(PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType()));
+      }
+    }
+
+    Set<IngestResult> results = entityService.ingestProposal(batch, async);
+
+    if (!async) {
+      return ResponseEntity.ok(toEntityListResponse(results, withSystemMetadata));
+    } else {
+      return ResponseEntity.accepted().body(toEntityListResponse(results, withSystemMetadata));
+    }
+  }
+
   @Tag(name = "Generic Aspects")
   @DeleteMapping(value = "/{entityName}/{entityUrn}/{aspectName}")
   @Operation(summary = "Delete an entity aspect.")
@@ -504,4 +557,75 @@ private UpsertItem toUpsertItem(
         AuditStampUtils.createAuditStamp(actor.toUrnStr()),
         entityService);
   }
+
+  private AspectsBatch toBatch(String entityArrayList, Actor actor)
+      throws JsonProcessingException, URISyntaxException {
+    JsonNode entities = objectMapper.readTree(entityArrayList);
+
+    List<BatchItem> items = new LinkedList<>();
+    if (entities.isArray()) {
+      Iterator<JsonNode> entityItr = entities.iterator();
+      while (entityItr.hasNext()) {
+        JsonNode entity = entityItr.next();
+        Urn entityUrn = UrnUtils.getUrn(entity.get("urn").asText());
+
+        Iterator<Map.Entry<String, JsonNode>> aspectItr = entity.get("aspects").fields();
+        while (aspectItr.hasNext()) {
+          Map.Entry<String, JsonNode> aspect = aspectItr.next();
+
+          AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey());
+
+          if (aspectSpec != null) {
+            MCPUpsertBatchItem.MCPUpsertBatchItemBuilder builder =
+                MCPUpsertBatchItem.builder()
+                    .urn(entityUrn)
+                    .aspectName(aspectSpec.getName())
+                    .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr()))
+                    .recordTemplate(
+                        GenericRecordUtils.deserializeAspect(
+                            ByteString.copyString(
+                                objectMapper.writeValueAsString(aspect.getValue().get("value")),
+                                StandardCharsets.UTF_8),
+                            GenericRecordUtils.JSON,
+                            aspectSpec));
+
+            if (aspect.getValue().has("systemMetadata")) {
+              builder.systemMetadata(
+                  EntityUtils.parseSystemMetadata(
+                      objectMapper.writeValueAsString(aspect.getValue().get("systemMetadata"))));
+            }
+
+            items.add(builder.build(entityService));
+          }
+        }
+      }
+    }
+
+    return AspectsBatchImpl.builder().items(items).build();
+  }
+
+  public List<GenericEntity> toEntityListResponse(
+      Set<IngestResult> ingestResults, boolean withSystemMetadata) {
+    List<GenericEntity> responseList = new LinkedList<>();
+
+    Map<Urn, List<IngestResult>> entityMap =
+        ingestResults.stream().collect(Collectors.groupingBy(IngestResult::getUrn));
+    for (Map.Entry<Urn, List<IngestResult>> urnAspects : entityMap.entrySet()) {
+      Map<String, Pair<RecordTemplate, SystemMetadata>> aspectsMap =
+          urnAspects.getValue().stream()
+              .map(
+                  ingest ->
+                      Map.entry(
+                          ingest.getRequest().getAspectName(),
+                          Pair.of(
+                              ingest.getRequest().getRecordTemplate(),
+                              withSystemMetadata ? ingest.getRequest().getSystemMetadata() : null)))
+              .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+      responseList.add(
+          GenericEntity.builder()
+              .urn(urnAspects.getKey().toString())
+              .build(objectMapper, aspectsMap));
+    }
+    return responseList;
+  }
 }
diff --git a/.../entities/PlatformEntitiesController.java → ...ontroller/PlatformEntitiesController.java b/.../entities/PlatformEntitiesController.java → ...ontroller/PlatformEntitiesController.java
@@ -1,4 +1,4 @@
-package io.datahubproject.openapi.platform.entities;
+package io.datahubproject.openapi.v2.controller;
 
 import com.datahub.authentication.Authentication;
 import com.datahub.authentication.AuthenticationContext;
@@ -38,7 +38,7 @@
 
 @RestController
 @RequiredArgsConstructor
-@RequestMapping("/platform/entities/v1")
+@RequestMapping("/v2/platform/entities/v1")
 @Slf4j
 @Tag(
     name = "Platform Entities",

diff --git a/.../openapi/timeline/TimelineController.java → ...api/v2/controller/TimelineController.java b/.../openapi/timeline/TimelineController.java → ...api/v2/controller/TimelineController.java
@@ -1,4 +1,4 @@
-package io.datahubproject.openapi.timeline;
+package io.datahubproject.openapi.v2.controller;
 
 import com.datahub.authentication.Authentication;
 import com.datahub.authentication.AuthenticationContext;
@@ -32,7 +32,7 @@
 
 @RestController
 @AllArgsConstructor
-@RequestMapping("/timeline/v1")
+@RequestMapping("/v2/timeline/v1")
 @Tag(
     name = "Timeline",
     description =