Merge branch 'master' into Sigma-Connector-Integration

.github/actions/docker-custom-build-and-push/action.yml

@@ -20,7 +20,7 @@ inputs:
     required: false
 
   images:
-    # e.g. linkedin/datahub-gms
+    # e.g. acryldata/datahub-gms
     description: "List of Docker images to use as base name for tags"
     required: true
   build-args:
@@ -55,7 +55,7 @@ runs:
 
     # Code for testing the build when not pushing to Docker Hub.
     - name: Build and Load image for testing (if not publishing)
-      uses: docker/build-push-action@v3
+      uses: docker/build-push-action@v5
       if: ${{ inputs.publish != 'true' }}
       with:
         context: ${{ inputs.context }}
@@ -87,19 +87,19 @@ runs:
 
     # Code for building multi-platform images and pushing to Docker Hub.
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
       if: ${{ inputs.publish == 'true' }}
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
       if: ${{ inputs.publish == 'true' }}
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       if: ${{ inputs.publish == 'true' }}
       with:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
     - name: Build and Push Multi-Platform image
-      uses: docker/build-push-action@v3
+      uses: docker/build-push-action@v5
       if: ${{ inputs.publish == 'true' }}
       with:
         context: ${{ inputs.context }}

README.md

@@ -27,7 +27,7 @@ HOSTED_DOCS_ONLY-->
 [![Version](https://img.shields.io/github/v/release/datahub-project/datahub?include_prereleases)](https://github.com/datahub-project/datahub/releases/latest)
 [![PyPI version](https://badge.fury.io/py/acryl-datahub.svg)](https://badge.fury.io/py/acryl-datahub)
 [![build & test](https://github.com/datahub-project/datahub/workflows/build%20&%20test/badge.svg?branch=master&event=push)](https://github.com/datahub-project/datahub/actions?query=workflow%3A%22build+%26+test%22+branch%3Amaster+event%3Apush)
-[![Docker Pulls](https://img.shields.io/docker/pulls/linkedin/datahub-gms.svg)](https://hub.docker.com/r/linkedin/datahub-gms)
+[![Docker Pulls](https://img.shields.io/docker/pulls/acryldata/datahub-gms.svg)](https://hub.docker.com/r/acryldata/datahub-gms)
 [![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://slack.datahubproject.io)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md)
 [![GitHub commit activity](https://img.shields.io/github/commit-activity/m/datahub-project/datahub)](https://github.com/datahub-project/datahub/pulls?q=is%3Apr)

build.gradle

@@ -55,7 +55,7 @@ buildscript {
   ext.openLineageVersion = '1.5.0'
   ext.logbackClassicJava8 = '1.2.12'
 
-  ext.docker_registry = 'linkedin'
+  ext.docker_registry = 'acryldata'
 
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
@@ -247,6 +247,7 @@ project.ext.externalDependency = [
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
     'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
+    'swaggerCore': 'io.swagger.core.v3:swagger-core:2.2.7',
     'springBootAutoconfigureJdk11': 'org.springframework.boot:spring-boot-autoconfigure:2.7.18',
     'testng': 'org.testng:testng:7.8.0',
     'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion,
@@ -265,7 +266,7 @@ project.ext.externalDependency = [
     'jline':'jline:jline:1.4.1',
     'jetbrains':' org.jetbrains.kotlin:kotlin-stdlib:1.6.0',
     'annotationApi': 'javax.annotation:javax.annotation-api:1.3.2',
-    'classGraph': 'io.github.classgraph:classgraph:4.8.165',
+    'classGraph': 'io.github.classgraph:classgraph:4.8.168',
 ]
 
 allprojects {

datahub-frontend/run/run-local-frontend

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 CURRENT_DIR=$(pwd)
-BUILD_DIR=../build/stage/playBinary
+BUILD_DIR=../build/stage/main
 CONF_DIR=$BUILD_DIR/conf
 
 set -a

datahub-frontend/run/run-local-frontend-debug

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 CURRENT_DIR=$(pwd)
-BUILD_DIR=../build/stage/playBinary
+BUILD_DIR=../build/stage/main
 CONF_DIR=$BUILD_DIR/conf
 
 set -a

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GraphQLEngine.java

@@ -87,6 +87,7 @@ private GraphQLEngine(
 
   public ExecutionResult execute(
       @Nonnull final String query,
+      @Nullable final String operationName,
       @Nullable final Map<String, Object> variables,
       @Nonnull final QueryContext context) {
     /*
@@ -100,6 +101,7 @@ public ExecutionResult execute(
     ExecutionInput executionInput =
         ExecutionInput.newExecutionInput()
             .query(query)
+            .operationName(operationName)
             .variables(variables)
             .dataLoaderRegistry(register)
             .context(context)

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java

@@ -77,10 +77,7 @@ public CompletableFuture<SearchResults> get(final DataFetchingEnvironment enviro
                   .getFilters()
                   .forEach(
                       filter -> {
-                        criteria.add(
-                            new Criterion()
-                                .setField(filter.getField())
-                                .setValue(filter.getValue()));
+                        criteria.add(criterionFromFilter(filter, true));
                       });
             }
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java

@@ -1,8 +1,10 @@
 package com.linkedin.datahub.graphql.resolvers.load;
 
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*;
+import static com.linkedin.datahub.graphql.types.mappers.MapperUtils.*;
 
 import com.datahub.authorization.AuthorizationConfiguration;
+import com.linkedin.common.UrnArrayArray;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.template.SetMode;
@@ -156,6 +158,11 @@ private LineageRelationship mapEntityRelationship(
       result.setUpdatedActor(UrnToEntityMapper.map(context, updatedActor));
     }
     result.setIsManual(lineageRelationship.hasIsManual() && lineageRelationship.isIsManual());
+    if (lineageRelationship.getPaths() != null) {
+      UrnArrayArray paths = lineageRelationship.getPaths();
+      result.setPaths(
+          paths.stream().map(path -> mapPath(context, path)).collect(Collectors.toList()));
+    }
 
     return result;
   }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java

@@ -90,7 +90,8 @@ private SearchUtils() {}
           EntityType.CORP_USER,
           EntityType.CORP_GROUP,
           EntityType.NOTEBOOK,
-          EntityType.DATA_PRODUCT);
+          EntityType.DATA_PRODUCT,
+          EntityType.DOMAIN);
 
   /** Entities that are part of browse by default */
   public static final List<EntityType> BROWSE_ENTITY_TYPES =

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java

@@ -3,9 +3,11 @@
 import static com.linkedin.datahub.graphql.util.SearchInsightsUtil.*;
 import static com.linkedin.metadata.utils.SearchUtil.*;
 
+import com.linkedin.common.UrnArray;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.AggregationMetadata;
+import com.linkedin.datahub.graphql.generated.EntityPath;
 import com.linkedin.datahub.graphql.generated.FacetMetadata;
 import com.linkedin.datahub.graphql.generated.MatchedField;
 import com.linkedin.datahub.graphql.generated.SearchResult;
@@ -104,4 +106,11 @@ public static SearchSuggestion mapSearchSuggestion(
     return new SearchSuggestion(
         suggestion.getText(), suggestion.getScore(), Math.toIntExact(suggestion.getFrequency()));
   }
+
+  public static EntityPath mapPath(@Nullable final QueryContext context, UrnArray path) {
+    EntityPath entityPath = new EntityPath();
+    entityPath.setPath(
+        path.stream().map(p -> UrnToEntityMapper.map(context, p)).collect(Collectors.toList()));
+    return entityPath;
+  }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/UrnSearchAcrossLineageResultsMapper.java

@@ -3,11 +3,9 @@
 import static com.linkedin.datahub.graphql.types.mappers.MapperUtils.*;
 import static com.linkedin.datahub.graphql.util.SearchInsightsUtil.*;
 
-import com.linkedin.common.UrnArray;
 import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.Entity;
-import com.linkedin.datahub.graphql.generated.EntityPath;
 import com.linkedin.datahub.graphql.generated.FreshnessStats;
 import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResult;
 import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResults;
@@ -16,6 +14,7 @@
 import com.linkedin.metadata.search.LineageSearchEntity;
 import com.linkedin.metadata.search.LineageSearchResult;
 import com.linkedin.metadata.search.SearchResultMetadata;
+import java.util.ArrayList;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
@@ -69,14 +68,9 @@ private SearchAcrossLineageResult mapResult(
                 .map(p -> mapPath(context, p))
                 .collect(Collectors.toList()))
         .setDegree(searchEntity.getDegree())
-        .setDegrees(searchEntity.getDegrees().stream().collect(Collectors.toList()))
+        .setDegrees(new ArrayList<>(searchEntity.getDegrees()))
+        .setExplored(Boolean.TRUE.equals(searchEntity.isExplored()))
+        .setIgnoredAsHop(Boolean.TRUE.equals(searchEntity.isIgnoredAsHop()))
         .build();
   }
-
-  private EntityPath mapPath(@Nullable final QueryContext context, UrnArray path) {
-    EntityPath entityPath = new EntityPath();
-    entityPath.setPath(
-        path.stream().map(p -> UrnToEntityMapper.map(context, p)).collect(Collectors.toList()));
-    return entityPath;
-  }
 }

datahub-graphql-core/src/main/resources/entity.graphql

@@ -1331,6 +1331,10 @@ type LineageRelationship {
     """
     isManual: Boolean
 
+    """
+    The paths traversed for this relationship
+    """
+    paths: [EntityPath]
 }
 
 """

datahub-graphql-core/src/main/resources/search.graphql

@@ -644,7 +644,7 @@ type ScrollResults {
 }
 
 """
-Results returned by issueing a search across relationships query
+Results returned by issuing a search across relationships query
 """
 type SearchAcrossLineageResults {
   """
@@ -679,7 +679,7 @@ type SearchAcrossLineageResults {
 }
 
 """
-Results returned by issueing a search across relationships query using scroll API
+Results returned by issuing a search across relationships query using scroll API
 """
 type ScrollAcrossLineageResults {
   """
@@ -742,6 +742,16 @@ type SearchAcrossLineageResult {
   """
   degrees: [Int!]
 
+  """
+  Marks whether or not this entity was explored further for lineage
+  """
+  explored: Boolean!
+
+  """
+  Whether this relationship was ignored as a hop
+  """
+  ignoredAsHop: Boolean!
+
 }
 
 """

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/ReindexDataJobViaNodesCLLConfig.java

@@ -2,6 +2,7 @@
 
 import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
 import com.linkedin.datahub.upgrade.system.vianodes.ReindexDataJobViaNodesCLL;
+import com.linkedin.metadata.entity.AspectDao;
 import com.linkedin.metadata.entity.EntityService;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
@@ -14,9 +15,13 @@ public class ReindexDataJobViaNodesCLLConfig {
 
   @Bean
   public NonBlockingSystemUpgrade reindexDataJobViaNodesCLL(
-      EntityService<?> entityService,
+      final EntityService<?> entityService,
+      final AspectDao aspectDao,
       @Value("${systemUpdate.dataJobNodeCLL.enabled}") final boolean enabled,
-      @Value("${systemUpdate.dataJobNodeCLL.batchSize}") final Integer batchSize) {
-    return new ReindexDataJobViaNodesCLL(entityService, enabled, batchSize);
+      @Value("${systemUpdate.dataJobNodeCLL.batchSize}") final Integer batchSize,
+      @Value("${systemUpdate.dataJobNodeCLL.delayMs}") final Integer delayMs,
+      @Value("${systemUpdate.dataJobNodeCLL.limit}") final Integer limit) {
+    return new ReindexDataJobViaNodesCLL(
+        entityService, aspectDao, enabled, batchSize, delayMs, limit);
   }
 }

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java

@@ -48,7 +48,7 @@ public KafkaJob(UpgradeContext context, RestoreIndicesArgs args) {
 
     @Override
     public RestoreIndicesResult call() {
-      return _entityService.restoreIndices(args, context.report()::addLine);
+      return _entityService.streamRestoreIndices(args, context.report()::addLine).findFirst().get();
     }
   }
 
@@ -85,7 +85,10 @@ private List<RestoreIndicesResult> iterateFutures(List<Future<RestoreIndicesResu
   private RestoreIndicesArgs getArgs(UpgradeContext context) {
     RestoreIndicesArgs result = new RestoreIndicesArgs();
     result.batchSize = getBatchSize(context.parsedArgs());
+    // this class assumes batch size == limit
+    result.limit = getBatchSize(context.parsedArgs());
     context.report().addLine(String.format("batchSize is %d", result.batchSize));
+    context.report().addLine(String.format("limit is %d", result.limit));
     result.numThreads = getThreadCount(context.parsedArgs());
     context.report().addLine(String.format("numThreads is %d", result.numThreads));
     result.batchDelayMs = getBatchDelayMs(context.parsedArgs());

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/vianodes/ReindexDataJobViaNodesCLL.java

@@ -3,6 +3,7 @@
 import com.google.common.collect.ImmutableList;
 import com.linkedin.datahub.upgrade.UpgradeStep;
 import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
+import com.linkedin.metadata.entity.AspectDao;
 import com.linkedin.metadata.entity.EntityService;
 import java.util.List;
 import lombok.extern.slf4j.Slf4j;
@@ -17,9 +18,17 @@ public class ReindexDataJobViaNodesCLL implements NonBlockingSystemUpgrade {
   private final List<UpgradeStep> _steps;
 
   public ReindexDataJobViaNodesCLL(
-      EntityService<?> entityService, boolean enabled, Integer batchSize) {
+      EntityService<?> entityService,
+      AspectDao aspectDao,
+      boolean enabled,
+      Integer batchSize,
+      Integer batchDelayMs,
+      Integer limit) {
     if (enabled) {
-      _steps = ImmutableList.of(new ReindexDataJobViaNodesCLLStep(entityService, batchSize));
+      _steps =
+          ImmutableList.of(
+              new ReindexDataJobViaNodesCLLStep(
+                  entityService, aspectDao, batchSize, batchDelayMs, limit));
     } else {
       _steps = ImmutableList.of();
     }