[BREAKING] Break dependency of ebean-dao on metadata-models. (#1895)

The coupling was between the static path extractor API. This broken by making a new `UrnPathExtractor` interface, and adding an overload of `EbeanLocalDAO`'s constructor to accept one (no breaking constructor change). The old constructors default to an `EmptyPathExtractor`, which does nothing (which is a breaking behavioral change, see below).

BREAKING: `DatasetUrnPathExtractor` was deleted. No one should've been depending on this directly. However, downstreams that were relying on it being there at runtime (dataset GMS) need to copy `DatasetUrnPathExtractor` and create their `EbeanLocalDAO` with one. Note that this is a little dangerous becasue it is a runtime behavioral change only. Potential impact is that SCSI suddenly stops working as intended.

SYNC=metadata-models_101.0.0

metadata-dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java

@@ -1,5 +1,6 @@
 package com.linkedin.metadata.dao;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.template.RecordTemplate;
@@ -9,6 +10,8 @@
 import com.linkedin.metadata.dao.producer.BaseMetadataEventProducer;
 import com.linkedin.metadata.dao.retention.TimeBasedRetention;
 import com.linkedin.metadata.dao.retention.VersionBasedRetention;
+import com.linkedin.metadata.dao.scsi.EmptyPathExtractor;
+import com.linkedin.metadata.dao.scsi.UrnPathExtractor;
 import com.linkedin.metadata.dao.storage.LocalDAOStorageConfig;
 import com.linkedin.metadata.dao.utils.ModelUtils;
 import com.linkedin.metadata.dao.utils.RecordUtils;
@@ -49,7 +52,6 @@
 import lombok.Value;
 
 import static com.linkedin.metadata.dao.EbeanMetadataAspect.*;
-import static com.linkedin.metadata.dao.utils.RegisteredUrnPathExtractors.*;
 
 
 /**
@@ -63,6 +65,7 @@ public class EbeanLocalDAO<ASPECT_UNION extends UnionTemplate, URN extends Urn>
 
   protected final EbeanServer _server;
   protected final Class<URN> _urnClass;
+  private UrnPathExtractor<URN> _urnPathExtractor;
 
   private static final int INDEX_QUERY_TIMEOUT_IN_SEC = 5;
 
@@ -83,6 +86,15 @@ static class GMAIndexPair {
         }
       });
 
+  @VisibleForTesting
+  EbeanLocalDAO(@Nonnull Class<ASPECT_UNION> aspectUnionClass, @Nonnull BaseMetadataEventProducer producer,
+      @Nonnull EbeanServer server, @Nonnull Class<URN> urnClass) {
+    super(aspectUnionClass, producer);
+    _server = server;
+    _urnClass = urnClass;
+    _urnPathExtractor = new EmptyPathExtractor<>();
+  }
+
   /**
    * Constructor for EbeanLocalDAO.
    *
@@ -93,9 +105,23 @@ static class GMAIndexPair {
    */
   public EbeanLocalDAO(@Nonnull Class<ASPECT_UNION> aspectUnionClass, @Nonnull BaseMetadataEventProducer producer,
       @Nonnull ServerConfig serverConfig, @Nonnull Class<URN> urnClass) {
-    super(aspectUnionClass, producer);
-    _server = createServer(serverConfig);
+    this(aspectUnionClass, producer, createServer(serverConfig), urnClass);
+  }
+
+  @VisibleForTesting
+  EbeanLocalDAO(@Nonnull BaseMetadataEventProducer producer, @Nonnull EbeanServer server,
+      @Nonnull LocalDAOStorageConfig storageConfig, @Nonnull Class<URN> urnClass,
+      @Nonnull UrnPathExtractor<URN> urnPathExtractor) {
+    super(producer, storageConfig);
+    _server = server;
     _urnClass = urnClass;
+    _urnPathExtractor = urnPathExtractor;
+  }
+
+  @VisibleForTesting
+  EbeanLocalDAO(@Nonnull BaseMetadataEventProducer producer, @Nonnull EbeanServer server,
+      @Nonnull LocalDAOStorageConfig storageConfig, @Nonnull Class<URN> urnClass) {
+    this(producer, server, storageConfig, urnClass, new EmptyPathExtractor<>());
   }
 
   /**
@@ -104,17 +130,30 @@ public EbeanLocalDAO(@Nonnull Class<ASPECT_UNION> aspectUnionClass, @Nonnull Bas
    * @param producer {@link BaseMetadataEventProducer} for the metadata event producer
    * @param serverConfig {@link ServerConfig} that defines the configuration of EbeanServer instances
    * @param storageConfig {@link LocalDAOStorageConfig} containing storage config of full list of supported aspects
-   * @param urnClass Class of the entity URN
+   * @param urnClass class of the entity URN
+   * @param urnPathExtractor path extractor to index parts of URNs to the secondary index
+   */
+  public EbeanLocalDAO(@Nonnull BaseMetadataEventProducer producer, @Nonnull ServerConfig serverConfig,
+      @Nonnull LocalDAOStorageConfig storageConfig, @Nonnull Class<URN> urnClass,
+      @Nonnull UrnPathExtractor<URN> urnPathExtractor) {
+    this(producer, createServer(serverConfig), storageConfig, urnClass, urnPathExtractor);
+  }
+
+  /**
+   * Constructor for EbeanLocalDAO.
+   *
+   * @param producer {@link BaseMetadataEventProducer} for the metadata event producer
+   * @param serverConfig {@link ServerConfig} that defines the configuration of EbeanServer instances
+   * @param storageConfig {@link LocalDAOStorageConfig} containing storage config of full list of supported aspects
+   * @param urnClass class of the entity URN
    */
   public EbeanLocalDAO(@Nonnull BaseMetadataEventProducer producer, @Nonnull ServerConfig serverConfig,
       @Nonnull LocalDAOStorageConfig storageConfig, @Nonnull Class<URN> urnClass) {
-    super(producer, storageConfig);
-    _server = createServer(serverConfig);
-    _urnClass = urnClass;
+    this(producer, createServer(serverConfig), storageConfig, urnClass, new EmptyPathExtractor<>());
   }
 
   @Nonnull
-  private EbeanServer createServer(@Nonnull ServerConfig serverConfig) {
+  private static EbeanServer createServer(@Nonnull ServerConfig serverConfig) {
     // Make sure that the serverConfig includes the package that contains DAO's Ebean model.
     if (!serverConfig.getPackages().contains(EBEAN_MODEL_PACKAGE)) {
       serverConfig.getPackages().add(EBEAN_MODEL_PACKAGE);
@@ -125,29 +164,17 @@ private EbeanServer createServer(@Nonnull ServerConfig serverConfig) {
     return EbeanServerFactory.create(serverConfig);
   }
 
-  // For testing purpose
-  EbeanLocalDAO(@Nonnull Class<ASPECT_UNION> aspectUnionClass, @Nonnull BaseMetadataEventProducer producer,
-      @Nonnull EbeanServer server, @Nonnull Class<URN> urnClass) {
-    super(aspectUnionClass, producer);
-    _server = server;
-    _urnClass = urnClass;
-  }
-
-  // For testing purpose
-  EbeanLocalDAO(@Nonnull BaseMetadataEventProducer producer, @Nonnull EbeanServer server, @Nonnull LocalDAOStorageConfig storageConfig,
-      @Nonnull Class<URN> urnClass) {
-    super(producer, storageConfig);
-    _server = server;
-    _urnClass = urnClass;
-  }
-
   /**
    * Return the {@link EbeanServer} server instance used for customized queries.
    */
   public EbeanServer getServer() {
     return _server;
   }
 
+  public void setUrnPathExtractor(@Nonnull UrnPathExtractor<URN> urnPathExtractor) {
+    _urnPathExtractor = urnPathExtractor;
+  }
+
   /**
    * Creates a private in-memory {@link EbeanServer} based on H2 for production.
    */
@@ -232,6 +259,9 @@ protected <ASPECT extends RecordTemplate> long saveLatest(@Nonnull URN urn, @Non
   @Override
   protected <ASPECT extends RecordTemplate> void updateLocalIndex(@Nonnull URN urn,
       @Nonnull ASPECT newValue, long version) {
+    if (!isLocalSecondaryIndexEnabled()) {
+      throw new UnsupportedOperationException("Local secondary index isn't supported");
+    }
 
     // Process and save URN
     // Only do this with the first version of each aspect
@@ -307,7 +337,7 @@ private void updateUrnInLocalIndex(@Nonnull URN urn) {
       return;
     }
 
-    final Map<String, Object> pathValueMap = getUrnPathExtractor(urn.getClass()).extractPaths(urn);
+    final Map<String, Object> pathValueMap = _urnPathExtractor.extractPaths(urn);
     pathValueMap.forEach(
         (path, value) -> saveSingleRecordToLocalIndex(urn, urn.getClass().getCanonicalName(), path, value)
     );

metadata-dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/scsi/EmptyPathExtractor.java

@@ -0,0 +1,18 @@
+package com.linkedin.metadata.dao.scsi;
+
+import com.linkedin.common.urn.Urn;
+import java.util.Collections;
+import java.util.Map;
+import javax.annotation.Nonnull;
+
+
+/**
+ * A path extractor which does nothing.
+ */
+public final class EmptyPathExtractor<URN extends Urn> implements UrnPathExtractor<URN> {
+  @Nonnull
+  @Override
+  public Map<String, Object> extractPaths(@Nonnull URN urn) {
+    return Collections.emptyMap();
+  }
+}

metadata-dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/scsi/UrnPathExtractor.java

@@ -0,0 +1,29 @@
+package com.linkedin.metadata.dao.scsi;
+
+import com.linkedin.common.urn.Urn;
+import java.util.Map;
+import javax.annotation.Nonnull;
+
+
+/**
+ * Given an URN, extracts a map containing parts of the urn to values to index in the secondary index.
+ *
+ * <p>This should map all urn parts and values you may wish to query in the secondary index. Nested values can also
+ * be extracted.
+ *
+ * <p>For example, if dataset URN is like {@code urn:li:dataset:(urn:li:platform:%PLATFORM_NAME%),%NAME%,%ORIGIN%)} then
+ * an implementation that parses datset URNs may return a map like:
+ *
+ * <pre>
+ *   /platform -> urn:li:platform:%PLATFORM_NAME%
+ *   /platform/platformName -> %PLATFORM_NAME%
+ *   /datasetName -> %NAME%
+ *   /origin -> %ORIGIN%
+ * </pre>
+ *
+ * @param <URN> the concrete URN type this can extract paths from
+ */
+public interface UrnPathExtractor<URN extends Urn> {
+  @Nonnull
+  Map<String, Object> extractPaths(@Nonnull URN urn);
+}