Merge branch 'master' into cr-iceberg-docs-spark

.github/workflows/docker-unified.yml

@@ -1274,6 +1274,11 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
   deploy_datahub_head:
     name: Deploy to Datahub HEAD
     runs-on: ubuntu-latest

datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx

@@ -103,6 +103,20 @@ export const LinkList = ({ refetch }: LinkListProps) => {
         }
     };
 
+    const onConfirmDelete = (link) => {
+        Modal.confirm({
+            title: `Delete Link '${link?.description}'`,
+            content: `Are you sure you want to remove this Link?`,
+            onOk() {
+                handleDeleteLink(link);
+            },
+            onCancel() {},
+            okText: 'Yes',
+            maskClosable: true,
+            closable: true,
+        });
+    };
+
     return entityData ? (
         <>
             <Modal
@@ -162,7 +176,7 @@ export const LinkList = ({ refetch }: LinkListProps) => {
                                     <Button onClick={() => handleEditLink(link)} type="text" shape="circle">
                                         <EditOutlined />
                                     </Button>
-                                    <Button onClick={() => handleDeleteLink(link)} type="text" shape="circle" danger>
+                                    <Button onClick={() => onConfirmDelete(link)} type="text" shape="circle" danger>
                                         <DeleteOutlined />
                                     </Button>
                                 </>

docs-website/adoptionStoriesIndexes.json

@@ -70,6 +70,17 @@
       "imageUrl": "/img/logos/companies/snap.png",
       "imageSize": "small"
     },
+    {
+      "name": "Slack",
+      "slug": "slack",
+      "imageUrl": "/img/logos/companies/slack.png",
+      "imageSize": "large",
+      "link": "https://youtu.be/G5B0W03dvuU",
+      "linkType": "video",
+      "tagline": "Why Slack chose DataHub to solve lineage and discovery.",
+      "category": "B2B & B2C",
+      "description": "Slack adopted DataHub to track their entire data landscape, build lineage, and add rich context to metadata for search & discovery."
+    },
     {
       "name": "Airtel",
       "slug": "airtel",
@@ -161,6 +172,18 @@
       "category": "Financial & Fintech",
       "description": "Discover how Checkout leverage DataHub for advanced data management and compliance, especially in managing sensitive data types."
     },
+    {
+      "name": "Etsy",
+      "slug": "etsy",
+      "imageUrl": "/img/logos/companies/etsy.png",
+      "imageSize": "medium",
+      "link": "https://youtu.be/kLe_xfTR_rM",
+      "linkType": "video",
+      "tagline": "Why Etsy used DataHub to solve their Data Discovery needs.",
+      "category": "E-Commerce",
+      "platform": "cloud",
+      "description": "Etsy leverages DataHub to solve their data discovery needs, enabling their data teams to find, understand, and trust their data."
+    },
     {
       "name": "MYOB",
       "slug": "myob",

docs-website/docusaurus.config.js

@@ -77,7 +77,7 @@ module.exports = {
     announcementBar: {
           id: "announcement-3",
           content:
-            '<div style="display: flex; justify-content: center; align-items: center;width: 100%;"><!--img src="/img/acryl-logo-white-mark.svg" / --><!--div style="font-size: .8rem; font-weight: 600; background-color: white; color: #111; padding: 0px 8px; border-radius: 4px; margin-right:12px;">NEW</div--><p>Learn about DataHub 1.0 launching at our 5th birthday party!</p><a href="https://lu.ma/0j5jcocn" target="_blank" class="button">Register<span> →</span></a></div>',
+            '<div style="display: flex; justify-content: center; align-items: center;width: 100%;"><!--img src="/img/acryl-logo-white-mark.svg" / --><!--div style="font-size: .8rem; font-weight: 600; background-color: white; color: #111; padding: 0px 8px; border-radius: 4px; margin-right:12px;">NEW</div--><p>Learn about DataHub 1.0</p><a href="https://youtu.be/B3IA6cLaKEk" target="_blank" class="button">Watch Now<span> →</span></a></div>',
           backgroundColor: "#111",
           textColor: "#ffffff",
           isCloseable: false,

docs-website/src/pages/_components/CaseStudy/caseStudyContent.js

@@ -50,14 +50,14 @@ const caseStudyData = [
     link: "https://www.youtube.com/watch?v=YoxTg8tQSwg",
   },
   {
-    title: "Reliable Data Products",
+    title: "Powering Discovery in Slack's data ecosystem",
     description:
-      "How Miro leverages DataHub Cloud to deliver reliable data products.",
+      "Learn why DataHub was the obvious choice for Slack to solve their lineage and discovery needs.",
     tag: "Technology",
     backgroundImage:
-      "https://miro.com/blog/wp-content/uploads/2024/08/header-diagramming-s4-02.png",
-    image: "/img/logos/companies/miro.png",
-    link: "https://miro.com/careers/life-at-miro/tech/data-products-reliability-the-power-of-metadata/",
+      "https://i.pcmag.com/imagery/reviews/07td46ju7p6lLVb0QGwc5VF-19.fit_lim.size_1050x.png",
+    image: "/img/logos/companies/slack.png",
+    link: "https://datahubproject.io/adoption-stories/#slack",
   },
   {
     title: "Working with Petabyte Scale Healthcare Data",
@@ -89,6 +89,16 @@ const caseStudyData = [
     image: "/img/logos/companies/zynga.png",
     link: "https://datahubproject.io/adoption-stories/#zynga",
   },
+  {
+    title: "Reliable Data Products",
+    description:
+      "How Miro leverages DataHub Cloud to deliver reliable data products.",
+    tag: "Technology",
+    backgroundImage:
+      "https://miro.com/blog/wp-content/uploads/2024/08/header-diagramming-s4-02.png",
+    image: "/img/logos/companies/miro.png",
+    link: "https://miro.com/careers/life-at-miro/tech/data-products-reliability-the-power-of-metadata/",
+  },
   {
     title: "And many more...",
     description:

docs/cli.md

@@ -736,7 +736,7 @@ Please see our [Integrations page](https://datahubproject.io/integrations) if yo
 | [datahub-lineage-file](./generated/ingestion/sources/file-based-lineage.md)                    | _no additional dependencies_                               | Lineage File source                     |
 | [datahub-business-glossary](./generated/ingestion/sources/business-glossary.md)                | _no additional dependencies_                               | Business Glossary File source           |
 | [dbt](./generated/ingestion/sources/dbt.md)                                                    | _no additional dependencies_                               | dbt source                              |
-| [dremio](./generated/ingestion/sources/dremio.md)                                              | `pip install 'acryl-datahub[dremio]'`                      | Dremio Source                          |
+| [dremio](./generated/ingestion/sources/dremio.md)                                              | `pip install 'acryl-datahub[dremio]'`                      | Dremio Source                           |
 | [druid](./generated/ingestion/sources/druid.md)                                                | `pip install 'acryl-datahub[druid]'`                       | Druid Source                            |
 | [feast](./generated/ingestion/sources/feast.md)                                                | `pip install 'acryl-datahub[feast]'`                       | Feast source (0.26.0)                   |
 | [glue](./generated/ingestion/sources/glue.md)                                                  | `pip install 'acryl-datahub[glue]'`                        | AWS Glue source                         |
@@ -759,6 +759,7 @@ Please see our [Integrations page](https://datahubproject.io/integrations) if yo
 | [redash](./generated/ingestion/sources/redash.md)                                              | `pip install 'acryl-datahub[redash]'`                      | Redash source                           |
 | [redshift](./generated/ingestion/sources/redshift.md)                                          | `pip install 'acryl-datahub[redshift]'`                    | Redshift source                         |
 | [sagemaker](./generated/ingestion/sources/sagemaker.md)                                        | `pip install 'acryl-datahub[sagemaker]'`                   | AWS SageMaker source                    |
+| [salesforce](./generated/ingestion/sources/salesforce.md)                                      | `pip install 'acryl-datahub[salesforce]'`                  | Salesforce source                       |
 | [snowflake](./generated/ingestion/sources/snowflake.md)                                        | `pip install 'acryl-datahub[snowflake]'`                   | Snowflake source                        |
 | [sqlalchemy](./generated/ingestion/sources/sqlalchemy.md)                                      | `pip install 'acryl-datahub[sqlalchemy]'`                  | Generic SQLAlchemy source               |
 | [superset](./generated/ingestion/sources/superset.md)                                          | `pip install 'acryl-datahub[superset]'`                    | Superset source                         |

entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectValidationException.java

@@ -18,53 +18,47 @@ public static AspectValidationException forItem(BatchItem item, String msg) {
   }
 
   public static AspectValidationException forItem(BatchItem item, String msg, Exception e) {
-    return new AspectValidationException(item, msg, SubType.VALIDATION, e);
+    return new AspectValidationException(item, msg, ValidationSubType.VALIDATION, e);
   }
 
   public static AspectValidationException forPrecondition(BatchItem item, String msg) {
     return forPrecondition(item, msg, null);
   }
 
   public static AspectValidationException forFilter(BatchItem item, String msg) {
-    return new AspectValidationException(item, msg, SubType.FILTER);
+    return new AspectValidationException(item, msg, ValidationSubType.FILTER);
   }
 
   public static AspectValidationException forPrecondition(BatchItem item, String msg, Exception e) {
-    return new AspectValidationException(item, msg, SubType.PRECONDITION, e);
+    return new AspectValidationException(item, msg, ValidationSubType.PRECONDITION, e);
   }
 
   @Nonnull BatchItem item;
   @Nonnull ChangeType changeType;
   @Nonnull Urn entityUrn;
   @Nonnull String aspectName;
-  @Nonnull SubType subType;
+  @Nonnull ValidationSubType subType;
   @Nullable String msg;
 
-  public AspectValidationException(@Nonnull BatchItem item, String msg, SubType subType) {
+  public AspectValidationException(@Nonnull BatchItem item, String msg, ValidationSubType subType) {
     this(item, msg, subType, null);
   }
 
   public AspectValidationException(
-      @Nonnull BatchItem item, @Nonnull String msg, @Nullable SubType subType, Exception e) {
+      @Nonnull BatchItem item,
+      @Nonnull String msg,
+      @Nullable ValidationSubType subType,
+      Exception e) {
     super(msg, e);
     this.item = item;
     this.changeType = item.getChangeType();
     this.entityUrn = item.getUrn();
     this.aspectName = item.getAspectName();
     this.msg = msg;
-    this.subType = subType != null ? subType : SubType.VALIDATION;
+    this.subType = subType != null ? subType : ValidationSubType.VALIDATION;
   }
 
   public Pair<Urn, String> getAspectGroup() {
     return Pair.of(entityUrn, aspectName);
   }
-
-  public enum SubType {
-    // A validation exception is thrown
-    VALIDATION,
-    // A failed precondition is thrown if the header constraints are not met
-    PRECONDITION,
-    // Exclude from processing further
-    FILTER
-  }
 }

entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/ValidationExceptionCollection.java

@@ -7,6 +7,7 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -15,17 +16,20 @@
 public class ValidationExceptionCollection
     extends HashMap<Pair<Urn, String>, Set<AspectValidationException>> {
 
-  private final Set<Integer> failedHashCodes;
-  private final Set<Integer> filteredHashCodes;
+  private final Map<ValidationSubType, Set<Integer>> subTypeHashCodes;
 
   public ValidationExceptionCollection() {
     super();
-    this.failedHashCodes = new HashSet<>();
-    this.filteredHashCodes = new HashSet<>();
+    this.subTypeHashCodes = new HashMap<>();
   }
 
   public boolean hasFatalExceptions() {
-    return !failedHashCodes.isEmpty();
+    return subTypeHashCodes.keySet().stream()
+        .anyMatch(subType -> !ValidationSubType.FILTER.equals(subType));
+  }
+
+  public Set<ValidationSubType> getSubTypes() {
+    return subTypeHashCodes.keySet();
   }
 
   public static ValidationExceptionCollection newCollection() {
@@ -34,11 +38,9 @@ public static ValidationExceptionCollection newCollection() {
 
   public void addException(AspectValidationException exception) {
     super.computeIfAbsent(exception.getAspectGroup(), key -> new HashSet<>()).add(exception);
-    if (!AspectValidationException.SubType.FILTER.equals(exception.getSubType())) {
-      failedHashCodes.add(exception.getItem().hashCode());
-    } else {
-      filteredHashCodes.add(exception.getItem().hashCode());
-    }
+    subTypeHashCodes
+        .computeIfAbsent(exception.getSubType(), key -> new HashSet<>())
+        .add(exception.getItem().hashCode());
   }
 
   public void addException(BatchItem item, String message) {
@@ -58,16 +60,27 @@ public <T extends BatchItem> Collection<T> successful(Collection<T> items) {
   }
 
   public <T extends BatchItem> Stream<T> streamSuccessful(Stream<T> items) {
-    return items.filter(
-        i -> !failedHashCodes.contains(i.hashCode()) && !filteredHashCodes.contains(i.hashCode()));
+    return items.filter(i -> isSuccessful(i.hashCode()));
   }
 
   public <T extends BatchItem> Collection<T> exceptions(Collection<T> items) {
     return streamExceptions(items.stream()).collect(Collectors.toList());
   }
 
   public <T extends BatchItem> Stream<T> streamExceptions(Stream<T> items) {
-    return items.filter(i -> failedHashCodes.contains(i.hashCode()));
+    return items.filter(i -> isException(i.hashCode()));
+  }
+
+  private boolean isException(int hashCode) {
+    return subTypeHashCodes.keySet().stream()
+        .filter(subType -> !ValidationSubType.FILTER.equals(subType))
+        .anyMatch(subType -> subTypeHashCodes.get(subType).contains(hashCode));
+  }
+
+  private boolean isSuccessful(int hashCode) {
+    return !isException(hashCode)
+        && (!subTypeHashCodes.containsKey(ValidationSubType.FILTER)
+            || !subTypeHashCodes.get(ValidationSubType.FILTER).contains(hashCode));
   }
 
   @Override

entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/ValidationSubType.java

@@ -0,0 +1,10 @@
+package com.linkedin.metadata.aspect.plugins.validation;
+
+public enum ValidationSubType {
+  // A validation exception is thrown
+  VALIDATION,
+  // A failed precondition is thrown if the header constraints are not met
+  PRECONDITION,
+  // Exclude from processing further
+  FILTER
+}