Merge branch 'master' into PowerBI-Usage-Ingestion

.github/workflows/docker-unified.yml

@@ -990,7 +990,7 @@ jobs:
           DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
           DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }}
           ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }}
-          ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5"
+          ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor] acryl-datahub-actions"
           ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"
         run: |
           ./smoke-test/run-quickstart.sh

build.gradle

@@ -52,7 +52,7 @@ buildscript {
   ext.hadoop3Version = '3.3.6'
   ext.kafkaVersion = '5.5.15'
   ext.hazelcastVersion = '5.3.6'
-  ext.ebeanVersion = '12.16.1'
+  ext.ebeanVersion = '15.5.2'
   ext.googleJavaFormatVersion = '1.18.1'
   ext.openLineageVersion = '1.19.0'
   ext.logbackClassicJava8 = '1.2.12'
@@ -104,8 +104,8 @@ project.ext.spec = [
 
 project.ext.externalDependency = [
     'akkaHttp': 'com.typesafe.akka:akka-http-core_2.12:10.2.10',
-    'antlr4Runtime': 'org.antlr:antlr4-runtime:4.7.2',
-    'antlr4': 'org.antlr:antlr4:4.7.2',
+    'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
+    'antlr4': 'org.antlr:antlr4:4.9.3',
     'assertJ': 'org.assertj:assertj-core:3.11.1',
     'avro': 'org.apache.avro:avro:1.11.3',
     'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3',
@@ -129,8 +129,10 @@ project.ext.externalDependency = [
     'dropwizardMetricsCore': 'io.dropwizard.metrics:metrics-core:4.2.3',
     'dropwizardMetricsJmx': 'io.dropwizard.metrics:metrics-jmx:4.2.3',
     'ebean': 'io.ebean:ebean:' + ebeanVersion,
+    'ebeanTest': 'io.ebean:ebean-test:' + ebeanVersion,
     'ebeanAgent': 'io.ebean:ebean-agent:' + ebeanVersion,
     'ebeanDdl': 'io.ebean:ebean-ddl-generator:' + ebeanVersion,
+    'ebeanQueryBean': 'io.ebean:querybean-generator:' + ebeanVersion,
     'elasticSearchRest': 'org.opensearch.client:opensearch-rest-high-level-client:' + elasticsearchVersion,
     'elasticSearchJava': 'org.opensearch.client:opensearch-java:2.6.0',
     'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1',
@@ -359,6 +361,9 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
     exclude group: "org.slf4j", module: "slf4j-log4j12"
     exclude group: "org.slf4j", module: "slf4j-nop"
     exclude group: "org.slf4j", module: "slf4j-ext"
+
+    resolutionStrategy.force externalDependency.antlr4Runtime
+    resolutionStrategy.force externalDependency.antlr4
   }
 }
 

datahub-graphql-core/src/main/resources/search.graphql

@@ -550,6 +550,21 @@ enum FilterOperator {
   Represent the relation less than or equal to, e.g. ownerCount <= 3
   """
   LESS_THAN_OR_EQUAL_TO
+
+  """
+  Represent the relation: URN field any nested children in addition to the given URN
+  """
+  DESCENDANTS_INCL
+
+  """
+  Represent the relation: URN field matches any nested parent in addition to the given URN
+  """
+  ANCESTORS_INCL
+
+  """
+  Represent the relation: URN field matches any nested child or parent in addition to the given URN
+  """
+  RELATED_INCL
 }
 
 """

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/CreateAspectTableStep.java

@@ -77,7 +77,7 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
       }
 
       try {
-        _server.execute(_server.createSqlUpdate(sqlUpdateStr));
+        _server.execute(_server.sqlUpdate(sqlUpdateStr));
       } catch (Exception e) {
         context.report().addLine("Failed to create table metadata_aspect_v2", e);
         return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED);

datahub-upgrade/src/main/resources/application.properties

@@ -3,3 +3,4 @@ management.health.neo4j.enabled=false
 ingestion.enabled=false
 spring.main.allow-bean-definition-overriding=true
 entityClient.impl=restli
+metadataChangeProposal.throttle.updateIntervalMs=0

datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTest.java

@@ -1,12 +1,18 @@
 package com.linkedin.datahub.upgrade;
 
-import static org.testng.AssertJUnit.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
 
 import com.linkedin.datahub.upgrade.restoreindices.RestoreIndices;
 import com.linkedin.datahub.upgrade.system.BlockingSystemUpgrade;
+import com.linkedin.metadata.dao.throttle.NoOpSensor;
+import com.linkedin.metadata.dao.throttle.ThrottleSensor;
 import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder;
 import javax.inject.Named;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.ActiveProfiles;
 import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
@@ -28,6 +34,10 @@ public class UpgradeCliApplicationTest extends AbstractTestNGSpringContextTests
 
   @Autowired private ESIndexBuilder esIndexBuilder;
 
+  @Qualifier("kafkaThrottle")
+  @Autowired
+  private ThrottleSensor kafkaThrottle;
+
   @Test
   public void testRestoreIndicesInit() {
     /*
@@ -46,4 +56,10 @@ public void testBuildIndicesInit() {
     assertFalse(
         esIndexBuilder.getElasticSearchConfiguration().getBuildIndices().isAllowDocCountMismatch());
   }
+
+  @Test
+  public void testNoThrottle() {
+    assertEquals(
+        new NoOpSensor(), kafkaThrottle, "No kafka throttle controls expected in datahub-upgrade");
+  }
 }

docker/datahub-ingestion-base/Dockerfile

@@ -43,7 +43,9 @@ RUN apt-get update && apt-get upgrade -y \
     krb5-user \
     krb5-config \
     libkrb5-dev \
+    librdkafka-dev \
     wget \
+    curl \
     zip \
     unzip \
     ldap-utils \

docker/profiles/docker-compose.actions.yml

@@ -1,7 +1,7 @@
 
 x-datahub-actions-service: &datahub-actions-service
   hostname: actions
-  image: ${DATAHUB_ACTIONS_IMAGE:-${DATAHUB_ACTIONS_REPO:-acryldata}/datahub-actions}:${ACTIONS_VERSION:-v0.0.14}
+  image: ${DATAHUB_ACTIONS_IMAGE:-${DATAHUB_ACTIONS_REPO:-acryldata}/datahub-actions}:${ACTIONS_VERSION:-v0.1.1}
   env_file:
     - datahub-actions/env/docker.env
     - ${DATAHUB_LOCAL_COMMON_ENV:-empty.env}
@@ -87,4 +87,4 @@ services:
       - debug-elasticsearch
     depends_on:
       datahub-gms-debug-elasticsearch:
-        condition: service_healthy
+        condition: service_healthy

docs-website/adoptionStoriesIndexes.json

@@ -22,6 +22,18 @@
       "category": "Financial & Fintech",
       "description": "<i>\"We found DataHub to provide excellent coverage for our needs. What we appreciate most about DataHub is <b>its powerful API platform.</b>\"<br /><br /><div style='color: gray;'>— Jean-Pierre Dijcks, Sr. Dir. Product Management at VISA</div></i><br />"
     },
+    {
+      "name": "Notion",
+      "slug": "notion",
+      "imageUrl": "/img/logos/companies/notion.png",
+      "imageSize": "small",
+      "link": "https://blog.datahubproject.io/how-notion-uses-acryl-data-to-ensure-data-reliability-800427a9ba66",
+      "linkType": "blog",
+      "tagline": "How Notion Uses Acryl Data to Ensure Data Reliability",
+      "category": "B2B & B2C",
+      "platform": "cloud",
+      "description": "<i>\"We rely on Acryl to gain insights and ensure our critical data is reliable. Acryl’s managed product takes DataHub to the next level through automation and emphasis on time-to-value.\"<br /><br /><div style='color: gray;'>—  Ada Draginda, Senior Data Engineer at Notion</div></i><br />"
+    },
     {
       "name": "Optum",
       "slug": "optum",
@@ -99,6 +111,30 @@
       "category": "Financial & Fintech",
       "description": "Discover how Checkout leverage DataHub for advanced data management and compliance, especially in managing sensitive data types."
     },
+    {
+      "name": "MYOB",
+      "slug": "myob",
+      "imageUrl": "/img/logos/companies/myob.png",
+      "imageSize": "medium",
+      "link": "https://blog.datahubproject.io/how-myob-improved-data-reliability-for-dbt-and-snowflake-with-acryl-a1aa26285480",
+      "linkType": "blog",
+      "tagline": "How MYOB Improved Data Reliability for dbt and Snowflake",
+      "category": "Financial & Fintech",
+      "platform": "cloud",
+      "description": "<i>\"Before bringing Acryl on board, MYOB’s data teams would see multiple breaking changes per week...Acryl has helped us significantly reduce the number of breaking changes, to the extent that they are no longer a burden on all teams.\"<br /><br /><div style='color: gray;'>—  Asad Naveed, Engineering Manager at MYOB</div></i><br />"
+    },
+    {
+      "name": "DPG Media",
+      "slug": "dpg-media",
+      "imageUrl": "/img/logos/companies/dpg-media.png",
+      "imageSize": "medium",
+      "link": "https://blog.datahubproject.io/how-acryl-data-helped-dpg-media-save-25-per-month-in-snowflake-costs-c29a1618a703",
+      "linkType": "blog",
+      "tagline": "How Acryl Data Helped DPG Media Save 25% Per Month in Snowflake Costs",
+      "category": "And More",
+      "platform": "cloud",
+      "description": "<i>DPG Media used DataHub Cloud to identify and safely retire redundant assets from their data warehouse, reducing costs by 25%. <br /><br />DataHub Cloud is used to bring federated governance to the data mesh architecture at DPG Media.<br />"
+    },
     {
       "name": "MediaMarkt Saturn",
       "slug": "mediamarkt-saturn",

docs-website/src/pages/adoption-stories/_components/LearnItemCard/styles.module.scss

@@ -16,8 +16,9 @@
 }
 
 .card_button {
- padding: 1rem;
+ padding: 1.5rem;
  text-align: center;
+ margin-top: auto;
 }
 
 .card {
@@ -53,6 +54,7 @@
 
 .featureBody {
   padding: 0 2rem;
+  flex-grow: 1;
 }
 
 .card_image {

docs-website/src/pages/adoption-stories/index.jsx

@@ -3,18 +3,21 @@ import Layout from "@theme/Layout";
 import BrowserOnly from "@docusaurus/BrowserOnly";
 import LearnItemCard from "./_components/LearnItemCard";
 import styles from "./styles.module.scss";
-
+import clsx from "clsx";
 import customerStoriesIndexes from "../../../adoptionStoriesIndexes.json";
 
 function AdoptionStoriesListPageContent() {
   const companies = (customerStoriesIndexes?.companies || []).filter((company) => company.link);
   const [activeFilters, setActiveFilters] = useState([]);
+  const [platformFilter, setPlatformFilter] = useState(false); // New state for platform filter
   const categories = ["B2B & B2C", "E-Commerce", "Financial & Fintech", "And More"];
   const selectedCardRef = useRef(null);
 
-  const filteredItems = activeFilters.length
-    ? companies.filter((company) => activeFilters.includes(company.category))
-    : companies;
+  const filteredItems = companies.filter((company) => {
+    const categoryMatch = activeFilters.length ? activeFilters.includes(company.category) : true;
+    const platformMatch = platformFilter ? company.platform === "cloud" : true;
+    return categoryMatch && platformMatch;
+  });
 
   const handleFilterToggle = (category) => {
     if (activeFilters.includes(category)) {
@@ -24,6 +27,10 @@ function AdoptionStoriesListPageContent() {
     }
   };
 
+  const handlePlatformFilterToggle = () => {
+    setPlatformFilter(!platformFilter);
+  };
+
   useEffect(() => {
     const selectedSlug = window.location.hash.substring(1);
     if (selectedCardRef.current) {
@@ -45,7 +52,6 @@ function AdoptionStoriesListPageContent() {
             </div>
           </div>
           <div className={styles.filterBar}>
-            <strong>For: </strong>
             {categories.map((category) => (
               <button
                 key={category}
@@ -55,6 +61,12 @@ function AdoptionStoriesListPageContent() {
                 {category}
               </button>
             ))}
+            <button
+              className={clsx(styles.cloudButton, 'button button--secondary', { 'button--active': platformFilter })}
+              onClick={handlePlatformFilterToggle}
+            >
+              DataHub Cloud
+            </button>
           </div>
         </div>
       </header>

docs-website/src/pages/adoption-stories/styles.module.scss

@@ -4,4 +4,4 @@
   align-items: center;
   gap: 10px;
   flex-wrap: wrap;
-}
+}

docs-website/src/pages/cloud/CompanyLogos/customersData.json

@@ -69,7 +69,7 @@
         "asset": {
           "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/b446f595b4b13a72ee82a285924715f950e012ca-540x270.png"
         },
-        "alt": "DPG Megia"
+        "alt": "DPG Media"
       }
     },
     {

docs-website/src/pages/cloud/Hero/hubspotFormStyles.css → docs-website/src/pages/cloud/DemoForm/hubspotFormStyles.css

@@ -1,28 +1,23 @@
-#hubspotForm {
-  text-align: left;
-}
-
-#hubspotForm .input {
-  margin: 0;
+.hs-input {
+  width: 100% !important;
 }
 
-#hubspotForm .hs-form input,
-#hubspotForm .hs-form select,
-#hubspotForm .hs-form textarea {
+.hs-form input,
+.hs-form select,
+.hs-form textarea {
   border: .5px solid #DDD;
   background-color: #FFF;
   padding: .75rem 1rem; 
   margin: 0.4rem auto;
   border-radius: 8px; 
   font-size: 16px;
-  width: 100%;
 }
 
-#hubspotForm .hs-form ::placeholder {
+.hs-form ::placeholder {
   font-style: Manrope;
 }
 
-#hubspotForm .hs-form input[type="submit"] {
+.hs-form input[type="submit"] {
   background-color: #1990FF;
   color: #fff; 
   border: none; 
@@ -34,45 +29,46 @@
   width: 100px;
 }
 
-#hubspotForm .hs-form input[type="submit"]:hover {
+.hs-form input[type="submit"]:hover {
   background-color: #0056b3; /* Button hover color */
 }
 
 /* hide the label */
 
-#hubspotForm .hs-form label span {
+.hs-form label span {
   display: none;
 }
 
 /* error labels */
 
-#hubspotForm .hs-form .hs-error-msgs {
+.hs-form .hs-error-msgs {
   font-size: 15px;
   color: red;
   list-style-type: none;
   padding: 0;
+  text-align: left;
 }
 
-#hubspotForm .hs-form .error {
+.hs-form .error {
   border: red 1.5px solid !important;
 }
 
 /* customize placeholder style */
 
-#hubspotForm .hs-form input::placeholder, 
-#hubspotForm .hs-form textarea::placeholder,
-#hubspotForm .hs-form textarea {
+.hs-form input::placeholder, 
+.hs-form textarea::placeholder,
+.hs-form textarea {
   font-size: 16px;
   font-weight: 400;
   font-family: sans-serif;
   color: gray;
 }
 
-#hubspotForm .hs-form .hs_firstname {
+.hs-form .hs_firstname {
   padding-right: 0.5rem;
 }
 
-#hubspotForm .hs-form .hs_lastname {
+.hs-form .hs_lastname {
   padding-left: 0.5rem;
 }