Merge remote-tracking branch 'linkedinoss/master' into feat/upgradePlay

build.gradle

@@ -2,7 +2,7 @@ buildscript {
   ext.junitJupiterVersion = '5.6.1'
   ext.pegasusVersion = '29.22.16'
   ext.mavenVersion = '3.6.3'
-  ext.springVersion = '5.3.18'
+  ext.springVersion = '5.3.19'
   ext.springBootVersion = '2.5.12'
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
@@ -47,7 +47,7 @@ project.ext.externalDependency = [
     'assertJ': 'org.assertj:assertj-core:3.11.1',
     'avro_1_7': 'org.apache.avro:avro:1.7.7',
     'avroCompiler_1_7': 'org.apache.avro:avro-compiler:1.7.7',
-    'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.1',
+    'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.10',
     'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.1',
     'cacheApi' : 'javax.cache:cache-api:1.1.0',
     'commonsIo': 'commons-io:commons-io:2.4',
@@ -77,9 +77,9 @@ project.ext.externalDependency = [
     'httpClient': 'org.apache.httpcomponents:httpclient:4.5.9',
     'httpAsyncClient': 'org.apache.httpcomponents:httpasyncclient:4.1.5',
     'iStackCommons': 'com.sun.istack:istack-commons-runtime:4.0.1',
-    'jacksonCore': 'com.fasterxml.jackson.core:jackson-core:2.9.10',
-    'jacksonDataBind': 'com.fasterxml.jackson.core:jackson-databind:2.9.10.7',
-    'jacksonDataFormatYaml': 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.10',
+    'jacksonCore': 'com.fasterxml.jackson.core:jackson-core:2.13.2',
+    'jacksonDataBind': 'com.fasterxml.jackson.core:jackson-databind:2.13.2.2',
+    'jacksonDataFormatYaml': 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.2',
     'javatuples': 'org.javatuples:javatuples:1.2',
     'javaxInject' : 'javax.inject:javax.inject:1',
     'javaxValidation' : 'javax.validation:validation-api:2.0.1.Final',
@@ -109,7 +109,7 @@ project.ext.externalDependency = [
     'opentelemetryApi': 'io.opentelemetry:opentelemetry-api:1.0.0',
     'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:1.0.0',
     'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
-    'parquet': 'org.apache.parquet:parquet-avro:1.12.0',
+    'parquet': 'org.apache.parquet:parquet-avro:1.12.2',
     'picocli': 'info.picocli:picocli:4.5.0',
     'playCache': 'com.typesafe.play:play-cache_2.12:2.7.6',
     'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.0.8',
@@ -191,6 +191,8 @@ subprojects {
         implementation('org.apache.commons:commons-compress:1.21')
         implementation('org.apache.velocity:velocity-engine-core:2.3')
         implementation('org.hibernate:hibernate-validator:6.0.20.Final')
+        implementation('com.fasterxml.jackson.core:jackson-databind:2.13.2.2')
+        implementation('com.fasterxml.jackson.core:jackson-dataformat-cbor:2.13.2')
       }
     }
 

datahub-web-react/src/app/entity/EntityRegistry.tsx

@@ -122,12 +122,16 @@ export default class EntityRegistry {
                 ...entity.getLineageVizConfig?.(data),
                 downstreamChildren: genericEntityProperties?.downstream?.relationships
                     ?.filter((relationship) => relationship.entity)
+                    // eslint-disable-next-line @typescript-eslint/dot-notation
+                    ?.filter((relationship) => !relationship.entity?.['status']?.removed)
                     ?.map((relationship) => ({
                         entity: relationship.entity as EntityInterface,
                         type: (relationship.entity as EntityInterface).type,
                     })),
                 upstreamChildren: genericEntityProperties?.upstream?.relationships
                     ?.filter((relationship) => relationship.entity)
+                    // eslint-disable-next-line @typescript-eslint/dot-notation
+                    ?.filter((relationship) => !relationship.entity?.['status']?.removed)
                     ?.map((relationship) => ({
                         entity: relationship.entity as EntityInterface,
                         type: (relationship.entity as EntityInterface).type,

datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaHeader.tsx

@@ -72,7 +72,7 @@ const SchemaBlameSelector = styled(Select)`
     &&& {
         font-weight: 400;
         margin-top: 6px;
-        min-width: 17%;
+        min-width: 30px;
         margin-right: 10px;
         border-radius: 0px 8px 8px 0px;
     }
@@ -89,6 +89,7 @@ const BlameRadio = styled(Radio.Group)`
     &&& {
         margin-top: 6px;
         margin-right: 10px;
+        min-width: 140px;
     }
 `;
 
@@ -104,6 +105,7 @@ const CurrentVersionTimestampText = styled(Typography.Text)`
         margin-top: 10px;
         margin-right: 10px;
         color: ${ANTD_GRAY[7]};
+        min-width: 220px;
     }
 `;
 

datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx

@@ -2,8 +2,10 @@ import React, { useState } from 'react';
 import {
     CheckOutlined,
     CopyOutlined,
+    ExclamationCircleOutlined,
     FolderOpenOutlined,
     InfoCircleOutlined,
+    LinkOutlined,
     MoreOutlined,
     RightOutlined,
 } from '@ant-design/icons';
@@ -238,6 +240,7 @@ export const EntityHeader = ({ showDeprecateOption }: Props) => {
     const hasDetails = entityData?.deprecation?.note !== '' || entityData?.deprecation?.decommissionTime !== null;
     const isDividerNeeded = entityData?.deprecation?.note !== '' && entityData?.deprecation?.decommissionTime !== null;
     const showAdditionalOptions = showDeprecateOption;
+    const pageUrl = window.location.href;
 
     return (
         <>
@@ -334,13 +337,23 @@ export const EntityHeader = ({ showDeprecateOption }: Props) => {
                                 overlay={
                                     <Menu>
                                         <Menu.Item key="0">
+                                            <MenuItem
+                                                onClick={() => {
+                                                    navigator.clipboard.writeText(pageUrl);
+                                                    message.info('Copied URL!', 1.2);
+                                                }}
+                                            >
+                                                <LinkOutlined /> &nbsp; Copy Url
+                                            </MenuItem>
+                                        </Menu.Item>
+                                        <Menu.Item key="1">
                                             {!entityData?.deprecation?.deprecated ? (
                                                 <MenuItem onClick={() => setShowAddDeprecationDetailsModal(true)}>
-                                                    Mark as deprecated
+                                                    <ExclamationCircleOutlined /> &nbsp; Mark as deprecated
                                                 </MenuItem>
                                             ) : (
                                                 <MenuItem onClick={() => handleUpdateDeprecation(false)}>
-                                                    Mark as un-deprecated
+                                                    <ExclamationCircleOutlined /> &nbsp; Mark as un-deprecated
                                                 </MenuItem>
                                             )}
                                         </Menu.Item>

datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTable.tsx

@@ -91,7 +91,7 @@ export default function SchemaTable({
         title: 'Field',
         dataIndex: 'fieldPath',
         key: 'fieldPath',
-        width: 250,
+        width: 300,
         render: schemaTitleRenderer,
         filtered: true,
     };

datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx

@@ -83,7 +83,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                                 Advanced: Provide a custom CLI version to use for ingestion.
                             </Typography.Paragraph>
                             <Input
-                                placeholder="0.8.32.1"
+                                placeholder="0.8.34.1"
                                 value={state.config?.version || ''}
                                 onChange={(event) => setVersion(event.target.value)}
                             />

docker/datahub-gms/env/docker-without-neo4j.env

@@ -16,7 +16,7 @@ MAE_CONSUMER_ENABLED=true
 MCE_CONSUMER_ENABLED=true
 PE_CONSUMER_ENABLED=true
 UI_INGESTION_ENABLED=true
-UI_INGESTION_DEFAULT_CLI_VERSION=0.8.32.1
+UI_INGESTION_DEFAULT_CLI_VERSION=0.8.34.1
 
 # Uncomment to disable persistence of client-side analytics events
 # DATAHUB_ANALYTICS_ENABLED=false

docker/datahub-gms/env/docker.env

@@ -19,7 +19,7 @@ MAE_CONSUMER_ENABLED=true
 MCE_CONSUMER_ENABLED=true
 PE_CONSUMER_ENABLED=true
 UI_INGESTION_ENABLED=true
-UI_INGESTION_DEFAULT_CLI_VERSION=0.8.32.1
+UI_INGESTION_DEFAULT_CLI_VERSION=0.8.34.1
 
 # Uncomment to enable Metadata Service Authentication
 # METADATA_SERVICE_AUTH_ENABLED=true

docker/quickstart/docker-compose-without-neo4j.quickstart.yml

@@ -80,7 +80,7 @@ services:
     - MCE_CONSUMER_ENABLED=true
     - PE_CONSUMER_ENABLED=true
     - UI_INGESTION_ENABLED=true
-    - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.32.1
+    - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.34.1
     hostname: datahub-gms
     image: linkedin/datahub-gms:${DATAHUB_VERSION:-head}
     ports:

docker/quickstart/docker-compose.quickstart.yml

@@ -86,7 +86,7 @@ services:
     - MCE_CONSUMER_ENABLED=true
     - PE_CONSUMER_ENABLED=true
     - UI_INGESTION_ENABLED=true
-    - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.32.1
+    - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.34.1
     hostname: datahub-gms
     image: linkedin/datahub-gms:${DATAHUB_VERSION:-head}
     ports:

docs-website/sidebars.js

@@ -215,6 +215,7 @@ module.exports = {
       "docs/domains",
       "docs/ui-ingestion",
       "docs/tags",
+      "docs/schema-history",
       "docs/how/search",
       "docs/how/auth/add-users",
       "docs/how/ui-tabs-guide",

docs/how/updating-datahub.md

@@ -4,6 +4,16 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ## Next
 
+### Breaking Changes
+
+### Potential Downtime
+
+### Deprecations
+
+### Other notable Changes
+
+## `v0.8.34`
+
 ### Breaking Changes
 - #4644 Remove `database` option from `snowflake` source which was deprecated since `v0.8.5`
 - #4595 Rename confusing config `report_upstream_lineage` to `upstream_lineage_in_report` in `snowflake` connector which was added in `0.8.32`

docs/policies.md

@@ -66,17 +66,50 @@ privileges [here](https://github.com/datahub-project/datahub/blob/master/metadat
 
 All edits on the UI are covered by a privilege, to make sure we have the ability to restrict write access.
 
-<!---
-TODO: Add table for edit privileges
---->
+We currently support the following:
+
+**Platform-level** privileges for DataHub operators to access & manage the administrative functionality of the system.
+
+| Platform Privileges             | Description                                                                                                                    |
+|---------------------------------|--------------------------------------------------------------------------------------------------------------------------------|
+| Manage Policies                 | Allow actor to create and remove access control policies. Be careful - Actors with this privilege are effectively super users. |
+| Manage Metadata Ingestion       | Allow actor to create, remove, and update Metadata Ingestion sources.                                                          |
+| Manage Secrets                  | Allow actor to create & remove secrets stored inside DataHub.                                                                  |
+| Manage Users & Groups           | Allow actor to create, remove, and update users and groups on DataHub.                                                         |
+| Manage Domains                  | Allow actor to create and remove Asset Domains.                                                                                |
+| View Analytics                  | Allow the actor access to the DataHub analytics dashboard.                                                                     |
+| Generate Personal Access Tokens | Allow the actor to generate personal access tokens for use with DataHub APIs.                                                  |
+
+**Common metadata privileges** to view & modify any entity within DataHub.
+
+| Common Privileges   | Description                                                                                                                      |
+|---------------------|----------------------------------------------------------------------------------------------------------------------------------|
+| View Entity Page    | Allow actor to access the entity page for the resource in the UI. If not granted, it will redirect them to an unauthorized page. |
+| Edit Tags           | Allow actor to add and remove tags to an asset.                                                                                  |
+| Edit Glossary Terms | Allow actor to add and remove glossary terms to an asset.                                                                        |
+| Edit Owners         | Allow actor to add and remove owners of an entity.                                                                               |
+| Edit Description    | Allow actor to edit the description (documentation) of an entity.                                                                |
+| Edit Links          | Allow actor to edit links associated with an entity.                                                                             |
+| Edit Status         | Allow actor to edit the status of an entity (soft deleted or not).                                                               |
+| Edit Domain         | Allow actor to edit the Domain of an entity.                                                                                     |
+| Edit Deprecation    | Allow actor to edit the Deprecation status of an entity.                                                                         |
+| Edit Assertions     | Allow actor to add and remove assertions from an entity.                                                                         |
+| Edit All            | Allow actor to edit any information about an entity. Super user privileges.                                                      |
+
+**Specific entity-level privileges** that are not generalizable.
+
+| Entity       | Privilege                          | Description                                                                                                                                                                |
+|--------------|------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Dataset      | Edit Dataset Column Tags           | Allow actor to edit the column (field) tags associated with a dataset schema.                                                                                              |
+| Dataset      | Edit Dataset Column Glossary Terms | Allow actor to edit the column (field) glossary terms associated with a dataset schema.                                                                                    |
+| Dataset      | Edit Dataset Column Descriptions   | Allow actor to edit the column (field) descriptions associated with a dataset schema.                                                                                      |
+| Dataset      | View Dataset Usage                 | Allow actor to access usage metadata about a dataset both in the UI and in the GraphQL API. This includes example queries, number of queries, etc.                         |
+| Dataset      | View Dataset Profile               | Allow actor to access a dataset's profile both in the UI and in the GraphQL API. This   includes snapshot statistics like #rows, #columns, null percentage per field, etc. |
+| Tag          | Edit Tag Color                     | Allow actor to change the color of a Tag.                                                                                                                                  |
+| Group        | Edit Group Members                 | Allow actor to add and remove members to a group.                                                                                                                          |
+| User         | Edit User Profile                  | Allow actor to change the user's profile including display name, bio, title, profile image, etc.                                                                           |
+| User + Group | Edit Contact Information           | Allow actor to change the contact information such as email & chat handles.                                                                                                |
 
-We currently support the following read privileges
-
-| Privilege            | Description                                                                                                                                                                |
-|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| VIEW_ENTITY_PAGE     | Allow actor to access the entity page for the resource in the UI. If not granted, it will redirect   them to an unauthorized page.                                         |
-| VIEW_DATASET_USAGE   | Allow actor to access usage metadata about a dataset both in the UI and in the GraphQL API. This   includes example queries, number of queries, etc.                       |
-| VIEW_DATASET_PROFILE | Allow actor to access a dataset's profile both in the UI and in the GraphQL API. This   includes snapshot statistics like #rows, #columns, null percentage per field, etc. |
 
 #### Resources
 

docs/schema-history.md

@@ -0,0 +1,44 @@
+# Schema History Guide
+
+## Introduction
+
+As your data evolves over time, with fields getting added or removed, with type changes happening, with your documentation
+and glossary term classifications changing, you may find yourself asking the question: "What did this dataset's schema
+look like three months ago? Six months? What about a year ago?" We're happy to announce that you can now use DataHub to
+answer these questions! We currently surface a Schema Blame view where you can examine what a schema looked like at a
+specific point in time, and what the most recent changes were to each of these fields in that schema.
+
+## Viewing a Dataset's Schema History
+
+When you go to a dataset page, you'll be treated to a view that outlines what the latest version of the dataset is and when
+the schema was last reported/modified. In addition, there's a version selector that outlines what the prior versions of
+the dataset schema are, and when those were reported. Here's an example from our Demo with the
+[Snowflake pets dataset](https://demo.datahubproject.io/dataset/urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.pets,PROD)/Schema?is_lineage_mode=false).
+
+![](./imgs/schema-blame-latest-version.png)
+
+If you click on an older version in the selector, you'll travel back in time and see what the schena looked like back then. Notice
+the changes here to the glossary terms for the `status` field, and to the descriptions for the `created_at` and `updated_at`
+fields.
+
+![](./imgs/schema-blame-older-version.png)
+
+In addition to this, you can also activate the blame view that shows you when the most recent changes were made to each field.
+You can active this by selecting the `Blame` button you see above the top right of the table.
+
+![](./imgs/schema-blame-blame-activated.png)
+
+You can see here that some of these fields were added at the oldest dataset version, while some were added only at this latest
+version. Some fields were modified at the latest version!
+
+The great thing about this blame view is that you can activate it for any version and see what the most recent changes were
+at that point in time for each field. Similar to what Git blame would show you, but for understanding your schemas better!
+
+## Coming Soon: Schema Timeline View
+
+In the future, we plan on adding a timeline view where you can see what changes were made to various schema fields over time
+in a linear fashion. Stay tuned for updates!
+
+## Feedback / Questions / Concerns
+
+We want to hear from you! For any inquiries, including Feedback, Questions, or Concerns, reach out on Slack!