Merge branch 'master' into graceful_bq_partition_id_failure

.github/workflows/docker-ingestion-base.yml

@@ -1,5 +1,7 @@
 name: ingestion base
 on:
+  release:
+    types: [published, edited]
   push:
     branches:
       - master

.github/workflows/docker-ingestion-smoke.yml

@@ -0,0 +1,42 @@
+name: ingestion smoke
+on:
+  release:
+    types: [published, edited]
+  push:
+    branches:
+      - master
+    paths:
+      - "docker/datahub-ingestion/**"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  build-smoke:
+    name: Build and Push Docker Image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
+          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
+      - name: Build and Push image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker/datahub-ingestion
+          file: ./docker/datahub-ingestion/smoke.Dockerfile
+          platforms: linux/amd64,linux/arm64
+          tags: acryldata/datahub-ingestion-base:smoke
+          push: true

datahub-web-react/src/app/identity/user/ViewInviteTokenModal.tsx

@@ -42,7 +42,7 @@ type Props = {
 
 export default function ViewInviteTokenModal({ visible, onClose }: Props) {
     const baseUrl = window.location.origin;
-    const { data: getNativeUserInviteTokenData } = useGetNativeUserInviteTokenQuery({});
+    const { data: getNativeUserInviteTokenData } = useGetNativeUserInviteTokenQuery({ skip: !visible });
 
     const [createNativeUserInviteToken, { data: createNativeUserInviteTokenData }] =
         useCreateNativeUserInviteTokenMutation({});

docker/datahub-ingestion/Dockerfile

@@ -25,7 +25,8 @@ ARG RELEASE_VERSION
 RUN cd /datahub-ingestion && \
     sed -i.bak "s/__version__ = \"0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub/__init__.py && \
     cat src/datahub/__init__.py && \
-    pip install ".[all]"
+    pip install ".[all]" && \
+    pip freeze
 
 FROM base as dev-install
 # Dummy stage for development. Assumes code is built on your machine and mounted to this image.

docker/datahub-ingestion/base-requirements.txt

@@ -1,3 +1,4 @@
+acryl-datahub
 absl-py==1.1.0
 acryl-iceberg-legacy==0.0.4
 acryl-PyHive==0.6.13

docker/datahub-ingestion/base.Dockerfile

@@ -12,4 +12,5 @@ RUN apt-get update && apt-get install -y \
 
 COPY ./base-requirements.txt requirements.txt
 
-RUN pip install -r requirements.txt
+RUN pip install -r requirements.txt && \
+    pip uninstall -y acryl-datahub

docker/datahub-ingestion/smoke.Dockerfile

@@ -0,0 +1,18 @@
+FROM acryldata/datahub-ingestion-base as base
+
+RUN apt-get update && apt-get install -y \
+    sudo \
+    python3-dev \
+    libgtk2.0-0 \
+    libgtk-3-0 \
+    libgbm-dev \
+    libnotify-dev \
+    libgconf-2-4 \
+    libnss3 \
+    libxss1 \
+    libasound2 \
+    libxtst6 \
+    xauth \
+    xvfb
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y  openjdk-11-jdk

docker/docker-compose.override.yml

@@ -9,7 +9,7 @@ services:
     env_file: mysql/env/docker.env
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
     ports:
-      - "3306:3306"
+      - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
     volumes:
       - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
       - mysqldata:/var/lib/mysql

docker/docker-compose.yml

@@ -12,7 +12,7 @@ services:
     hostname: zookeeper
     container_name: zookeeper
     ports:
-      - "2181:2181"
+      - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     volumes:
       - zkdata:/var/opt/zookeeper
 
@@ -24,8 +24,7 @@ services:
     depends_on:
       - zookeeper
     ports:
-      - "29092:29092"
-      - "9092:9092"
+      - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
     volumes:
       - broker:/var/lib/kafka/data/
 
@@ -50,15 +49,15 @@ services:
       - zookeeper
       - broker
     ports:
-      - "8081:8081"
+      - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
 
   elasticsearch:
     image: elasticsearch:7.9.3
     env_file: elasticsearch/env/docker.env
     container_name: elasticsearch
     hostname: elasticsearch
     ports:
-      - "9200:9200"
+      - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
     environment:
       - discovery.type=single-node
       - xpack.security.enabled=false
@@ -75,8 +74,8 @@ services:
     hostname: neo4j
     container_name: neo4j
     ports:
-      - "7474:7474"
-      - "7687:7687"
+      - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474
+      - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687
     volumes:
       - neo4jdata:/data
 
@@ -100,7 +99,7 @@ services:
     hostname: datahub-gms
     container_name: datahub-gms
     ports:
-      - "8080:8080"
+      - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
     depends_on:
       - elasticsearch-setup
       - kafka-setup
@@ -116,7 +115,7 @@ services:
     hostname: datahub-frontend-react
     container_name: datahub-frontend-react
     ports:
-      - "9002:9002"
+      - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
     depends_on:
       - datahub-gms
     volumes:

docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml

@@ -17,8 +17,7 @@ services:
     hostname: broker
     image: kymeric/cp-kafka:latest
     ports:
-    - 29092:29092
-    - 9092:9092
+    - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
   datahub-actions:
     depends_on:
       - datahub-gms
@@ -57,7 +56,7 @@ services:
     hostname: datahub-frontend-react
     image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head}
     ports:
-    - 9002:9002
+    - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
     volumes:
       - ${HOME}/.datahub/plugins:/etc/datahub/plugins
   datahub-gms:
@@ -86,7 +85,7 @@ services:
     hostname: datahub-gms
     image: linkedin/datahub-gms:${DATAHUB_VERSION:-head}
     ports:
-    - 8080:8080
+    - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
     volumes:
     - ${HOME}/.datahub/plugins:/etc/datahub/plugins
   elasticsearch:
@@ -106,7 +105,7 @@ services:
     image: elasticsearch:7.9.3
     mem_limit: 1g
     ports:
-    - 9200:9200
+    - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
     volumes:
     - esdata:/usr/share/elasticsearch/data
   elasticsearch-setup:
@@ -140,7 +139,7 @@ services:
     hostname: mysql
     image: mariadb:10.5.8
     ports:
-    - 3306:3306
+    - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
     volumes:
     - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
     - mysqldata:/var/lib/mysql
@@ -167,7 +166,7 @@ services:
     hostname: schema-registry
     image: eugenetea/schema-registry-arm64:latest
     ports:
-    - 8081:8081
+    - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
   zookeeper:
     container_name: zookeeper
     environment:
@@ -176,7 +175,7 @@ services:
     hostname: zookeeper
     image: kymeric/cp-zookeeper:latest
     ports:
-    - 2181:2181
+    - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     volumes:
     - zkdata:/var/opt/zookeeper
 version: '2.3'

docker/quickstart/docker-compose.quickstart.yml

@@ -17,8 +17,7 @@ services:
     hostname: broker
     image: confluentinc/cp-kafka:5.4.0
     ports:
-    - 29092:29092
-    - 9092:9092
+    - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
     volumes:
     - broker:/var/lib/kafka/data/
   datahub-actions:
@@ -59,7 +58,7 @@ services:
     hostname: datahub-frontend-react
     image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head}
     ports:
-    - 9002:9002
+    - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002
     volumes:
     - ${HOME}/.datahub/plugins:/etc/datahub/plugins
   datahub-gms:
@@ -94,7 +93,7 @@ services:
     hostname: datahub-gms
     image: linkedin/datahub-gms:${DATAHUB_VERSION:-head}
     ports:
-    - 8080:8080
+    - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080
     volumes:
     - ${HOME}/.datahub/plugins/:/etc/datahub/plugins
   elasticsearch:
@@ -114,7 +113,7 @@ services:
     image: elasticsearch:7.9.3
     mem_limit: 1g
     ports:
-    - 9200:9200
+    - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200
     volumes:
     - esdata:/usr/share/elasticsearch/data
   elasticsearch-setup:
@@ -148,7 +147,7 @@ services:
     hostname: mysql
     image: mysql:5.7
     ports:
-    - 3306:3306
+    - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
     volumes:
     - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
     - mysqldata:/var/lib/mysql
@@ -173,8 +172,8 @@ services:
     hostname: neo4j
     image: neo4j:4.0.6
     ports:
-    - 7474:7474
-    - 7687:7687
+    - ${DATAHUB_MAPPED_NEO4J_HTTP_PORT:-7474}:7474
+    - ${DATAHUB_MAPPED_NEO4J_BOLT_PORT:-7687}:7687
     volumes:
     - neo4jdata:/data
   schema-registry:
@@ -188,7 +187,7 @@ services:
     hostname: schema-registry
     image: confluentinc/cp-schema-registry:5.4.0
     ports:
-    - 8081:8081
+    - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
   zookeeper:
     container_name: zookeeper
     environment:
@@ -197,7 +196,7 @@ services:
     hostname: zookeeper
     image: confluentinc/cp-zookeeper:5.4.0
     ports:
-    - 2181:2181
+    - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     volumes:
     - zkdata:/var/opt/zookeeper
 version: '2.3'