feat(datahub cli): DataHub CLI Quickstart

.github/workflows/build-and-test.yml

@@ -65,14 +65,21 @@ jobs:
       - name: Gradle build
         run: ./gradlew build -x check -x docs-website:build
       - name: Smoke test
-        run: |
-          ./docker/dev.sh -d
-          sleep 30
-          ./smoke-test/smoke.sh
+        run: ./smoke-test/smoke.sh
       - name: Slack failure notification
         if: failure() && github.event_name == 'push'
         uses: kpritam/slack-job-status-action@v1
         with:
           job-status: ${{ job.status }}
           slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
           channel: github-activities
+
+  quickstart-compose-validation:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.6"
+      - name: Quickstart Compose Validation
+        run: ./docker/quickstart/generate_and_compare.sh

docker/docker-compose.override.yml

@@ -14,6 +14,17 @@ services:
       - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
       - mysqldata:/var/lib/mysql
 
+  mysql-setup:
+    build:
+      context: ../
+      dockerfile: docker/mysql-setup/Dockerfile
+    image: acryldata/datahub-mysql-setup:head
+    env_file: mysql-setup/env/docker.env
+    hostname: mysql-setup
+    container_name: mysql-setup
+    depends_on:
+      - mysql
+
   datahub-gms:
     env_file: datahub-gms/env/docker.env
     depends_on:

docker/elasticsearch/env/docker.env

@@ -1,3 +1,3 @@
 discovery.type=single-node
 xpack.security.enabled=false
-ES_JAVA_OPTS=-Xms1g -Xmx1g
+ES_JAVA_OPTS=-Xms512m -Xmx512m

docker/mysql-setup/env/docker.env

@@ -0,0 +1,5 @@
+MYSQL_HOST=mysql
+MYSQL_PORT=3306
+MYSQL_USERNAME=datahub
+MYSQL_PASSWORD=datahub
+DATAHUB_DB_NAME=datahub

docker/quickstart/docker-compose.quickstart.yml

@@ -0,0 +1,208 @@
+networks:
+  default:
+    name: datahub_network
+services:
+  broker:
+    container_name: broker
+    depends_on:
+    - zookeeper
+    environment:
+    - KAFKA_BROKER_ID=1
+    - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+    - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+    - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
+    - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+    - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
+    hostname: broker
+    image: confluentinc/cp-kafka:5.4.0
+    ports:
+    - 29092:29092
+    - 9092:9092
+  datahub-frontend-react:
+    container_name: datahub-frontend-react
+    depends_on:
+    - datahub-gms
+    environment:
+    - DATAHUB_GMS_HOST=datahub-gms
+    - DATAHUB_GMS_PORT=8080
+    - DATAHUB_SECRET=YouKnowNothing
+    - DATAHUB_APP_VERSION=1.0
+    - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
+    - ELASTIC_CLIENT_HOST=elasticsearch
+    - ELASTIC_CLIENT_PORT=9200
+    hostname: datahub-frontend-react
+    image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-latest}
+    ports:
+    - 9002:9002
+  datahub-gms:
+    container_name: datahub-gms
+    depends_on:
+    - mysql
+    environment:
+    - DATASET_ENABLE_SCSI=false
+    - EBEAN_DATASOURCE_USERNAME=datahub
+    - EBEAN_DATASOURCE_PASSWORD=datahub
+    - EBEAN_DATASOURCE_HOST=mysql:3306
+    - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
+    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - ELASTICSEARCH_HOST=elasticsearch
+    - ELASTICSEARCH_PORT=9200
+    - NEO4J_HOST=http://neo4j:7474
+    - NEO4J_URI=bolt://neo4j
+    - NEO4J_USERNAME=neo4j
+    - NEO4J_PASSWORD=datahub
+    hostname: datahub-gms
+    image: linkedin/datahub-gms:${DATAHUB_VERSION:-latest}
+    mem_limit: 850m
+    ports:
+    - 8080:8080
+  datahub-mae-consumer:
+    container_name: datahub-mae-consumer
+    depends_on:
+    - kafka-setup
+    - elasticsearch-setup
+    - neo4j
+    environment:
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - ELASTICSEARCH_HOST=elasticsearch
+    - ELASTICSEARCH_PORT=9200
+    - NEO4J_HOST=http://neo4j:7474
+    - NEO4J_URI=bolt://neo4j
+    - NEO4J_USERNAME=neo4j
+    - NEO4J_PASSWORD=datahub
+    - GMS_HOST=datahub-gms
+    - GMS_PORT=8080
+    hostname: datahub-mae-consumer
+    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-latest}
+    mem_limit: 256m
+    ports:
+    - 9091:9091
+  datahub-mce-consumer:
+    container_name: datahub-mce-consumer
+    depends_on:
+    - kafka-setup
+    - datahub-gms
+    environment:
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - GMS_HOST=datahub-gms
+    - GMS_PORT=8080
+    hostname: datahub-mce-consumer
+    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-latest}
+    mem_limit: 384m
+    ports:
+    - 9090:9090
+  elasticsearch:
+    container_name: elasticsearch
+    environment:
+    - discovery.type=single-node
+    - xpack.security.enabled=false
+    - ES_JAVA_OPTS=-Xms512m -Xmx512m
+    healthcheck:
+      retries: 4
+      start_period: 2m
+      test:
+      - CMD-SHELL
+      - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s'
+        || exit 1
+    hostname: elasticsearch
+    image: elasticsearch:7.9.3
+    mem_limit: 1g
+    ports:
+    - 9200:9200
+    volumes:
+    - esdata:/usr/share/elasticsearch/data
+  elasticsearch-setup:
+    container_name: elasticsearch-setup
+    depends_on:
+    - elasticsearch
+    environment:
+    - ELASTICSEARCH_HOST=elasticsearch
+    - ELASTICSEARCH_PORT=9200
+    - ELASTICSEARCH_PROTOCOL=http
+    hostname: elasticsearch-setup
+    image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-latest}
+  kafka-setup:
+    container_name: kafka-setup
+    depends_on:
+    - broker
+    - schema-registry
+    environment:
+    - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    hostname: kafka-setup
+    image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-latest}
+  mysql:
+    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+    container_name: mysql
+    environment:
+    - MYSQL_DATABASE=datahub
+    - MYSQL_USER=datahub
+    - MYSQL_PASSWORD=datahub
+    - MYSQL_ROOT_PASSWORD=datahub
+    hostname: mysql
+    image: mysql:5.7
+    ports:
+    - 3306:3306
+    volumes:
+    - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
+    - mysqldata:/var/lib/mysql
+  mysql-setup:
+    container_name: mysql-setup
+    depends_on:
+    - mysql
+    environment:
+    - MYSQL_HOST=mysql
+    - MYSQL_PORT=3306
+    - MYSQL_USERNAME=datahub
+    - MYSQL_PASSWORD=datahub
+    - DATAHUB_DB_NAME=datahub
+    hostname: mysql-setup
+    image: acryldata/datahub-mysql-setup:head
+  neo4j:
+    container_name: neo4j
+    environment:
+    - NEO4J_AUTH=neo4j/datahub
+    - NEO4J_dbms_default__database=graph.db
+    - NEO4J_dbms_allow__upgrade=true
+    hostname: neo4j
+    image: neo4j:4.0.6
+    ports:
+    - 7474:7474
+    - 7687:7687
+    volumes:
+    - neo4jdata:/data
+  schema-registry:
+    container_name: schema-registry
+    depends_on:
+    - zookeeper
+    - broker
+    environment:
+    - SCHEMA_REGISTRY_HOST_NAME=schemaregistry
+    - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
+    hostname: schema-registry
+    image: confluentinc/cp-schema-registry:5.4.0
+    ports:
+    - 8081:8081
+  zookeeper:
+    container_name: zookeeper
+    environment:
+    - ZOOKEEPER_CLIENT_PORT=2181
+    - ZOOKEEPER_TICK_TIME=2000
+    hostname: zookeeper
+    image: confluentinc/cp-zookeeper:5.4.0
+    ports:
+    - 2181:2181
+    volumes:
+    - zkdata:/var/opt/zookeeper
+version: '2'
+volumes:
+  esdata: null
+  mysqldata: null
+  neo4jdata: null
+  zkdata: null

docker/quickstart/generate_and_compare.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR"
+
+set -euxo pipefail
+
+python3 -m venv venv
+source venv/bin/activate
+
+pip install -r requirements.txt
+python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml temp.quickstart.yml
+
+if cmp docker-compose.quickstart.yml temp.quickstart.yml; then
+    printf 'docker-compose.quickstart.yml is up to date.'
+    exit 0
+else
+    printf 'docker-compose.quickstart.yml is out of date.'
+    exit 1
+fi