From 0040d5930264ae37b76619c72680d20a92c35215 Mon Sep 17 00:00:00 2001 From: Mars Lan Date: Fri, 8 May 2020 16:16:15 -0700 Subject: [PATCH] refactor: use named volume instead of bind mount in quickstart Volume is the preferred method over bind mount (https://docs.docker.com/storage/volumes/) for persistent container data. This also eliminate the need for the ugly chmod hack for elasticsearch and hopefully fix https://github.com/linkedin/datahub/issues/1650 --- README.md | 2 +- docker/quickstart/README.md | 6 ++---- docker/quickstart/docker-compose.yml | 14 ++++++++++---- docker/quickstart/quickstart.sh | 10 ++-------- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index cf02621f8205d..589478c3dbe3b 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ This repository contains the complete source code for both DataHub's frontend & 3. Clone this repo and `cd` into the root directory of the cloned repository. 4. Run the following command to download and run all Docker containers locally: ``` - cd docker/quickstart && source ./quickstart.sh + ./docker/quickstart/quickstart.sh ``` This step takes a while to run the first time, and it may be difficult to tell if DataHub is fully up and running from the combined log. Please use [this guide](https://github.com/linkedin/datahub/blob/master/docs/debugging.md#how-can-i-confirm-if-all-docker-containers-are-running-as-expected-after-a-quickstart) to verify that each container is running correctly. 5. At this point, you should be able to start DataHub by opening [http://localhost:9001](http://localhost:9001) in your browser. You can sign in using `datahub` as both username and password. However, you'll notice that no data has been ingested yet. diff --git a/docker/quickstart/README.md b/docker/quickstart/README.md index 86b954e6ae121..fa30ba3196d8d 100644 --- a/docker/quickstart/README.md +++ b/docker/quickstart/README.md @@ -1,11 +1,9 @@ # DataHub Quickstart -To start all Docker containers at once, please run below command: +To start all Docker containers at once, please run below command from project root directory: ```bash -cd docker/quickstart && source ./quickstart.sh +./docker/quickstart/quickstart.sh ``` -By default, data will be stored at `/tmp/datahub`, however it can be overwritten by specifying the DATA_STORAGE_FOLDER env var. - At this point, all containers are ready and DataHub can be considered up and running. Check specific containers guide for details: * [Elasticsearch & Kibana](../elasticsearch) diff --git a/docker/quickstart/docker-compose.yml b/docker/quickstart/docker-compose.yml index 98e2d17e99f70..22645b43135d2 100644 --- a/docker/quickstart/docker-compose.yml +++ b/docker/quickstart/docker-compose.yml @@ -16,7 +16,7 @@ services: - "3306:3306" volumes: - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - - ${DATA_STORAGE_FOLDER}/mysql:/var/lib/mysql + - mysqldata:/var/lib/mysql zookeeper: image: confluentinc/cp-zookeeper:5.4.0 @@ -28,7 +28,7 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_TICK_TIME: 2000 volumes: - - ${DATA_STORAGE_FOLDER}/zookeeper:/var/opt/zookeeper + - zkdata:/var/opt/zookeeper broker: image: confluentinc/cp-kafka:5.4.0 @@ -137,7 +137,7 @@ services: - xpack.security.enabled=false - "ES_JAVA_OPTS=-Xms1g -Xmx1g" volumes: - - ${DATA_STORAGE_FOLDER}/elasticsearch:/usr/share/elasticsearch/data + - esdata:/usr/share/elasticsearch/data kibana: image: docker.elastic.co/kibana/kibana:5.6.8 @@ -161,7 +161,7 @@ services: - "7474:7474" - "7687:7687" volumes: - - ${DATA_STORAGE_FOLDER}/neo4j:/data + - neo4jdata:/data # This "container" is a workaround to pre-create search indices elasticsearch-setup: @@ -261,3 +261,9 @@ services: networks: default: name: datahub_network + +volumes: + mysqldata: + esdata: + neo4jdata: + zkdata: diff --git a/docker/quickstart/quickstart.sh b/docker/quickstart/quickstart.sh index 8b6620945e8d5..7f4798e777377 100755 --- a/docker/quickstart/quickstart.sh +++ b/docker/quickstart/quickstart.sh @@ -1,10 +1,4 @@ #!/bin/bash -export DATA_STORAGE_FOLDER=${DATA_STORAGE_FOLDER:=/tmp/datahub} -mkdir -p ${DATA_STORAGE_FOLDER} - -# https://discuss.elastic.co/t/elastic-elasticsearch-docker-not-assigning-permissions-to-data-directory-on-run/65812/4 -mkdir -p ${DATA_STORAGE_FOLDER}/elasticsearch -sudo chmod 777 ${DATA_STORAGE_FOLDER}/elasticsearch - -docker-compose pull && docker-compose up --build +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR && docker-compose pull && docker-compose -p datahub up --build \ No newline at end of file