feature: 🐳 - Allow to store Quickstart dockers data in a folder for persistance

docker/kafka/docker-compose.yml

@@ -2,7 +2,7 @@
 version: '3.5'
 services:
   zookeeper:
-    image: confluentinc/cp-zookeeper:5.3.0
+    image: confluentinc/cp-zookeeper:5.4.0
     hostname: zookeeper
     container_name: zookeeper
     ports:
@@ -12,7 +12,7 @@ services:
       ZOOKEEPER_TICK_TIME: 2000
 
   broker:
-    image: confluentinc/cp-kafka:5.3.0
+    image: confluentinc/cp-kafka:5.4.0
     hostname: broker
     container_name: broker
     depends_on:
@@ -30,7 +30,7 @@ services:
 
   # This "container" is a workaround to pre-create topics
   kafka-setup:
-    image: confluentinc/cp-kafka:5.3.0
+    image: confluentinc/cp-kafka:5.4.0
     hostname: kafka-setup
     container_name: kafka-setup
     depends_on:
@@ -46,8 +46,37 @@ services:
       KAFKA_BROKER_ID: ignored
       KAFKA_ZOOKEEPER_CONNECT: ignored
 
+  kafka-rest-proxy:
+    image: confluentinc/cp-kafka-rest:5.4.0
+    hostname: kafka-rest-proxy
+    ports:
+      - "8082:8082"
+    environment:
+      KAFKA_REST_LISTENERS: http://0.0.0.0:8082/
+      KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry:8081/
+      KAFKA_REST_HOST_NAME: kafka-rest-proxy
+      KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://broker:29092
+    depends_on:
+      - zookeeper
+      - broker
+      - schema-registry
+
+  kafka-topics-ui:
+    image: landoop/kafka-topics-ui:0.9.4
+    hostname: kafka-topics-ui
+    ports:
+      - "18000:8000"
+    environment:
+      KAFKA_REST_PROXY_URL: "http://kafka-rest-proxy:8082/"
+      PROXY: "true"
+    depends_on:
+      - zookeeper
+      - broker
+      - schema-registry
+      - kafka-rest-proxy
+
   schema-registry:
-    image: confluentinc/cp-schema-registry:5.3.0
+    image: confluentinc/cp-schema-registry:5.4.0
     hostname: schema-registry
     container_name: schema-registry
     depends_on:

docker/quickstart/README.md

@@ -1,6 +1,10 @@
 # DataHub Quickstart
 To start all Docker containers at once, please run below command:
 ```
+export DATA_STORAGE_FOLDER=/tmp/datahub
+mkdir -p ${DATA_STORAGE_FOLDER}
+mkdir -p ${DATA_STORAGE_FOLDER}/elasticsearch
+chown 1000:1000 ${DATA_STORAGE_FOLDER}/elasticsearch
 cd docker/quickstart && docker-compose pull && docker-compose up --build
 ```
 At this point, all containers are ready and DataHub can be considered up and running. Check specific containers guide

docker/quickstart/docker-compose.yml

@@ -15,19 +15,22 @@ services:
       - "3306:3306"
     volumes:
       - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
+      - ${DATA_STORAGE_FOLDER}/mysql:/var/lib/mysql
 
   zookeeper:
-    image: confluentinc/cp-zookeeper:5.2.1
+    image: confluentinc/cp-zookeeper:5.4.0
     hostname: zookeeper
     container_name: zookeeper
     ports:
       - "2181:2181"
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
+    volumes:
+      - ${DATA_STORAGE_FOLDER}/zookeeper:/var/opt/zookeeper
 
   broker:
-    image: confluentinc/cp-kafka:5.2.1
+    image: confluentinc/cp-kafka:5.4.0
     hostname: broker
     container_name: broker
     depends_on:
@@ -43,9 +46,38 @@ services:
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
 
+  kafka-rest-proxy:
+    image: confluentinc/cp-kafka-rest:5.4.0
+    hostname: kafka-rest-proxy
+    ports:
+      - "8082:8082"
+    environment:
+      KAFKA_REST_LISTENERS: http://0.0.0.0:8082/
+      KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry:8081/
+      KAFKA_REST_HOST_NAME: kafka-rest-proxy
+      KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://broker:29092
+    depends_on:
+      - zookeeper
+      - broker
+      - schema-registry
+
+  kafka-topics-ui:
+    image: landoop/kafka-topics-ui:0.9.4
+    hostname: kafka-topics-ui
+    ports:
+      - "18000:8000"
+    environment:
+      KAFKA_REST_PROXY_URL: "http://kafka-rest-proxy:8082/"
+      PROXY: "true"
+    depends_on:
+      - zookeeper
+      - broker
+      - schema-registry
+      - kafka-rest-proxy
+
   # This "container" is a workaround to pre-create topics
   kafka-setup:
-    image: confluentinc/cp-kafka:5.3.0
+    image: confluentinc/cp-kafka:5.4.0
     hostname: kafka-setup
     container_name: kafka-setup
     depends_on:
@@ -62,7 +94,7 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: ignored
 
   schema-registry:
-    image: confluentinc/cp-schema-registry:5.2.1
+    image: confluentinc/cp-schema-registry:5.4.0
     hostname: schema-registry
     container_name: schema-registry
     depends_on:
@@ -100,13 +132,18 @@ services:
       - discovery.type=single-node
       - xpack.security.enabled=false
       - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+    volumes:
+      - ${DATA_STORAGE_FOLDER}/elasticsearch:/usr/share/elasticsearch/data
 
   kibana:
     image: docker.elastic.co/kibana/kibana:5.6.8
     container_name: kibana
     hostname: kibana
     ports:
       - "5601:5601"
+    environment:
+      - SERVER_HOST=0.0.0.0
+      - ELASTICSEARCH_URL=http://elasticsearch:9200
     depends_on:
       - elasticsearch
 
@@ -119,6 +156,8 @@ services:
     ports:
       - "7474:7474"
       - "7687:7687"
+    volumes:
+      - ${DATA_STORAGE_FOLDER}/neo4j:/data
 
   # This "container" is a workaround to pre-create search indices
   elasticsearch-setup: