datahub-project · shirshanka · Jun 23, 2021 · Jun 23, 2021 · Jun 23, 2021 · Jun 23, 2021
diff --git a/docker/broker/env/docker.env b/docker/broker/env/docker.env
@@ -4,4 +4,4 @@ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEX
 KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
-
+KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
diff --git a/docker/datahub-frontend/env/docker.env b/docker/datahub-frontend/env/docker.env
@@ -3,6 +3,7 @@ DATAHUB_GMS_PORT=8080
 DATAHUB_SECRET=YouKnowNothing
 DATAHUB_APP_VERSION=1.0
 DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
+JAVA_OPTS=-Xms256m -Xmx256m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=true -Dpidfile.path=/dev/null
 
 # Uncomment and set these to support SSL connection to GMS
 # NOTE: Currently GMS itself does not offer SSL support, these settings are intended for when there is a proxy in front

diff --git a/docker/datahub-gms/env/docker-without-neo4j.env b/docker/datahub-gms/env/docker-without-neo4j.env
@@ -9,6 +9,7 @@ KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
 ELASTICSEARCH_HOST=elasticsearch
 ELASTICSEARCH_PORT=9200
 GRAPH_SERVICE_IMPL=elasticsearch
+JAVA_OPTS=-Xms256m -Xmx256m
 
 MAE_CONSUMER_ENABLED=true
 MCE_CONSUMER_ENABLED=true

diff --git a/docker/datahub-gms/env/docker.env b/docker/datahub-gms/env/docker.env
@@ -12,6 +12,7 @@ NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
+JAVA_OPTS=-Xms256m -Xmx256m
 GRAPH_SERVICE_IMPL=neo4j
 
 MAE_CONSUMER_ENABLED=true

diff --git a/docker/datahub-gms/env/docker.mariadb.env b/docker/datahub-gms/env/docker.mariadb.env
@@ -13,3 +13,5 @@ NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
 MAE_CONSUMER_ENABLED=true
 MCE_CONSUMER_ENABLED=true
+JAVA_OPTS=-Xms256m -Xmx256m
+
diff --git a/docker/datahub-gms/env/docker.postgres.env b/docker/datahub-gms/env/docker.postgres.env
@@ -13,4 +13,6 @@ NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
 MAE_CONSUMER_ENABLED=true
 MCE_CONSUMER_ENABLED=true
+JAVA_OPTS=-Xms256m -Xmx256m
+
 
diff --git a/docker/docker-compose.tools.yml b/docker/docker-compose.tools.yml
@@ -0,0 +1,38 @@
+# Tools useful for operating & debugging DataHub.
+---
+version: '3.8'
+services:
+  kafka-rest-proxy:
+    image: confluentinc/cp-kafka-rest:5.4.0
+    env_file: kafka-rest-proxy/env/docker.env
+    hostname: kafka-rest-proxy
+    container_name: kafka-rest-proxy
+    ports:
+      - "8082:8082"
+    depends_on:
+      - zookeeper
+      - broker
+      - schema-registry
+
+  kafka-topics-ui:
+    image: landoop/kafka-topics-ui:0.9.4
+    env_file: kafka-topics-ui/env/docker.env
+    hostname: kafka-topics-ui
+    container_name: kafka-topics-ui
+    ports:
+      - "18000:8000"
+    depends_on:
+      - zookeeper
+      - broker
+      - schema-registry
+      - kafka-rest-proxy
+
+  kibana:
+    image: kibana:7.9.3
+    env_file: kibana/env/docker.env
+    container_name: kibana
+    hostname: kibana
+    ports:
+      - "5601:5601"
+    depends_on:
+      - elasticsearch
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -27,31 +27,6 @@ services:
       - "29092:29092"
       - "9092:9092"
 
-  kafka-rest-proxy:
-    image: confluentinc/cp-kafka-rest:5.4.0
-    env_file: kafka-rest-proxy/env/docker.env
-    hostname: kafka-rest-proxy
-    container_name: kafka-rest-proxy
-    ports:
-      - "8082:8082"
-    depends_on:
-      - zookeeper
-      - broker
-      - schema-registry
-
-  kafka-topics-ui:
-    image: landoop/kafka-topics-ui:0.9.4
-    env_file: kafka-topics-ui/env/docker.env
-    hostname: kafka-topics-ui
-    container_name: kafka-topics-ui
-    ports:
-      - "18000:8000"
-    depends_on:
-      - zookeeper
-      - broker
-      - schema-registry
-      - kafka-rest-proxy
-
   # This "container" is a workaround to pre-create topics
   kafka-setup:
     build:
@@ -75,16 +50,6 @@ services:
     ports:
       - "8081:8081"
 
-  schema-registry-ui:
-    image: landoop/schema-registry-ui:latest
-    env_file: schema-registry-ui/env/docker.env
-    container_name: schema-registry-ui
-    hostname: schema-registry-ui
-    ports:
-      - "8000:8000"
-    depends_on:
-      - schema-registry
-
   elasticsearch:
     image: elasticsearch:7.9.3
     env_file: elasticsearch/env/docker.env
@@ -99,16 +64,6 @@ services:
         start_period: 2m
         retries: 4
 
-  kibana:
-    image: kibana:7.9.3
-    env_file: kibana/env/docker.env
-    container_name: kibana
-    hostname: kibana
-    ports:
-      - "5601:5601"
-    depends_on:
-      - elasticsearch
-
   neo4j:
     image: neo4j:4.0.6
     env_file: neo4j/env/docker.env

diff --git a/docker/elasticsearch/env/docker.env b/docker/elasticsearch/env/docker.env
@@ -1,3 +1,3 @@
 discovery.type=single-node
 xpack.security.enabled=false
-ES_JAVA_OPTS=-Xms512m -Xmx512m
+ES_JAVA_OPTS=-Xms256m -Xmx256m
diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
@@ -13,6 +13,7 @@ services:
     - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
     - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
     - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
+    - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
     hostname: broker
     image: confluentinc/cp-kafka:5.4.0
     ports:
@@ -28,12 +29,16 @@ services:
     - DATAHUB_SECRET=YouKnowNothing
     - DATAHUB_APP_VERSION=1.0
     - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
+    - JAVA_OPTS=-Xms256m -Xmx256m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf
+      -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
+      -Dlogback.debug=true -Dpidfile.path=/dev/null
     - KAFKA_BOOTSTRAP_SERVER=broker:29092
     - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
     - ELASTIC_CLIENT_HOST=elasticsearch
     - ELASTIC_CLIENT_PORT=9200
     hostname: datahub-frontend-react
     image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-latest}
+    mem_limit: 512m
     ports:
     - 9002:9002
   datahub-gms:
@@ -52,19 +57,20 @@ services:
     - ELASTICSEARCH_HOST=elasticsearch
     - ELASTICSEARCH_PORT=9200
     - GRAPH_SERVICE_IMPL=elasticsearch
+    - JAVA_OPTS=-Xms256m -Xmx256m
     - MAE_CONSUMER_ENABLED=true
     - MCE_CONSUMER_ENABLED=true
     hostname: datahub-gms
     image: linkedin/datahub-gms:${DATAHUB_VERSION:-latest}
-    mem_limit: 850m
+    mem_limit: 512m
     ports:
     - 8080:8080
   elasticsearch:
     container_name: elasticsearch
     environment:
     - discovery.type=single-node
     - xpack.security.enabled=false
-    - ES_JAVA_OPTS=-Xms512m -Xmx512m
+    - ES_JAVA_OPTS=-Xms256m -Xmx256m
     healthcheck:
       retries: 4
       start_period: 2m

diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml
@@ -13,6 +13,7 @@ services:
     - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
     - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
     - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
+    - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
     hostname: broker
     image: confluentinc/cp-kafka:5.4.0
     ports:
@@ -28,12 +29,16 @@ services:
     - DATAHUB_SECRET=YouKnowNothing
     - DATAHUB_APP_VERSION=1.0
     - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
+    - JAVA_OPTS=-Xms256m -Xmx256m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf
+      -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
+      -Dlogback.debug=true -Dpidfile.path=/dev/null
     - KAFKA_BOOTSTRAP_SERVER=broker:29092
     - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
     - ELASTIC_CLIENT_HOST=elasticsearch
     - ELASTIC_CLIENT_PORT=9200
     hostname: datahub-frontend-react
     image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head}
+    mem_limit: 512m
     ports:
     - 9002:9002
   datahub-gms:
@@ -55,20 +60,21 @@ services:
     - NEO4J_URI=bolt://neo4j
     - NEO4J_USERNAME=neo4j
     - NEO4J_PASSWORD=datahub
+    - JAVA_OPTS=-Xms256m -Xmx256m
     - GRAPH_SERVICE_IMPL=neo4j
     - MAE_CONSUMER_ENABLED=true
     - MCE_CONSUMER_ENABLED=true
     hostname: datahub-gms
     image: linkedin/datahub-gms:${DATAHUB_VERSION:-head}
-    mem_limit: 850m
+    mem_limit: 512m
     ports:
     - 8080:8080
   elasticsearch:
     container_name: elasticsearch
     environment:
     - discovery.type=single-node
     - xpack.security.enabled=false
-    - ES_JAVA_OPTS=-Xms512m -Xmx512m
+    - ES_JAVA_OPTS=-Xms256m -Xmx256m
     healthcheck:
       retries: 4
       start_period: 2m

diff --git a/docker/quickstart/generate_docker_quickstart.py b/docker/quickstart/generate_docker_quickstart.py
@@ -15,10 +15,11 @@
     "schema-registry-ui",
     "kibana",
 ]
+# Note that these are upper bounds on memory usage. Once exceeded, the container is killed.
+# Each service will be configured to use much less Java heap space than allocated here.
 mem_limits = {
-    "datahub-gms": "850m",
-    "datahub-mae-consumer": "256m",
-    "datahub-mce-consumer": "384m",
+    "datahub-gms": "512m",
+    "datahub-frontend-react": "512m",
     "elasticsearch": "1g",
 }