fix(docker): start issue when there are multiple kafka brokers in boo…

…tstrap config (#2231)
datahub-project · Mar 12, 2021 · 8d731b5 · 8d731b5
1 parent 81fa8d6
commit 8d731b5
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 4 deletions.
diff --git a/docker/datahub-gms/start.sh b/docker/datahub-gms/start.sh
@@ -7,7 +7,7 @@ fi
 
 dockerize \
   -wait tcp://$EBEAN_DATASOURCE_HOST \
-  -wait tcp://$KAFKA_BOOTSTRAP_SERVER \
+  -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \
   -wait http://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT \
   -wait $NEO4J_HOST \
   -timeout 240s \

diff --git a/docker/datahub-mae-consumer/start.sh b/docker/datahub-mae-consumer/start.sh
@@ -6,7 +6,7 @@ if ! echo $NEO4J_HOST | grep -q "://" ; then
 fi
 
 dockerize \
-  -wait tcp://$KAFKA_BOOTSTRAP_SERVER \
+  -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \
   -wait http://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT \
   -wait $NEO4J_HOST \
   -timeout 240s \

diff --git a/docker/datahub-mce-consumer/start.sh b/docker/datahub-mce-consumer/start.sh
@@ -2,6 +2,6 @@
 
 #    -wait tcp://GMS_HOST:$GMS_PORT \
 dockerize \
-  -wait tcp://$KAFKA_BOOTSTRAP_SERVER \
+  -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \
   -timeout 240s \
   java -jar /datahub/datahub-mce-consumer/bin/mce-consumer-job.jar
diff --git a/...a-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/KafkaConfig.java b/...a-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/KafkaConfig.java
@@ -3,6 +3,7 @@
 import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig;
 import io.confluent.kafka.serializers.KafkaAvroDeserializer;
 import java.time.Duration;
+import java.util.Arrays;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.avro.generic.GenericRecord;
@@ -40,7 +41,7 @@ public KafkaListenerContainerFactory<?> kafkaListenerContainerFactory(KafkaPrope
 
     // KAFKA_BOOTSTRAP_SERVER has precedence over SPRING_KAFKA_BOOTSTRAP_SERVERS
     if (kafkaBootstrapServer != null && kafkaBootstrapServer.length() > 0) {
-      props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBootstrapServer);
+      props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, Arrays.asList(kafkaBootstrapServer.split(",")));
     } // else we rely on KafkaProperties which defaults to localhost:9092
 
     props.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, kafkaSchemaRegistryUrl);