From 31e55d63a7ea39a610b4bd025543e0612747fe50 Mon Sep 17 00:00:00 2001 From: taylan isikdemir Date: Mon, 6 May 2024 17:03:00 -0700 Subject: [PATCH 1/4] Temporary workaround for broken wurstmeister kafka/zookeeper images --- docker/buildkite/docker-compose-es7.yml | 4 ++-- docker/buildkite/docker-compose-local-async-wf.yml | 4 ++-- docker/buildkite/docker-compose-local-es7.yml | 4 ++-- docker/buildkite/docker-compose-local.yml | 4 ++-- docker/buildkite/docker-compose-opensearch2.yml | 4 ++-- docker/buildkite/docker-compose.yml | 4 ++-- docker/dev/cassandra-esv7-kafka.yml | 4 ++-- docker/dev/cassandra-opensearch-kafka.yml | 4 ++-- docker/dev/cassandra-pinot-kafka.yml | 2 +- docker/dev/mongo-esv7-kafka.yml | 6 +++--- docker/dev/mysql-esv7-kafka.yml | 6 +++--- docker/docker-compose-async-wf-kafka.yml | 4 ++-- docker/docker-compose-es-v7.yml | 4 ++-- docker/docker-compose-es.yml | 4 ++-- docker/docker-compose-multiclusters-cass-mysql-es.yaml | 4 ++-- docker/docker-compose-multiclusters-es.yml | 4 ++-- docker/docker-compose-pinot.yml | 2 +- 17 files changed, 34 insertions(+), 34 deletions(-) diff --git a/docker/buildkite/docker-compose-es7.yml b/docker/buildkite/docker-compose-es7.yml index bbf3703d02a..721a6e483ed 100644 --- a/docker/buildkite/docker-compose-es7.yml +++ b/docker/buildkite/docker-compose-es7.yml @@ -17,14 +17,14 @@ services: retries: 10 zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 networks: services-network: aliases: - zookeeper kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper networks: diff --git a/docker/buildkite/docker-compose-local-async-wf.yml b/docker/buildkite/docker-compose-local-async-wf.yml index ce2fa7d3339..039b857e64b 100644 --- a/docker/buildkite/docker-compose-local-async-wf.yml +++ b/docker/buildkite/docker-compose-local-async-wf.yml @@ -19,7 +19,7 @@ services: retries: 10 zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 ports: - "2181:2181" networks: @@ -28,7 +28,7 @@ services: - zookeeper kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: diff --git a/docker/buildkite/docker-compose-local-es7.yml b/docker/buildkite/docker-compose-local-es7.yml index ca6eba498b4..72c6b4158bd 100644 --- a/docker/buildkite/docker-compose-local-es7.yml +++ b/docker/buildkite/docker-compose-local-es7.yml @@ -19,7 +19,7 @@ services: retries: 10 zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 ports: - "2181:2181" networks: @@ -28,7 +28,7 @@ services: - zookeeper kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: diff --git a/docker/buildkite/docker-compose-local.yml b/docker/buildkite/docker-compose-local.yml index 665bac1dea4..113ad3a9388 100644 --- a/docker/buildkite/docker-compose-local.yml +++ b/docker/buildkite/docker-compose-local.yml @@ -45,7 +45,7 @@ services: - postgres zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 expose: - "2181" networks: @@ -54,7 +54,7 @@ services: - zookeeper kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper expose: diff --git a/docker/buildkite/docker-compose-opensearch2.yml b/docker/buildkite/docker-compose-opensearch2.yml index 49547065bbc..a8affd72dcc 100644 --- a/docker/buildkite/docker-compose-opensearch2.yml +++ b/docker/buildkite/docker-compose-opensearch2.yml @@ -17,14 +17,14 @@ services: retries: 10 zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 networks: services-network: aliases: - zookeeper kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper networks: diff --git a/docker/buildkite/docker-compose.yml b/docker/buildkite/docker-compose.yml index 343421fab1c..e4644075a88 100644 --- a/docker/buildkite/docker-compose.yml +++ b/docker/buildkite/docker-compose.yml @@ -37,14 +37,14 @@ services: - postgres zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 networks: services-network: aliases: - zookeeper kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper networks: diff --git a/docker/dev/cassandra-esv7-kafka.yml b/docker/dev/cassandra-esv7-kafka.yml index ad9301276cb..de15e64ad62 100644 --- a/docker/dev/cassandra-esv7-kafka.yml +++ b/docker/dev/cassandra-esv7-kafka.yml @@ -14,7 +14,7 @@ services: environment: - discovery.type=single-node kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: @@ -24,6 +24,6 @@ services: KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 ports: - "2181:2181" diff --git a/docker/dev/cassandra-opensearch-kafka.yml b/docker/dev/cassandra-opensearch-kafka.yml index 84f2ddd01f2..73e311417bc 100644 --- a/docker/dev/cassandra-opensearch-kafka.yml +++ b/docker/dev/cassandra-opensearch-kafka.yml @@ -15,7 +15,7 @@ services: environment: - discovery.type=single-node kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: @@ -25,6 +25,6 @@ services: KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 ports: - "2181:2181" diff --git a/docker/dev/cassandra-pinot-kafka.yml b/docker/dev/cassandra-pinot-kafka.yml index f1cfa2bccd3..8a9afec5a1e 100644 --- a/docker/dev/cassandra-pinot-kafka.yml +++ b/docker/dev/cassandra-pinot-kafka.yml @@ -49,7 +49,7 @@ services: depends_on: - pinot-broker kafka: - image: wurstmeister/kafka:2.13-2.8.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: diff --git a/docker/dev/mongo-esv7-kafka.yml b/docker/dev/mongo-esv7-kafka.yml index 0eb6be347d8..2cfc707fae1 100644 --- a/docker/dev/mongo-esv7-kafka.yml +++ b/docker/dev/mongo-esv7-kafka.yml @@ -25,7 +25,7 @@ services: environment: - discovery.type=single-node kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: @@ -35,6 +35,6 @@ services: KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 ports: - - "2181:2181" \ No newline at end of file + - "2181:2181" diff --git a/docker/dev/mysql-esv7-kafka.yml b/docker/dev/mysql-esv7-kafka.yml index adb03e1ce5c..430e450d6f1 100644 --- a/docker/dev/mysql-esv7-kafka.yml +++ b/docker/dev/mysql-esv7-kafka.yml @@ -13,7 +13,7 @@ services: environment: - discovery.type=single-node kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: @@ -23,6 +23,6 @@ services: KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 ports: - - "2181:2181" \ No newline at end of file + - "2181:2181" diff --git a/docker/docker-compose-async-wf-kafka.yml b/docker/docker-compose-async-wf-kafka.yml index c88edd1d649..a449494d6aa 100644 --- a/docker/docker-compose-async-wf-kafka.yml +++ b/docker/docker-compose-async-wf-kafka.yml @@ -75,11 +75,11 @@ services: ports: - '3000:3000' zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 ports: - "2181:2181" kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: diff --git a/docker/docker-compose-es-v7.yml b/docker/docker-compose-es-v7.yml index fe2761e74f5..1b0fbb79dd7 100644 --- a/docker/docker-compose-es-v7.yml +++ b/docker/docker-compose-es-v7.yml @@ -21,11 +21,11 @@ services: ports: - '9090:9090' zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 ports: - "2181:2181" kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: diff --git a/docker/docker-compose-es.yml b/docker/docker-compose-es.yml index 0f147ff1987..f0e686dec98 100644 --- a/docker/docker-compose-es.yml +++ b/docker/docker-compose-es.yml @@ -21,11 +21,11 @@ services: ports: - '9090:9090' zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 ports: - "2181:2181" kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: diff --git a/docker/docker-compose-multiclusters-cass-mysql-es.yaml b/docker/docker-compose-multiclusters-cass-mysql-es.yaml index 9adec5cf889..6c959e0c15b 100644 --- a/docker/docker-compose-multiclusters-cass-mysql-es.yaml +++ b/docker/docker-compose-multiclusters-cass-mysql-es.yaml @@ -27,11 +27,11 @@ services: ports: - '9090:9090' zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 ports: - "2181:2181" kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: diff --git a/docker/docker-compose-multiclusters-es.yml b/docker/docker-compose-multiclusters-es.yml index 70986eebe25..4b9e7de42ff 100644 --- a/docker/docker-compose-multiclusters-es.yml +++ b/docker/docker-compose-multiclusters-es.yml @@ -21,11 +21,11 @@ services: ports: - '9090:9090' zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: ubercadence/wurstmeister-zookeeper:3.4.6 ports: - "2181:2181" kafka: - image: wurstmeister/kafka:2.12-2.1.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: diff --git a/docker/docker-compose-pinot.yml b/docker/docker-compose-pinot.yml index 1cbeada2681..4c9853468d7 100644 --- a/docker/docker-compose-pinot.yml +++ b/docker/docker-compose-pinot.yml @@ -13,7 +13,7 @@ services: ports: - '9090:9090' kafka: - image: wurstmeister/kafka:2.13-2.8.1 + image: ubercadence/wurstmeister-kafka:2.12-2.1.1 depends_on: - zookeeper ports: From 3860de275c9e7a0f4cb59f333cf27e4b003002aa Mon Sep 17 00:00:00 2001 From: taylan isikdemir Date: Tue, 7 May 2024 09:53:49 -0700 Subject: [PATCH 2/4] try bitnami kafka image --- docker/buildkite/docker-compose-es7.yml | 32 ++++++++------- .../docker-compose-local-async-wf.yml | 39 +++++++++---------- docker/buildkite/docker-compose-local-es7.yml | 32 +++++++-------- docker/buildkite/docker-compose-local.yml | 36 ++++++++--------- .../buildkite/docker-compose-opensearch2.yml | 32 ++++++++------- docker/buildkite/docker-compose.yml | 35 +++++++++-------- docker/dev/cassandra-esv7-kafka.yml | 25 +++++++----- docker/dev/cassandra-opensearch-kafka.yml | 25 +++++++----- docker/dev/cassandra-pinot-kafka.yml | 28 +++++++------ docker/dev/mongo-esv7-kafka.yml | 25 +++++++----- docker/dev/mysql-esv7-kafka.yml | 25 +++++++----- docker/docker-compose-async-wf-kafka.yml | 28 ++++++------- docker/docker-compose-es-v7.yml | 25 +++++++----- docker/docker-compose-es.yml | 25 +++++++----- ...r-compose-multiclusters-cass-mysql-es.yaml | 25 +++++++----- docker/docker-compose-multiclusters-es.yml | 25 +++++++----- docker/docker-compose-pinot.yml | 26 +++++++------ 17 files changed, 272 insertions(+), 216 deletions(-) diff --git a/docker/buildkite/docker-compose-es7.yml b/docker/buildkite/docker-compose-es7.yml index 721a6e483ed..272b05bcaf4 100644 --- a/docker/buildkite/docker-compose-es7.yml +++ b/docker/buildkite/docker-compose-es7.yml @@ -16,25 +16,29 @@ services: timeout: 30s retries: 10 - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - networks: - services-network: - aliases: - - zookeeper - kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka + ports: + - "9092:9092" + environment: + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" networks: services-network: aliases: - kafka - environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.9.3 diff --git a/docker/buildkite/docker-compose-local-async-wf.yml b/docker/buildkite/docker-compose-local-async-wf.yml index 039b857e64b..3021fcd7f5e 100644 --- a/docker/buildkite/docker-compose-local-async-wf.yml +++ b/docker/buildkite/docker-compose-local-async-wf.yml @@ -1,4 +1,4 @@ -version: "3.5" +# version: "3.5" services: cassandra: @@ -18,32 +18,29 @@ services: timeout: 30s retries: 10 - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - ports: - - "2181:2181" - networks: - services-network: - aliases: - - zookeeper - kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" + environment: + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" networks: services-network: aliases: - - kafka - environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - # create a topic with 10 partitions and 1 replica - # topic name must match ASYNC_WF_KAFKA_QUEUE_TOPIC specified in cadence container above - KAFKA_CREATE_TOPICS: "async-wf-topic1:10:1" + - integration-test integration-test-async-wf: build: diff --git a/docker/buildkite/docker-compose-local-es7.yml b/docker/buildkite/docker-compose-local-es7.yml index 72c6b4158bd..ef3331e1a7e 100644 --- a/docker/buildkite/docker-compose-local-es7.yml +++ b/docker/buildkite/docker-compose-local-es7.yml @@ -18,29 +18,29 @@ services: timeout: 30s retries: 10 - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - ports: - - "2181:2181" - networks: - services-network: - aliases: - - zookeeper - kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" + environment: + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" networks: services-network: aliases: - kafka - environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.9.3 diff --git a/docker/buildkite/docker-compose-local.yml b/docker/buildkite/docker-compose-local.yml index 113ad3a9388..bd15ad4309d 100644 --- a/docker/buildkite/docker-compose-local.yml +++ b/docker/buildkite/docker-compose-local.yml @@ -44,29 +44,29 @@ services: aliases: - postgres - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - expose: - - "2181" - networks: - services-network: - aliases: - - zookeeper - kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper - expose: - - "9092" + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka + ports: + - "9092:9092" + environment: + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" networks: services-network: aliases: - kafka - environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.22 diff --git a/docker/buildkite/docker-compose-opensearch2.yml b/docker/buildkite/docker-compose-opensearch2.yml index a8affd72dcc..727e32d659c 100644 --- a/docker/buildkite/docker-compose-opensearch2.yml +++ b/docker/buildkite/docker-compose-opensearch2.yml @@ -16,25 +16,29 @@ services: timeout: 30s retries: 10 - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - networks: - services-network: - aliases: - - zookeeper - kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka + ports: + - "9092:9092" + environment: + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" networks: services-network: aliases: - kafka - environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 elasticsearch: image: opensearchproject/opensearch:2.5.0 diff --git a/docker/buildkite/docker-compose.yml b/docker/buildkite/docker-compose.yml index e4644075a88..0eca8b67c56 100644 --- a/docker/buildkite/docker-compose.yml +++ b/docker/buildkite/docker-compose.yml @@ -36,28 +36,29 @@ services: aliases: - postgres - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - networks: - services-network: - aliases: - - zookeeper - kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka + ports: + - "9092:9092" + environment: + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" networks: services-network: aliases: - kafka - environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - # for async wf tests, create a topic with 10 partitions and 1 replica - # topic name must match ASYNC_WF_KAFKA_QUEUE_TOPIC - KAFKA_CREATE_TOPICS: "async-wf-topic1:10:1" elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.22 diff --git a/docker/dev/cassandra-esv7-kafka.yml b/docker/dev/cassandra-esv7-kafka.yml index de15e64ad62..115af4961f6 100644 --- a/docker/dev/cassandra-esv7-kafka.yml +++ b/docker/dev/cassandra-esv7-kafka.yml @@ -14,16 +14,21 @@ services: environment: - discovery.type=single-node kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - ports: - - "2181:2181" + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" diff --git a/docker/dev/cassandra-opensearch-kafka.yml b/docker/dev/cassandra-opensearch-kafka.yml index 73e311417bc..52542cdfb24 100644 --- a/docker/dev/cassandra-opensearch-kafka.yml +++ b/docker/dev/cassandra-opensearch-kafka.yml @@ -15,16 +15,21 @@ services: environment: - discovery.type=single-node kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - ports: - - "2181:2181" + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" diff --git a/docker/dev/cassandra-pinot-kafka.yml b/docker/dev/cassandra-pinot-kafka.yml index 8a9afec5a1e..30f4ecdeaa7 100644 --- a/docker/dev/cassandra-pinot-kafka.yml +++ b/docker/dev/cassandra-pinot-kafka.yml @@ -48,21 +48,25 @@ services: JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-server.log" depends_on: - pinot-broker - kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + kafka: + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" - expose: - - "9093" environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181/kafka - KAFKA_BROKER_ID: 0 - KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,OUTSIDE://localhost:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.9.3 ports: diff --git a/docker/dev/mongo-esv7-kafka.yml b/docker/dev/mongo-esv7-kafka.yml index 2cfc707fae1..35af14d4376 100644 --- a/docker/dev/mongo-esv7-kafka.yml +++ b/docker/dev/mongo-esv7-kafka.yml @@ -25,16 +25,21 @@ services: environment: - discovery.type=single-node kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - ports: - - "2181:2181" + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" diff --git a/docker/dev/mysql-esv7-kafka.yml b/docker/dev/mysql-esv7-kafka.yml index 430e450d6f1..72100f50427 100644 --- a/docker/dev/mysql-esv7-kafka.yml +++ b/docker/dev/mysql-esv7-kafka.yml @@ -13,16 +13,21 @@ services: environment: - discovery.type=single-node kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - ports: - - "2181:2181" + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" diff --git a/docker/docker-compose-async-wf-kafka.yml b/docker/docker-compose-async-wf-kafka.yml index a449494d6aa..278e330b1bd 100644 --- a/docker/docker-compose-async-wf-kafka.yml +++ b/docker/docker-compose-async-wf-kafka.yml @@ -74,20 +74,22 @@ services: - prometheus ports: - '3000:3000' - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - ports: - - "2181:2181" kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - # create a topic with 10 partitions and 1 replica - # topic name must match ASYNC_WF_KAFKA_QUEUE_TOPIC specified in cadence container above - KAFKA_CREATE_TOPICS: "async-wf-topic1:10:1" + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" diff --git a/docker/docker-compose-es-v7.yml b/docker/docker-compose-es-v7.yml index 1b0fbb79dd7..f11200bdc0e 100644 --- a/docker/docker-compose-es-v7.yml +++ b/docker/docker-compose-es-v7.yml @@ -20,20 +20,25 @@ services: - '--config.file=/etc/prometheus/prometheus.yml' ports: - '9090:9090' - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - ports: - - "2181:2181" kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.9.3 ports: diff --git a/docker/docker-compose-es.yml b/docker/docker-compose-es.yml index f0e686dec98..ad5fc7696ca 100644 --- a/docker/docker-compose-es.yml +++ b/docker/docker-compose-es.yml @@ -20,20 +20,25 @@ services: - '--config.file=/etc/prometheus/prometheus.yml' ports: - '9090:9090' - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - ports: - - "2181:2181" kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.22 ports: diff --git a/docker/docker-compose-multiclusters-cass-mysql-es.yaml b/docker/docker-compose-multiclusters-cass-mysql-es.yaml index 6c959e0c15b..5f967b8b7fc 100644 --- a/docker/docker-compose-multiclusters-cass-mysql-es.yaml +++ b/docker/docker-compose-multiclusters-cass-mysql-es.yaml @@ -26,20 +26,25 @@ services: - '--config.file=/etc/prometheus/prometheus.yml' ports: - '9090:9090' - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - ports: - - "2181:2181" kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.22 ports: diff --git a/docker/docker-compose-multiclusters-es.yml b/docker/docker-compose-multiclusters-es.yml index 4b9e7de42ff..6ccd8f749b7 100644 --- a/docker/docker-compose-multiclusters-es.yml +++ b/docker/docker-compose-multiclusters-es.yml @@ -20,20 +20,25 @@ services: - '--config.file=/etc/prometheus/prometheus.yml' ports: - '9090:9090' - zookeeper: - image: ubercadence/wurstmeister-zookeeper:3.4.6 - ports: - - "2181:2181" kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.22 ports: diff --git a/docker/docker-compose-pinot.yml b/docker/docker-compose-pinot.yml index 4c9853468d7..6bd7be3dcf2 100644 --- a/docker/docker-compose-pinot.yml +++ b/docker/docker-compose-pinot.yml @@ -13,20 +13,24 @@ services: ports: - '9090:9090' kafka: - image: ubercadence/wurstmeister-kafka:2.12-2.1.1 - depends_on: - - zookeeper + image: docker.io/bitnami/kafka:3.7 + hostname: kafka + container_name: kafka ports: - "9092:9092" - expose: - - "9093" environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181/kafka - KAFKA_BROKER_ID: 0 - KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,OUTSIDE://localhost:9092 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT + # KRaft settings + - "KAFKA_CFG_NODE_ID=0" + - "KAFKA_CFG_PROCESS_ROLES=controller,broker" + - "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093" + # Listeners + - "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093" + - "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092" + - "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER" + - "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT" + # Topic settings + - "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true" zookeeper: image: zookeeper:3.5.8 container_name: zookeeper From 98a86a3c989397745d4d106a5c0082919029c166 Mon Sep 17 00:00:00 2001 From: taylan isikdemir Date: Tue, 7 May 2024 09:58:42 -0700 Subject: [PATCH 3/4] fix --- docker/buildkite/docker-compose-local-async-wf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/buildkite/docker-compose-local-async-wf.yml b/docker/buildkite/docker-compose-local-async-wf.yml index 3021fcd7f5e..41f897e4cc2 100644 --- a/docker/buildkite/docker-compose-local-async-wf.yml +++ b/docker/buildkite/docker-compose-local-async-wf.yml @@ -40,7 +40,7 @@ services: networks: services-network: aliases: - - integration-test + - kafka integration-test-async-wf: build: From de28065b83a79cbc4dfa569ed05c170cad95c240 Mon Sep 17 00:00:00 2001 From: taylan isikdemir Date: Tue, 7 May 2024 11:02:47 -0700 Subject: [PATCH 4/4] revert version --- docker/buildkite/docker-compose-local-async-wf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/buildkite/docker-compose-local-async-wf.yml b/docker/buildkite/docker-compose-local-async-wf.yml index 41f897e4cc2..88f497d18e4 100644 --- a/docker/buildkite/docker-compose-local-async-wf.yml +++ b/docker/buildkite/docker-compose-local-async-wf.yml @@ -1,4 +1,4 @@ -# version: "3.5" +version: "3.5" services: cassandra: