Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace wurstmeister kafka/zookeeper images with bitnami kafka image #5975

Merged
merged 4 commits into from
May 7, 2024
Merged

Replace wurstmeister kafka/zookeeper images with bitnami kafka image #5975

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
31e55d6
Temporary workaround for broken wurstmeister kafka/zookeeper images
taylanisikdemir May 7, 2024
3860de2
try bitnami kafka image
taylanisikdemir May 7, 2024
98a86a3
fix
taylanisikdemir May 7, 2024
de28065
revert version
taylanisikdemir May 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 18 additions & 14 deletions docker/buildkite/docker-compose-es7.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,29 @@ services:
timeout: 30s
retries: 10

zookeeper:
image: wurstmeister/zookeeper:3.4.6
networks:
services-network:
aliases:
- zookeeper

kafka:
image: wurstmeister/kafka:2.12-2.1.1
depends_on:
- zookeeper
image: docker.io/bitnami/kafka:3.7
hostname: kafka
container_name: kafka
ports:
- "9092:9092"
environment:
# KRaft settings
- "KAFKA_CFG_NODE_ID=0"
- "KAFKA_CFG_PROCESS_ROLES=controller,broker"
- "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093"
# Listeners
- "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093"
- "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092"
- "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
- "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER"
- "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT"
# Topic settings
- "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true"
networks:
services-network:
aliases:
- kafka
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181

elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.9.3
Expand Down
35 changes: 16 additions & 19 deletions docker/buildkite/docker-compose-local-async-wf.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,32 +18,29 @@ services:
timeout: 30s
retries: 10

zookeeper:
image: wurstmeister/zookeeper:3.4.6
ports:
- "2181:2181"
networks:
services-network:
aliases:
- zookeeper

kafka:
image: wurstmeister/kafka:2.12-2.1.1
depends_on:
- zookeeper
image: docker.io/bitnami/kafka:3.7
hostname: kafka
container_name: kafka
ports:
- "9092:9092"
environment:
# KRaft settings
- "KAFKA_CFG_NODE_ID=0"
- "KAFKA_CFG_PROCESS_ROLES=controller,broker"
- "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093"
# Listeners
- "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093"
- "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092"
- "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
- "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER"
- "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT"
# Topic settings
- "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true"
networks:
services-network:
aliases:
- kafka
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
# create a topic with 10 partitions and 1 replica
# topic name must match ASYNC_WF_KAFKA_QUEUE_TOPIC specified in cadence container above
KAFKA_CREATE_TOPICS: "async-wf-topic1:10:1"

integration-test-async-wf:
build:
Expand Down
32 changes: 16 additions & 16 deletions docker/buildkite/docker-compose-local-es7.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,29 +18,29 @@ services:
timeout: 30s
retries: 10

zookeeper:
image: wurstmeister/zookeeper:3.4.6
ports:
- "2181:2181"
networks:
services-network:
aliases:
- zookeeper

kafka:
image: wurstmeister/kafka:2.12-2.1.1
depends_on:
- zookeeper
image: docker.io/bitnami/kafka:3.7
hostname: kafka
container_name: kafka
ports:
- "9092:9092"
environment:
# KRaft settings
- "KAFKA_CFG_NODE_ID=0"
- "KAFKA_CFG_PROCESS_ROLES=controller,broker"
- "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093"
# Listeners
- "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093"
- "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092"
- "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
- "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER"
- "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT"
# Topic settings
- "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true"
networks:
services-network:
aliases:
- kafka
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181

elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.9.3
Expand Down
36 changes: 18 additions & 18 deletions docker/buildkite/docker-compose-local.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,29 +44,29 @@ services:
aliases:
- postgres

zookeeper:
image: wurstmeister/zookeeper:3.4.6
expose:
- "2181"
networks:
services-network:
aliases:
- zookeeper

kafka:
image: wurstmeister/kafka:2.12-2.1.1
depends_on:
- zookeeper
expose:
- "9092"
image: docker.io/bitnami/kafka:3.7
hostname: kafka
container_name: kafka
ports:
- "9092:9092"
environment:
# KRaft settings
- "KAFKA_CFG_NODE_ID=0"
- "KAFKA_CFG_PROCESS_ROLES=controller,broker"
- "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093"
# Listeners
- "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093"
- "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092"
- "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
- "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER"
- "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT"
# Topic settings
- "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true"
networks:
services-network:
aliases:
- kafka
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181

elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.22
Expand Down
32 changes: 18 additions & 14 deletions docker/buildkite/docker-compose-opensearch2.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,29 @@ services:
timeout: 30s
retries: 10

zookeeper:
image: wurstmeister/zookeeper:3.4.6
networks:
services-network:
aliases:
- zookeeper

kafka:
image: wurstmeister/kafka:2.12-2.1.1
depends_on:
- zookeeper
image: docker.io/bitnami/kafka:3.7
hostname: kafka
container_name: kafka
ports:
- "9092:9092"
environment:
# KRaft settings
- "KAFKA_CFG_NODE_ID=0"
- "KAFKA_CFG_PROCESS_ROLES=controller,broker"
- "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093"
# Listeners
- "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093"
- "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092"
- "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
- "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER"
- "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT"
# Topic settings
- "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true"
networks:
services-network:
aliases:
- kafka
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181

elasticsearch:
image: opensearchproject/opensearch:2.5.0
Expand Down
35 changes: 18 additions & 17 deletions docker/buildkite/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,28 +36,29 @@ services:
aliases:
- postgres

zookeeper:
image: wurstmeister/zookeeper:3.4.6
networks:
services-network:
aliases:
- zookeeper

kafka:
image: wurstmeister/kafka:2.12-2.1.1
depends_on:
- zookeeper
image: docker.io/bitnami/kafka:3.7
hostname: kafka
container_name: kafka
ports:
- "9092:9092"
environment:
# KRaft settings
- "KAFKA_CFG_NODE_ID=0"
- "KAFKA_CFG_PROCESS_ROLES=controller,broker"
- "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093"
# Listeners
- "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093"
- "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092"
- "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
- "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER"
- "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT"
# Topic settings
- "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true"
networks:
services-network:
aliases:
- kafka
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
# for async wf tests, create a topic with 10 partitions and 1 replica
# topic name must match ASYNC_WF_KAFKA_QUEUE_TOPIC
KAFKA_CREATE_TOPICS: "async-wf-topic1:10:1"

elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.22
Expand Down
25 changes: 15 additions & 10 deletions docker/dev/cassandra-esv7-kafka.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,21 @@ services:
environment:
- discovery.type=single-node
kafka:
image: wurstmeister/kafka:2.12-2.1.1
depends_on:
- zookeeper
image: docker.io/bitnami/kafka:3.7
hostname: kafka
container_name: kafka
ports:
- "9092:9092"
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
zookeeper:
image: wurstmeister/zookeeper:3.4.6
ports:
- "2181:2181"
# KRaft settings
- "KAFKA_CFG_NODE_ID=0"
- "KAFKA_CFG_PROCESS_ROLES=controller,broker"
- "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093"
# Listeners
- "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093"
- "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092"
- "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
- "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER"
- "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT"
# Topic settings
- "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true"
25 changes: 15 additions & 10 deletions docker/dev/cassandra-opensearch-kafka.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,21 @@ services:
environment:
- discovery.type=single-node
kafka:
image: wurstmeister/kafka:2.12-2.1.1
depends_on:
- zookeeper
image: docker.io/bitnami/kafka:3.7
hostname: kafka
container_name: kafka
ports:
- "9092:9092"
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
zookeeper:
image: wurstmeister/zookeeper:3.4.6
ports:
- "2181:2181"
# KRaft settings
- "KAFKA_CFG_NODE_ID=0"
- "KAFKA_CFG_PROCESS_ROLES=controller,broker"
- "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093"
# Listeners
- "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093"
- "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092"
- "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
- "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER"
- "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT"
# Topic settings
- "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true"
28 changes: 16 additions & 12 deletions docker/dev/cassandra-pinot-kafka.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,21 +48,25 @@ services:
JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-server.log"
depends_on:
- pinot-broker
kafka:
image: wurstmeister/kafka:2.13-2.8.1
depends_on:
- zookeeper
kafka:
image: docker.io/bitnami/kafka:3.7
hostname: kafka
container_name: kafka
ports:
- "9092:9092"
expose:
- "9093"
environment:
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181/kafka
KAFKA_BROKER_ID: 0
KAFKA_ADVERTISED_HOST_NAME: kafka
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,OUTSIDE://localhost:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
# KRaft settings
- "KAFKA_CFG_NODE_ID=0"
- "KAFKA_CFG_PROCESS_ROLES=controller,broker"
- "KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093"
# Listeners
- "KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093"
- "KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092"
- "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
- "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER"
- "KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT"
# Topic settings
- "KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true"
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.9.3
ports:
Expand Down
Loading
Loading