diff --git a/README.md b/README.md index d4da96ac..b98121aa 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,13 @@ If you begin to rely on this kafka setup we recommend you fork, for example to e ## Version history -| tag | k8s ≥ | highlights | +| tag | k8s ≥ | highlights | | ----- | ------ | ---------- | -| 4.x | 1.9+ | Kafka 1.1 dynamic config | -| v4.1 | 1.9+ | Kafka 1.0.1 new [default](#148) [config](#170) | -| v3.2 | 1.9.4, 1.8.9, 1.7.14 | Required for read-only ConfigMaps [#162](https://github.com/Yolean/kubernetes-kafka/issues/162) [#163](https://github.com/Yolean/kubernetes-kafka/pull/163) [k8s #58720](https://github.com/kubernetes/kubernetes/pull/58720) | +| v5.0 | 1.11+ | Destabilize because in Docker we want Java 11 [#197](https://github.com/Yolean/kubernetes-kafka/pull/197) [#191](https://github.com/Yolean/kubernetes-kafka/pull/191) | +| v4.3 | 1.9+ | Adds a prpper shutdown hook [207](https://github.com/Yolean/kubernetes-kafka/pull/207) | +| v4.2 | 1.9+ | Kafka 1.0.2 and tools upgrade | +| v4.1 | 1.9+ | Kafka 1.0.1 new [default](#148) [config](#170) | +| v3.2 | 1.9.4, 1.8.9, 1.7.14 | Required for read-only ConfigMaps [#162](https://github.com/Yolean/kubernetes-kafka/issues/162) [#163](https://github.com/Yolean/kubernetes-kafka/pull/163) [k8s #58720](https://github.com/kubernetes/kubernetes/pull/58720) | | v3.1 | 1.8 | The painstaking path to `min.insync.replicas`=2 | | v3.0 | 1.8 | [Outside access](#78), [modern manifests](#84), [bootstrap.kafka](#52) | | v2.1 | 1.5 | Kafka 1.0, the init script concept | diff --git a/avro-tools/rest.yml b/avro-tools/rest.yml index be299e6a..1f02cefe 100644 --- a/avro-tools/rest.yml +++ b/avro-tools/rest.yml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: avro-rest diff --git a/avro-tools/schemas.yml b/avro-tools/schemas.yml index b811d17d..964f988e 100644 --- a/avro-tools/schemas.yml +++ b/avro-tools/schemas.yml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: avro-schemas diff --git a/avro-tools/test/rest-curl.yml b/avro-tools/test/rest-curl.yml index 7a42219b..2cdadd24 100644 --- a/avro-tools/test/rest-curl.yml +++ b/avro-tools/test/rest-curl.yml @@ -114,7 +114,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 command: - ./bin/kafka-topics.sh - --zookeeper @@ -129,7 +129,7 @@ spec: - "1" restartPolicy: Never --- -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: rest-curl diff --git a/events-kube/events-kube-kafka.yml b/events-kube/events-kube-kafka.yml index 41fe35e3..6a90e667 100644 --- a/events-kube/events-kube-kafka.yml +++ b/events-kube/events-kube-kafka.yml @@ -30,7 +30,7 @@ spec: serviceAccount: eventrouter containers: - name: kube-eventrouter - image: gcr.io/heptio-images/eventrouter@sha256:30e36ce7bad4a7c539e0a0cb1833d309089919fb0ef0c165ee28aabe97740d02 + image: gcr.io/heptio-images/eventrouter@sha256:e613b48c6235426fa334867d661118322b4e1973c23e2e8cf5c066b982cc8596 resources: requests: memory: "5Mi" diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index 659c9d4d..41940062 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -20,7 +20,7 @@ spec: terminationGracePeriodSeconds: 30 initContainers: - name: init-config - image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2 + image: solsson/kafka-initutils@sha256:2cdb90ea514194d541c7b869ac15d2d530ca64889f56e270161fe4e5c3d076ea env: - name: NODE_NAME valueFrom: @@ -42,7 +42,7 @@ spec: mountPath: /etc/kafka containers: - name: broker - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties @@ -61,7 +61,12 @@ spec: resources: requests: cpu: 100m - memory: 512Mi + memory: 100Mi + limits: + # This limit was intentionally set low as a reminder that + # the entire Yolean/kubernetes-kafka is meant to be tweaked + # before you run production workloads + memory: 600Mi readinessProbe: tcpSocket: port: 9092 diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index f551fbdd..7f2eca6b 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -72,7 +72,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 command: - ./bin/kafka-topics.sh - --zookeeper @@ -87,7 +87,7 @@ spec: - "3" restartPolicy: Never --- -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: ReplicaSet metadata: name: kafkacat diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index d5aaf8b6..44e54db2 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -55,7 +55,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 command: - ./bin/kafka-topics.sh - --zookeeper @@ -70,7 +70,7 @@ spec: - "3" restartPolicy: Never --- -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: ReplicaSet metadata: name: produce-consume @@ -95,7 +95,7 @@ spec: spec: containers: - name: producer - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 @@ -114,7 +114,7 @@ spec: - name: shared mountPath: /shared - name: consumer - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 diff --git a/kafka/test/replication-config.yml b/kafka/test/replication-config.yml index ec528071..68c0872a 100644 --- a/kafka/test/replication-config.yml +++ b/kafka/test/replication-config.yml @@ -45,7 +45,7 @@ data: exit 0 --- -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: ReplicaSet metadata: name: replication-config diff --git a/linkedin-burrow/burrow.yml b/linkedin-burrow/burrow.yml index c6a3cf3e..1e9ff47c 100644 --- a/linkedin-burrow/burrow.yml +++ b/linkedin-burrow/burrow.yml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: burrow diff --git a/maintenance/preferred-replica-election-job.yml b/maintenance/preferred-replica-election-job.yml index f3cfa83e..704d9fc0 100644 --- a/maintenance/preferred-replica-election-job.yml +++ b/maintenance/preferred-replica-election-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 command: - ./bin/kafka-preferred-replica-election.sh - --zookeeper diff --git a/maintenance/reassign-paritions-job.yml b/maintenance/reassign-paritions-job.yml index 317d9110..e56e21bd 100644 --- a/maintenance/reassign-paritions-job.yml +++ b/maintenance/reassign-paritions-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/maintenance/replication-factor-increase-job.yml b/maintenance/replication-factor-increase-job.yml index 00a13ae8..48742722 100644 --- a/maintenance/replication-factor-increase-job.yml +++ b/maintenance/replication-factor-increase-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/maintenance/test/replicated-partitions.yml b/maintenance/test/replicated-partitions.yml index fc535371..72728b7d 100644 --- a/maintenance/test/replicated-partitions.yml +++ b/maintenance/test/replicated-partitions.yml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: replicated-partitions diff --git a/pixy/pixy.yml b/pixy/pixy.yml index 7226aa6b..8d4ea2c4 100644 --- a/pixy/pixy.yml +++ b/pixy/pixy.yml @@ -17,7 +17,7 @@ spec: spec: containers: - name: pixy - image: mailgun/kafka-pixy:0.15.0@sha256:088210d53945a0db5f93921ceff3a79c012449b7845baebe8898452741764e7c + image: mailgun/kafka-pixy:0.16.0@sha256:2a2d426f9fe17ede9c21b2f8b7418d8787293e39327b638fd6c403f3726597fb ports: - containerPort: 80 command: diff --git a/prometheus/50-kafka-jmx-exporter-patch.yml b/prometheus/50-kafka-jmx-exporter-patch.yml index 7876ef47..179cbb5e 100644 --- a/prometheus/50-kafka-jmx-exporter-patch.yml +++ b/prometheus/50-kafka-jmx-exporter-patch.yml @@ -1,6 +1,6 @@ # meant to be applied using # kubectl --namespace kafka patch statefulset kafka --patch "$(cat prometheus/50-kafka-jmx-exporter-patch.yml )" -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: StatefulSet metadata: name: kafka @@ -14,13 +14,9 @@ spec: spec: containers: - name: metrics - image: solsson/kafka-prometheus-jmx-exporter@sha256:a23062396cd5af1acdf76512632c20ea6be76885dfc20cd9ff40fb23846557e8 + image: solsson/kafka-prometheus-jmx-exporter@sha256:d237a12cc0cde42b539bcb5efc0008ba5e6ca1351b7843ed52bd574d181c5efd command: - java - - -XX:+UnlockExperimentalVMOptions - - -XX:+UseCGroupMemoryLimitForHeap - - -XX:MaxRAMFraction=1 - - -XshowSettings:vm - -jar - jmx_prometheus_httpserver.jar - "5556" diff --git a/yahoo-kafka-manager/kafka-manager.yml b/yahoo-kafka-manager/kafka-manager.yml index b801982f..8d659041 100644 --- a/yahoo-kafka-manager/kafka-manager.yml +++ b/yahoo-kafka-manager/kafka-manager.yml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: kafka-manager diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index d923ee97..ecf88926 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -22,7 +22,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2 + image: solsson/kafka-initutils@sha256:2cdb90ea514194d541c7b869ac15d2d530ca64889f56e270161fe4e5c3d076ea command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] volumeMounts: - name: configmap @@ -33,7 +33,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties @@ -51,6 +51,8 @@ spec: requests: cpu: 10m memory: 100Mi + limits: + memory: 100Mi readinessProbe: exec: command: diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 88b781af..aea5763d 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -22,7 +22,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2 + image: solsson/kafka-initutils@sha256:2cdb90ea514194d541c7b869ac15d2d530ca64889f56e270161fe4e5c3d076ea command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] env: - name: ID_OFFSET @@ -36,7 +36,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties @@ -54,6 +54,8 @@ spec: requests: cpu: 10m memory: 100Mi + limits: + memory: 100Mi readinessProbe: exec: command: