From e4465207185f4792556ad7d9bcfbe4a5bc8790ed Mon Sep 17 00:00:00 2001 From: Koenraad Verheyden Date: Mon, 21 Jun 2021 11:04:58 +0200 Subject: [PATCH] Jsonnet: use dedicated configmaps for distributors and ingesters (#775) * Jsonnet: make dedicated configmap for distributor and ingester * Update CHANGELOG.md * Move distributor config into its own configmap * Update kube-manifests Signed-off-by: Annanay Co-authored-by: Annanay --- CHANGELOG.md | 1 + .../jsonnet/microservices/configmap.libsonnet | 33 +++++++++--- .../microservices/distributor.libsonnet | 4 +- .../jsonnet/microservices/ingester.libsonnet | 5 +- .../ConfigMap-tempo-compactor.yaml | 11 +--- .../ConfigMap-tempo-distributor.yaml | 52 +++++++++++++++++++ .../ConfigMap-tempo-ingester.yaml | 43 +++++++++++++++ .../ConfigMap-tempo-querier.yaml | 11 +--- .../ConfigMap-tempo-query-frontend.yaml | 11 +--- .../kube-manifests/Deployment-compactor.yaml | 2 +- .../Deployment-distributor.yaml | 2 +- .../kube-manifests/Deployment-querier.yaml | 2 +- .../Deployment-query-frontend.yaml | 2 +- .../kube-manifests/StatefulSet-ingester.yaml | 4 +- .../kube-manifests/StatefulSet-memcached.yaml | 3 +- 15 files changed, 139 insertions(+), 47 deletions(-) create mode 100644 operations/kube-manifests/ConfigMap-tempo-distributor.yaml create mode 100644 operations/kube-manifests/ConfigMap-tempo-ingester.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 31eaeb87682..d1df44a4a63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## main / unreleased +* [CHANGE] Jsonnet: use dedicated configmaps for distributors and ingesters [#775](https://github.com/grafana/tempo/pull/775) (@kvrhdn) * [FEATURE] Added the ability to hedge requests with all backends [#750](https://github.com/grafana/tempo/pull/750) (@joe-elliott) * [ENHANCEMENT] Performance: improve compaction speed with concurrent reads and writes [#754](https://github.com/grafana/tempo/pull/754) (@mdisibio) * [ENHANCEMENT] Improve readability of cpu and memory metrics on operational dashboard [#764](https://github.com/grafana/tempo/pull/764) (@bboreham) diff --git a/operations/jsonnet/microservices/configmap.libsonnet b/operations/jsonnet/microservices/configmap.libsonnet index 52a4292dc3e..35ca2640993 100644 --- a/operations/jsonnet/microservices/configmap.libsonnet +++ b/operations/jsonnet/microservices/configmap.libsonnet @@ -5,9 +5,7 @@ server: { http_listen_port: $._config.port, }, - distributor: { - receivers: $._config.distributor.receivers, - }, + distributor: null, ingester: { lifecycler: { ring: { @@ -52,6 +50,14 @@ }, }, + tempo_distributor_config:: $.tempo_config { + distributor: { + receivers: $._config.distributor.receivers, + }, + }, + + tempo_ingester_config:: $.tempo_config{}, + tempo_compactor_config:: $.tempo_config { compactor: { compaction: { @@ -90,10 +96,23 @@ } }, - tempo_configmap: - configMap.new('tempo') + + tempo_query_frontend_config:: $.tempo_config{}, + + tempo_distributor_configmap: + configMap.new('tempo-distributor') + configMap.withData({ - 'tempo.yaml': $.util.manifestYaml($.tempo_config), + 'tempo.yaml': $.util.manifestYaml($.tempo_distributor_config), + }) + + configMap.withDataMixin({ + 'overrides.yaml': $.util.manifestYaml({ + overrides: $._config.overrides, + }), + }), + + tempo_ingester_configmap: + configMap.new('tempo-ingester') + + configMap.withData({ + 'tempo.yaml': $.util.manifestYaml($.tempo_ingester_config), }) + configMap.withDataMixin({ 'overrides.yaml': $.util.manifestYaml({ @@ -112,8 +131,6 @@ }), }), - tempo_query_frontend_config:: $.tempo_config{}, - tempo_querier_configmap: configMap.new('tempo-querier') + configMap.withData({ diff --git a/operations/jsonnet/microservices/distributor.libsonnet b/operations/jsonnet/microservices/distributor.libsonnet index e876c3387e6..4e5a6f3dd5c 100644 --- a/operations/jsonnet/microservices/distributor.libsonnet +++ b/operations/jsonnet/microservices/distributor.libsonnet @@ -40,10 +40,10 @@ deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) + deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(60) + deployment.mixin.spec.template.metadata.withAnnotations({ - config_hash: std.md5(std.toString($.tempo_configmap.data['tempo.yaml'])), + config_hash: std.md5(std.toString($.tempo_distributor_configmap.data['tempo.yaml'])), }) + deployment.mixin.spec.template.spec.withVolumes([ - volume.fromConfigMap(tempo_config_volume, $.tempo_configmap.metadata.name), + volume.fromConfigMap(tempo_config_volume, $.tempo_distributor_configmap.metadata.name), ]), tempo_distributor_service: diff --git a/operations/jsonnet/microservices/ingester.libsonnet b/operations/jsonnet/microservices/ingester.libsonnet index 5b179154fed..9c15fa30a78 100644 --- a/operations/jsonnet/microservices/ingester.libsonnet +++ b/operations/jsonnet/microservices/ingester.libsonnet @@ -50,8 +50,11 @@ ) + $.util.antiAffinityStatefulSet + statefulset.mixin.spec.withServiceName(target_name) + + statefulset.mixin.spec.template.metadata.withAnnotations({ + config_hash: std.md5(std.toString($.tempo_ingester_configmap.data['tempo.yaml'])), + }) + statefulset.mixin.spec.template.spec.withVolumes([ - volume.fromConfigMap(tempo_config_volume, $.tempo_configmap.metadata.name), + volume.fromConfigMap(tempo_config_volume, $.tempo_ingester_configmap.metadata.name), ]), tempo_ingester_service: diff --git a/operations/kube-manifests/ConfigMap-tempo-compactor.yaml b/operations/kube-manifests/ConfigMap-tempo-compactor.yaml index b1960422506..797c4b91cab 100644 --- a/operations/kube-manifests/ConfigMap-tempo-compactor.yaml +++ b/operations/kube-manifests/ConfigMap-tempo-compactor.yaml @@ -10,16 +10,7 @@ data: ring: kvstore: store: memberlist - distributor: - receivers: - jaeger: - protocols: - grpc: - endpoint: 0.0.0.0:14250 - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:55680 + distributor: null ingester: lifecycler: ring: diff --git a/operations/kube-manifests/ConfigMap-tempo-distributor.yaml b/operations/kube-manifests/ConfigMap-tempo-distributor.yaml new file mode 100644 index 00000000000..519dafebcb2 --- /dev/null +++ b/operations/kube-manifests/ConfigMap-tempo-distributor.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +data: + overrides.yaml: | + overrides: {} + tempo.yaml: | + compactor: null + distributor: + receivers: + jaeger: + protocols: + grpc: + endpoint: 0.0.0.0:14250 + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:55680 + ingester: + lifecycler: + ring: + replication_factor: 3 + memberlist: + abort_if_cluster_join_fails: false + bind_port: 7946 + join_members: + - gossip-ring.tracing.svc.cluster.local:7946 + overrides: + per_tenant_override_config: /conf/overrides.yaml + server: + http_listen_port: 3100 + storage: + trace: + backend: gcs + blocklist_poll: "0" + cache: memcached + gcs: + bucket_name: tempo + chunk_buffer_size: 1.048576e+07 + memcached: + consistent_hash: true + host: memcached + service: memcached-client + timeout: 500ms + pool: + queue_depth: 2000 + s3: + bucket: tempo + wal: + path: /var/tempo/wal +kind: ConfigMap +metadata: + name: tempo-distributor + namespace: tracing diff --git a/operations/kube-manifests/ConfigMap-tempo-ingester.yaml b/operations/kube-manifests/ConfigMap-tempo-ingester.yaml new file mode 100644 index 00000000000..d1a27f987e3 --- /dev/null +++ b/operations/kube-manifests/ConfigMap-tempo-ingester.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +data: + overrides.yaml: | + overrides: {} + tempo.yaml: | + compactor: null + distributor: null + ingester: + lifecycler: + ring: + replication_factor: 3 + memberlist: + abort_if_cluster_join_fails: false + bind_port: 7946 + join_members: + - gossip-ring.tracing.svc.cluster.local:7946 + overrides: + per_tenant_override_config: /conf/overrides.yaml + server: + http_listen_port: 3100 + storage: + trace: + backend: gcs + blocklist_poll: "0" + cache: memcached + gcs: + bucket_name: tempo + chunk_buffer_size: 1.048576e+07 + memcached: + consistent_hash: true + host: memcached + service: memcached-client + timeout: 500ms + pool: + queue_depth: 2000 + s3: + bucket: tempo + wal: + path: /var/tempo/wal +kind: ConfigMap +metadata: + name: tempo-ingester + namespace: tracing diff --git a/operations/kube-manifests/ConfigMap-tempo-querier.yaml b/operations/kube-manifests/ConfigMap-tempo-querier.yaml index f9763e92d5c..e739ba9670f 100644 --- a/operations/kube-manifests/ConfigMap-tempo-querier.yaml +++ b/operations/kube-manifests/ConfigMap-tempo-querier.yaml @@ -2,16 +2,7 @@ apiVersion: v1 data: tempo.yaml: | compactor: null - distributor: - receivers: - jaeger: - protocols: - grpc: - endpoint: 0.0.0.0:14250 - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:55680 + distributor: null ingester: lifecycler: ring: diff --git a/operations/kube-manifests/ConfigMap-tempo-query-frontend.yaml b/operations/kube-manifests/ConfigMap-tempo-query-frontend.yaml index 50bd9805be2..cfe57fd857b 100644 --- a/operations/kube-manifests/ConfigMap-tempo-query-frontend.yaml +++ b/operations/kube-manifests/ConfigMap-tempo-query-frontend.yaml @@ -2,16 +2,7 @@ apiVersion: v1 data: tempo.yaml: | compactor: null - distributor: - receivers: - jaeger: - protocols: - grpc: - endpoint: 0.0.0.0:14250 - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:55680 + distributor: null ingester: lifecycler: ring: diff --git a/operations/kube-manifests/Deployment-compactor.yaml b/operations/kube-manifests/Deployment-compactor.yaml index 513f90dac9d..bdc6fe2bc0d 100644 --- a/operations/kube-manifests/Deployment-compactor.yaml +++ b/operations/kube-manifests/Deployment-compactor.yaml @@ -18,7 +18,7 @@ spec: template: metadata: annotations: - config_hash: 3bc06ce225fc97d13d89fbee2dd6ea25 + config_hash: 391906d5363b2172f5ef468fcc2fca50 labels: app: compactor name: compactor diff --git a/operations/kube-manifests/Deployment-distributor.yaml b/operations/kube-manifests/Deployment-distributor.yaml index e3b75dbfb55..60c953e2528 100644 --- a/operations/kube-manifests/Deployment-distributor.yaml +++ b/operations/kube-manifests/Deployment-distributor.yaml @@ -48,5 +48,5 @@ spec: terminationGracePeriodSeconds: 60 volumes: - configMap: - name: tempo + name: tempo-distributor name: tempo-conf diff --git a/operations/kube-manifests/Deployment-querier.yaml b/operations/kube-manifests/Deployment-querier.yaml index 95cd713c331..570e5351847 100644 --- a/operations/kube-manifests/Deployment-querier.yaml +++ b/operations/kube-manifests/Deployment-querier.yaml @@ -19,7 +19,7 @@ spec: template: metadata: annotations: - config_hash: af35e647b8504eed98436ddd2f35360c + config_hash: 5ae9881ddd5abc1d3609268198a0aacd labels: app: querier name: querier diff --git a/operations/kube-manifests/Deployment-query-frontend.yaml b/operations/kube-manifests/Deployment-query-frontend.yaml index c5f8073d231..aab0bf54ded 100644 --- a/operations/kube-manifests/Deployment-query-frontend.yaml +++ b/operations/kube-manifests/Deployment-query-frontend.yaml @@ -18,7 +18,7 @@ spec: template: metadata: annotations: - config_hash: 76b3caf721a80349e206778e56d41a66 + config_hash: 215474a01327c5068b4236ec4a327838 labels: app: query-frontend name: query-frontend diff --git a/operations/kube-manifests/StatefulSet-ingester.yaml b/operations/kube-manifests/StatefulSet-ingester.yaml index 663a0ff7435..36815a50508 100644 --- a/operations/kube-manifests/StatefulSet-ingester.yaml +++ b/operations/kube-manifests/StatefulSet-ingester.yaml @@ -14,6 +14,8 @@ spec: serviceName: ingester template: metadata: + annotations: + config_hash: 215474a01327c5068b4236ec4a327838 labels: app: ingester name: ingester @@ -50,7 +52,7 @@ spec: name: ingester-data volumes: - configMap: - name: tempo + name: tempo-ingester name: tempo-conf updateStrategy: type: RollingUpdate diff --git a/operations/kube-manifests/StatefulSet-memcached.yaml b/operations/kube-manifests/StatefulSet-memcached.yaml index e065e3ad9e2..d4d67b832b0 100644 --- a/operations/kube-manifests/StatefulSet-memcached.yaml +++ b/operations/kube-manifests/StatefulSet-memcached.yaml @@ -25,7 +25,7 @@ spec: - args: - -m 1024 - -I 5m - - -c 1024 + - -c 4096 - -v image: memcached:1.5.17-alpine imagePullPolicy: IfNotPresent @@ -50,3 +50,4 @@ spec: name: http-metrics updateStrategy: type: RollingUpdate + volumeClaimTemplates: []