[rhythm] git merge origin/main (grafana#4376)

* chore: remove gofakeit dependency (grafana#4274) * Further reduce Labes() calls in the metrics registry (grafana#4283) * Respect passed headers in read path requests (grafana#4287) * Ingester: Validate completed blocks (grafana#4256) * Add validate method to block Signed-off-by: Joe Elliott <[email protected]> * Add Validate usage in the ingester Signed-off-by: Joe Elliott <[email protected]> * changelog Signed-off-by: Joe Elliott <[email protected]> * add test and fix replay Signed-off-by: Joe Elliott <[email protected]> * increment metric Signed-off-by: Joe Elliott <[email protected]> --------- Signed-off-by: Joe Elliott <[email protected]> * Add `invalid_utf8` to reasons spans could be rejected (grafana#4293) * Add `invalid_utf8` to reasons spans could be rejected * Update changelog * Update docs * Ensure test covers invalid UTF-8 and not slack time * add signals for duplicate rf1 data (grafana#4296) Signed-off-by: Joe Elliott <[email protected]> * Bump anchore/sbom-action from 0.17.5 to 0.17.7 (grafana#4307) Bumps [anchore/sbom-action](https://github.com/anchore/sbom-action) from 0.17.5 to 0.17.7. - [Release notes](https://github.com/anchore/sbom-action/releases) - [Changelog](https://github.com/anchore/sbom-action/blob/main/RELEASE.md) - [Commits](anchore/sbom-action@v0.17.5...v0.17.7) --- updated-dependencies: - dependency-name: anchore/sbom-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * docs: Update readme with explore traces info (grafana#4263) * docs: Update readme with explore traces info Co-authored-by: Kim Nylander <[email protected]> * chore: remove spanlogger (grafana#4312) * chore: remove spanlogger * Query-Frontend: Add middleware to drop headers (grafana#4298) * header strip ware Signed-off-by: Joe Elliott <[email protected]> * comment Signed-off-by: Joe Elliott <[email protected]> * changelog Signed-off-by: Joe Elliott <[email protected]> * remove header strip wear from metrics summary Signed-off-by: Joe Elliott <[email protected]> --------- Signed-off-by: Joe Elliott <[email protected]> * Increase length of time compactions have to fail (grafana#4315) * increase length of time compactions have to fail Signed-off-by: Joe Elliott <[email protected]> * gen Signed-off-by: Joe Elliott <[email protected]> --------- Signed-off-by: Joe Elliott <[email protected]> * docs: mark serverless as deprecated (grafana#4017) * docs: mark serverless as deprecated * Changelog + readme * docs: Remove duplicated examples (grafana#4295) This removes duplicates examples from the Configure TraceQL metrics page. Signed-off-by: Alex Bikfalvi <[email protected]> * tempo-cli: support dropping multiple traces in a single operation (grafana#4266) * tempo-cli: support dropping multiple traces in a single operation * update final log message --------- Co-authored-by: Suraj Nath <[email protected]> * [DOC] Add clarification for metrics summary and traceQL metrics (grafana#4316) * Add clarification for metrics summary and traceQL metrics * Apply suggestions from code review Co-authored-by: Jennifer Villa <[email protected]> * Update docs/sources/tempo/api_docs/metrics-summary.md --------- Co-authored-by: Jennifer Villa <[email protected]> * TraceQL metrics time range fixes (grafana#4325) * Disconnect job time range filtering from step, so that results in split backend/recent range is accurate * changelog * Fix to assert metrics query range before alignment because alignment may increase it, which is not the responsibility of the caller to account for (grafana#4331) * Add doc about configuring TLS with Helm (grafana#4328) * Add doc about configuring TLS with Helm * Add memberlist and readinessProbe to example * Include server config for listening on TLS * Add note about scraping * Update docs/sources/tempo/configuration/network/tls.md Co-authored-by: Markus Toivonen <[email protected]> * Update docs/sources/tempo/configuration/network/tls.md Co-authored-by: Kim Nylander <[email protected]> * Update docs/sources/tempo/configuration/network/tls.md Co-authored-by: Kim Nylander <[email protected]> * Add memcached config for TLS --------- Co-authored-by: Markus Toivonen <[email protected]> Co-authored-by: Kim Nylander <[email protected]> * [DOC] Add TLS info to Helm chart doc (grafana#4334) * fix deprecation warning by switching to DoBatchWithOptions (grafana#4343) Signed-off-by: Daniel Strobusch <[email protected]> * bump dskit to v0.0.0-20241115082728-f2a7eb3aa0e9 to leverage benefits for context causes for DoBatch calls. (grafana#4341) See grafana/dskit#576 Signed-off-by: Daniel Strobusch <[email protected]> * Bump github.com/minio/minio-go/v7 from 7.0.70 to 7.0.80 (grafana#4282) * Bump github.com/minio/minio-go/v7 from 7.0.70 to 7.0.80 Bumps [github.com/minio/minio-go/v7](https://github.com/minio/minio-go) from 7.0.70 to 7.0.80. - [Release notes](https://github.com/minio/minio-go/releases) - [Commits](minio/minio-go@v7.0.70...v7.0.80) --- updated-dependencies: - dependency-name: github.com/minio/minio-go/v7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> * Update serverless vendor --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Zach Leslie <[email protected]> * update default config values to better align with production workloads (grafana#4340) * update default config values to better align with production workloads * Update CHANGELOG.md and config docs * Ingester memory improvements by adjusting prealloc (grafana#4344) * remove trace ids Signed-off-by: Joe Elliott <[email protected]> * linear buckets Signed-off-by: Joe Elliott <[email protected]> * changelog Signed-off-by: Joe Elliott <[email protected]> * tuney tune Signed-off-by: Joe Elliott <[email protected]> * metric misses and increase pool size Signed-off-by: Joe Elliott <[email protected]> * lint Signed-off-by: Joe Elliott <[email protected]> --------- Signed-off-by: Joe Elliott <[email protected]> * Bump github.com/Azure/azure-sdk-for-go/sdk/azcore from 1.13.0 to 1.16.0 (grafana#4302) * Bump github.com/Azure/azure-sdk-for-go/sdk/azcore from 1.13.0 to 1.16.0 Bumps [github.com/Azure/azure-sdk-for-go/sdk/azcore](https://github.com/Azure/azure-sdk-for-go) from 1.13.0 to 1.16.0. - [Release notes](https://github.com/Azure/azure-sdk-for-go/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md) - [Commits](Azure/azure-sdk-for-go@sdk/azcore/v1.13.0...sdk/azcore/v1.16.0) --- updated-dependencies: - dependency-name: github.com/Azure/azure-sdk-for-go/sdk/azcore dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <[email protected]> * Update serverless vendor --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Zach Leslie <[email protected]> * Use Prometheus fast regexp (grafana#4329) * basic integration Signed-off-by: Joe Elliott <[email protected]> * patch tests for new meaning Signed-off-by: Joe Elliott <[email protected]> * patch up more tests Signed-off-by: Joe Elliott <[email protected]> * add basic tests Signed-off-by: Joe Elliott <[email protected]> * changelog + docs Signed-off-by: Joe Elliott <[email protected]> * remove benches Signed-off-by: Joe Elliott <[email protected]> * Cleaned up + tests Signed-off-by: Joe Elliott <[email protected]> * comment Signed-off-by: Joe Elliott <[email protected]> * lint Signed-off-by: Joe Elliott <[email protected]> * Update docs/sources/tempo/traceql/_index.md Co-authored-by: Kim Nylander <[email protected]> * comment Signed-off-by: Joe Elliott <[email protected]> --------- Signed-off-by: Joe Elliott <[email protected]> Co-authored-by: Kim Nylander <[email protected]> * Fix broken link in service-graphs docs (grafana#4351) * Fix minor typo in TraceQL docs (grafana#4356) * Bump default memcached version (grafana#4363) * Exemplar fixes (grafana#4366) * Fix exemplars based on duration to convert to seconds, fix various other issues * changelog * fix: initialize histogram buckets to 0 to avoid them being downsampled (grafana#4368) * initialized histogram buckets to 0 to avoid them being downsampled * Ingester/Generator Live trace cleanup (grafana#4365) * moved trace sizes somewhere shareable Signed-off-by: Joe Elliott <[email protected]> * use tracesizes in ingester Signed-off-by: Joe Elliott <[email protected]> * make tests work Signed-off-by: Joe Elliott <[email protected]> * trace bytes in generator Signed-off-by: Joe Elliott <[email protected]> * remove traceCount Signed-off-by: Joe Elliott <[email protected]> * live trace shenanigans Signed-off-by: Joe Elliott <[email protected]> * changelog Signed-off-by: Joe Elliott <[email protected]> * Update modules/generator/processor/localblocks/livetraces.go Co-authored-by: Mario <[email protected]> * Update modules/ingester/instance.go Co-authored-by: Mario <[email protected]> * Test cleanup. Add sz test, restore commented out and fix e2e Signed-off-by: Joe Elliott <[email protected]> * remove todo comment Signed-off-by: Joe Elliott <[email protected]> --------- Signed-off-by: Joe Elliott <[email protected]> Co-authored-by: Mario <[email protected]> * Bump anchore/sbom-action from 0.17.7 to 0.17.8 (grafana#4371) Bumps [anchore/sbom-action](https://github.com/anchore/sbom-action) from 0.17.7 to 0.17.8. - [Release notes](https://github.com/anchore/sbom-action/releases) - [Changelog](https://github.com/anchore/sbom-action/blob/main/RELEASE.md) - [Commits](anchore/sbom-action@v0.17.7...v0.17.8) --- updated-dependencies: - dependency-name: anchore/sbom-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Update for IDs change * Only run blockbuilder if ingest enabled --------- Signed-off-by: Joe Elliott <[email protected]> Signed-off-by: dependabot[bot] <[email protected]> Signed-off-by: Alex Bikfalvi <[email protected]> Signed-off-by: Daniel Strobusch <[email protected]> Co-authored-by: Javier Molina Reyes <[email protected]> Co-authored-by: Zach Leslie <[email protected]> Co-authored-by: Joe Elliott <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan Perry <[email protected]> Co-authored-by: Kim Nylander <[email protected]> Co-authored-by: Suraj Nath <[email protected]> Co-authored-by: Alex Bikfalvi <[email protected]> Co-authored-by: Andrey Karpov <[email protected]> Co-authored-by: Jennifer Villa <[email protected]> Co-authored-by: Martin Disibio <[email protected]> Co-authored-by: Markus Toivonen <[email protected]> Co-authored-by: Daniel Strobusch <[email protected]> Co-authored-by: Carles Garcia <[email protected]>
mapno · Jan 10, 2025 · 2be0e01 · 2be0e01
1 parent d27b489
commit 2be0e01
Show file tree

Hide file tree

Showing 7 changed files with 87 additions and 11 deletions.
diff --git a/cmd/tempo/app/modules.go b/cmd/tempo/app/modules.go
@@ -338,6 +338,10 @@ func (t *App) initGenerator() (services.Service, error) {
 }
 
 func (t *App) initBlockBuilder() (services.Service, error) {
+	if !t.cfg.Ingest.Enabled {
+		return services.NewIdleService(nil, nil), nil
+	}
+
 	t.cfg.BlockBuilder.IngestStorageConfig = t.cfg.Ingest
 	t.cfg.BlockBuilder.IngestStorageConfig.Kafka.ConsumerGroup = blockbuilder.ConsumerGroup
 

diff --git a/docs/sources/tempo/configuration/manifest.md b/docs/sources/tempo/configuration/manifest.md
@@ -183,6 +183,22 @@ distributor:
         cost_attribution:
             max_cardinality: 10000
             stale_duration: 15m0s
+    kafka_write_path_enabled: false
+    kafka_config:
+        address: ""
+        topic: ""
+        client_id: ""
+        dial_timeout: 0s
+        write_timeout: 0s
+        consumer_group: ""
+        consumer_group_offset_commit_interval: 0s
+        last_produced_offset_retry_timeout: 0s
+        auto_create_topic_enabled: false
+        auto_create_topic_default_partitions: 0
+        producer_max_record_size_bytes: 0
+        producer_max_buffered_bytes: 0
+        target_consumer_lag_at_startup: 0s
+        max_consumer_lag_at_startup: 0s
     extend_writes: true
     retry_after_on_resource_exhausted: 0s
     max_span_attr_byte: 2048
@@ -439,6 +455,40 @@ ingester:
         address: ""
         port: 0
         id: hostname
+    partition_ring:
+        kvstore:
+            store: memberlist
+            prefix: collectors/
+            consul:
+                host: localhost:8500
+                acl_token: ""
+                http_client_timeout: 20s
+                consistent_reads: false
+                watch_rate_limit: 1
+                watch_burst_size: 1
+                cas_retry_delay: 1s
+            etcd:
+                endpoints: []
+                dial_timeout: 10s
+                max_retries: 10
+                tls_enabled: false
+                tls_cert_path: ""
+                tls_key_path: ""
+                tls_ca_path: ""
+                tls_server_name: ""
+                tls_insecure_skip_verify: false
+                tls_cipher_suites: ""
+                tls_min_version: ""
+                username: ""
+                password: ""
+            multi:
+                primary: ""
+                secondary: ""
+                mirror_enabled: false
+                mirror_timeout: 2s
+        min_partition_owners_count: 1
+        min_partition_owners_duration: 10s
+        delete_inactive_partition_after: 13h0m0s
     concurrent_flushes: 4
     flush_check_period: 10s
     flush_op_timeout: 5m0s
@@ -600,6 +650,28 @@ metrics_generator:
     metrics_ingestion_time_range_slack: 30s
     query_timeout: 30s
     override_ring_key: metrics-generator
+    assigned_partitions: []
+ingest:
+    enabled: false
+    kafka:
+        address: localhost:9092
+        topic: ""
+        client_id: ""
+        dial_timeout: 2s
+        write_timeout: 10s
+        consumer_group: ""
+        consumer_group_offset_commit_interval: 1s
+        last_produced_offset_retry_timeout: 10s
+        auto_create_topic_enabled: true
+        auto_create_topic_default_partitions: 1000
+        producer_max_record_size_bytes: 15983616
+        producer_max_buffered_bytes: 1073741824
+        target_consumer_lag_at_startup: 2s
+        max_consumer_lag_at_startup: 15s
+block_builder:
+    assigned_partitions: []
+    consume_cycle_duration: 5m0s
+    lookback_on_no_commit: 12h0m0s
 storage:
     trace:
         pool:

diff --git a/modules/blockbuilder/partition_writer.go b/modules/blockbuilder/partition_writer.go
@@ -53,7 +53,7 @@ func (p *writer) pushBytes(tenant string, req *tempopb.PushBytesRequest) error {
 		"msg", "pushing bytes",
 		"tenant", tenant,
 		"num_traces", len(req.Traces),
-		"id", util.TraceIDToHexString(req.Ids[0].Slice),
+		"id", util.TraceIDToHexString(req.Ids[0]),
 	)
 
 	i, err := p.instanceForTenant(tenant)
@@ -67,7 +67,7 @@ func (p *writer) pushBytes(tenant string, req *tempopb.PushBytesRequest) error {
 			return fmt.Errorf("failed to unmarshal trace: %w", err)
 		}
 
-		if err := i.AppendTrace(req.Ids[j].Slice, tr, 0, 0); err != nil {
+		if err := i.AppendTrace(req.Ids[j], tr, 0, 0); err != nil {
 			return err
 		}
 	}

diff --git a/modules/distributor/distributor.go b/modules/distributor/distributor.go
@@ -593,12 +593,12 @@ func (d *Distributor) sendToKafka(ctx context.Context, userID string, keys []uin
 
 		req := &tempopb.PushBytesRequest{
 			Traces: make([]tempopb.PreallocBytes, len(indexes)),
-			Ids:    make([]tempopb.PreallocBytes, len(indexes)),
+			Ids:    make([][]byte, len(indexes)),
 		}
 
 		for i, j := range indexes {
 			req.Traces[i].Slice = marshalledTraces[j][0:]
-			req.Ids[i].Slice = traces[j].id
+			req.Ids[i] = traces[j].id
 		}
 
 		// The partition ID is stored in the ring.InstanceDesc ID.

diff --git a/pkg/ingest/encoding.go b/pkg/ingest/encoding.go
@@ -25,7 +25,7 @@ func encoderPoolGet() *tempopb.PushBytesRequest {
 
 	return &tempopb.PushBytesRequest{
 		Traces: make([]tempopb.PreallocBytes, 0, 10),
-		Ids:    make([]tempopb.PreallocBytes, 0, 10),
+		Ids:    make([][]byte, 0, 10),
 	}
 }
 
@@ -54,7 +54,7 @@ func Encode(partitionID int32, tenantID string, req *tempopb.PushBytesRequest, m
 	currentSize := 0
 
 	for i, entry := range req.Traces {
-		l := entry.Size() + req.Ids[i].Size()
+		l := entry.Size() + len(req.Ids[i])
 		// Size of the entry in the req
 		entrySize := 1 + l + sovPush(uint64(l))
 

diff --git a/pkg/ingest/encoding_test.go b/pkg/ingest/encoding_test.go
@@ -44,7 +44,7 @@ func TestEncoderDecoder(t *testing.T) {
 			}
 
 			var decodedEntries []tempopb.PreallocBytes
-			var decodedIDs []tempopb.PreallocBytes
+			var decodedIDs [][]byte
 
 			for _, record := range records {
 				decoder.Reset()
@@ -115,12 +115,12 @@ func BenchmarkEncodeDecode(b *testing.B) {
 func generateRequest(entries, lineLength int) *tempopb.PushBytesRequest {
 	stream := &tempopb.PushBytesRequest{
 		Traces: make([]tempopb.PreallocBytes, entries),
-		Ids:    make([]tempopb.PreallocBytes, entries),
+		Ids:    make([][]byte, entries),
 	}
 
 	for i := 0; i < entries; i++ {
 		stream.Traces[i].Slice = generateRandomString(lineLength)
-		stream.Ids[i].Slice = generateRandomString(lineLength)
+		stream.Ids[i] = generateRandomString(lineLength)
 	}
 
 	return stream

diff --git a/pkg/util/test/req.go b/pkg/util/test/req.go
@@ -373,10 +373,10 @@ func MakePushBytesRequest(t *testing.T, requests int, traceID []byte) *tempopb.P
 
 	req := &tempopb.PushBytesRequest{
 		Traces: make([]tempopb.PreallocBytes, 0),
-		Ids:    make([]tempopb.PreallocBytes, 0),
+		Ids:    make([][]byte, 0),
 	}
 	req.Traces = append(req.Traces, tempopb.PreallocBytes{Slice: b})
-	req.Ids = append(req.Ids, tempopb.PreallocBytes{Slice: traceID})
+	req.Ids = append(req.Ids, traceID)
 
 	return req
 }