Merge branch 'main' into feat/optional-tls-version

Signed-off-by: pureiboi <[email protected]>

.circleci/config.yml

@@ -15,49 +15,6 @@ executors:
       - image: quay.io/thanos/docker-swift-onlyone-authv2-keystone:v0.1
 
 jobs:
-  test:
-    executor: golang-test
-    environment:
-      GO111MODULE: "on"
-    steps:
-      - git-shallow-clone/checkout
-      - go/load-cache
-      - go/mod-download
-      - run:
-          name: Download bingo modules
-          command: |
-            make install-tool-deps
-      - go/save-cache
-      - setup_remote_docker:
-          version: docker24
-      - run:
-          name: Create Secret if PR is not forked
-          # GCS integration tests are run only for author's PR that have write access, because these tests
-          # require credentials. Env variables that sets up these tests will work only for these kind of PRs.
-          command: |
-            if ! [ -z ${GCP_PROJECT} ]; then
-              echo $GOOGLE_APPLICATION_CREDENTIALS_CONTENT > $GOOGLE_APPLICATION_CREDENTIALS
-              echo "Awesome! GCS and S3 AWS integration tests are enabled."
-            fi
-      - run:
-          name: "Run unit tests."
-          no_output_timeout: "30m"
-          environment:
-            THANOS_TEST_OBJSTORE_SKIP: GCS,S3,AZURE,COS,ALIYUNOSS,BOS,OCI,OBS
-            # Variables for Swift testing.
-            OS_AUTH_URL: http://127.0.0.1:5000/v2.0
-            OS_PASSWORD: s3cr3t
-            OS_PROJECT_NAME: admin
-            OS_REGION_NAME: RegionOne
-            OS_USERNAME: admin
-          # taskset sets CPU affinity to 2 (current CPU limit).
-          command: |
-            if [ -z ${GCP_PROJECT} ]; then
-              export THANOS_TEST_OBJSTORE_SKIP=${THANOS_TEST_OBJSTORE_SKIP}
-            fi
-            echo "Skipping tests for object storages: ${THANOS_TEST_OBJSTORE_SKIP}"
-            taskset 2 make test
-
   # Cross build is needed for publish_release but needs to be done outside of docker.
   cross_build:
     machine: true
@@ -127,27 +84,18 @@ workflows:
   version: 2
   thanos:
     jobs:
-      - test:
-          filters:
-            tags:
-              only: /.*/
       - publish_main:
-          requires:
-            - test
           filters:
             branches:
               only: main
       - cross_build:
-          requires:
-            - test
           filters:
             tags:
               only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
             branches:
               ignore: /.*/
       - publish_release:
           requires:
-            - test
             - cross_build
           filters:
             tags:

.github/workflows/go.yaml

@@ -7,11 +7,42 @@ on:
     tags:
   pull_request:
 
-# TODO(bwplotka): Add tests here.
 permissions:  
   contents: read
 
 jobs:
+  unit:
+    runs-on: ubuntu-latest
+    name: Thanos unit tests
+    env:
+      THANOS_TEST_OBJSTORE_SKIP: GCS,S3,AZURE,COS,ALIYUNOSS,BOS,OCI,OBS,SWIFT
+      OS_AUTH_URL: http://127.0.0.1:5000/v2.0
+      OS_PASSWORD: s3cr3t
+      OS_PROJECT_NAME: admin
+      OS_REGION_NAME: RegionOne
+      OS_USERNAME: admin
+      GOBIN: /tmp/.bin
+    services:
+      swift:
+        image: 'quay.io/thanos/docker-swift-onlyone-authv2-keystone:v0.1'
+        ports:
+          - 5000:5000
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Install Go.
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version: 1.23.x
+
+      - name: Install bingo modules
+        run: make install-tool-deps
+      - name: Add GOBIN to path
+        run: echo "/tmp/.bin" >> $GITHUB_PATH
+      - name: Run unit tests
+        run: make test
+
   cross-build-check:
     runs-on: ubuntu-latest
     name: Go build for different platforms

CHANGELOG.md

@@ -33,6 +33,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#7652](https://github.com/thanos-io/thanos/pull/7652) Store: Implement metadata API limit in stores.
 - [#7659](https://github.com/thanos-io/thanos/pull/7659) Receive: Add support for replication using [Cap'n Proto](https://capnproto.org/). This protocol has a lower CPU and memory footprint, which leads to a reduction in resource usage in Receivers. Before enabling it, make sure that all receivers are updated to a version which supports this replication method.
 - [#7853](https://github.com/thanos-io/thanos/pull/7853) UI: Add support for selecting graph time range with mouse drag.
+- [#7855](https://github.com/thanos-io/thanos/pull/7855) Compcat/Query: Add support for comma separated replica labels.
 - [#7654](https://github.com/thanos-io/thanos/pull/7654) *: Add '--grpc-server-tls-min-version' flag to allow user to specify TLS version, otherwise default to TLS 1.3
 
 ### Changed

cmd/thanos/compact.go

@@ -46,6 +46,7 @@ import (
 	"github.com/thanos-io/thanos/pkg/runutil"
 	httpserver "github.com/thanos-io/thanos/pkg/server/http"
 	"github.com/thanos-io/thanos/pkg/store"
+	"github.com/thanos-io/thanos/pkg/strutil"
 	"github.com/thanos-io/thanos/pkg/tracing"
 	"github.com/thanos-io/thanos/pkg/ui"
 )
@@ -254,10 +255,11 @@ func runCompact(
 	}
 
 	enableVerticalCompaction := conf.enableVerticalCompaction
-	if len(conf.dedupReplicaLabels) > 0 {
+	dedupReplicaLabels := strutil.ParseFlagLabels(conf.dedupReplicaLabels)
+	if len(dedupReplicaLabels) > 0 {
 		enableVerticalCompaction = true
 		level.Info(logger).Log(
-			"msg", "deduplication.replica-label specified, enabling vertical compaction", "dedupReplicaLabels", strings.Join(conf.dedupReplicaLabels, ","),
+			"msg", "deduplication.replica-label specified, enabling vertical compaction", "dedupReplicaLabels", strings.Join(dedupReplicaLabels, ","),
 		)
 	}
 	if enableVerticalCompaction {
@@ -275,7 +277,7 @@ func runCompact(
 			labelShardedMetaFilter,
 			consistencyDelayMetaFilter,
 			ignoreDeletionMarkFilter,
-			block.NewReplicaLabelRemover(logger, conf.dedupReplicaLabels),
+			block.NewReplicaLabelRemover(logger, dedupReplicaLabels),
 			duplicateBlocksFilter,
 			noCompactMarkerFilter,
 		}
@@ -326,7 +328,7 @@ func runCompact(
 	case compact.DedupAlgorithmPenalty:
 		mergeFunc = dedup.NewChunkSeriesMerger()
 
-		if len(conf.dedupReplicaLabels) == 0 {
+		if len(dedupReplicaLabels) == 0 {
 			return errors.New("penalty based deduplication needs at least one replica label specified")
 		}
 	case "":
@@ -818,8 +820,9 @@ func (cc *compactConfig) registerFlag(cmd extkingpin.FlagClause) {
 		"When set to penalty, penalty based deduplication algorithm will be used. At least one replica label has to be set via --deduplication.replica-label flag.").
 		Default("").EnumVar(&cc.dedupFunc, compact.DedupAlgorithmPenalty, "")
 
-	cmd.Flag("deduplication.replica-label", "Label to treat as a replica indicator of blocks that can be deduplicated (repeated flag). This will merge multiple replica blocks into one. This process is irreversible."+
-		"Experimental. When one or more labels are set, compactor will ignore the given labels so that vertical compaction can merge the blocks."+
+	cmd.Flag("deduplication.replica-label", "Experimental. Label to treat as a replica indicator of blocks that can be deduplicated (repeated flag). This will merge multiple replica blocks into one. This process is irreversible. "+
+		"Flag may be specified multiple times as well as a comma separated list of labels. "+
+		"When one or more labels are set, compactor will ignore the given labels so that vertical compaction can merge the blocks."+
 		"Please note that by default this uses a NAIVE algorithm for merging which works well for deduplication of blocks with **precisely the same samples** like produced by Receiver replication."+
 		"If you need a different deduplication algorithm (e.g one that works well with Prometheus replicas), please set it via --deduplication.func.").
 		StringsVar(&cc.dedupReplicaLabels)

cmd/thanos/query.go

@@ -56,6 +56,7 @@ import (
 	httpserver "github.com/thanos-io/thanos/pkg/server/http"
 	"github.com/thanos-io/thanos/pkg/store"
 	"github.com/thanos-io/thanos/pkg/store/labelpb"
+	"github.com/thanos-io/thanos/pkg/strutil"
 	"github.com/thanos-io/thanos/pkg/targets"
 	"github.com/thanos-io/thanos/pkg/tenancy"
 	"github.com/thanos-io/thanos/pkg/tls"
@@ -122,7 +123,7 @@ func registerQuery(app *extkingpin.App) {
 		Default(string(query.ExternalLabels), string(query.StoreType)).
 		Enums(string(query.ExternalLabels), string(query.StoreType))
 
-	queryReplicaLabels := cmd.Flag("query.replica-label", "Labels to treat as a replica indicator along which data is deduplicated. Still you will be able to query without deduplication using 'dedup=false' parameter. Data includes time series, recording rules, and alerting rules.").
+	queryReplicaLabels := cmd.Flag("query.replica-label", "Labels to treat as a replica indicator along which data is deduplicated. Still you will be able to query without deduplication using 'dedup=false' parameter. Data includes time series, recording rules, and alerting rules. Flag may be specified multiple times as well as a comma separated list of labels.").
 		Strings()
 	queryPartitionLabels := cmd.Flag("query.partition-label", "Labels that partition the leaf queriers. This is used to scope down the labelsets of leaf queriers when using the distributed query mode. If set, these labels must form a partition of the leaf queriers. Partition labels must not intersect with replica labels. Every TSDB of a leaf querier must have these labels. This is useful when there are multiple external labels that are irrelevant for the partition as it allows the distributed engine to ignore them for some optimizations. If this is empty then all labels are used as partition labels.").Strings()
 
@@ -539,6 +540,9 @@ func runQuery(
 		store.WithProxyStoreDebugLogging(debugLogging),
 	}
 
+	// Parse and sanitize the provided replica labels flags.
+	queryReplicaLabels = strutil.ParseFlagLabels(queryReplicaLabels)
+
 	var (
 		endpoints = prepareEndpointSet(
 			g,

docs/components/compact.md

@@ -342,14 +342,16 @@ Flags:
                                 At least one replica label has to be set via
                                 --deduplication.replica-label flag.
       --deduplication.replica-label=DEDUPLICATION.REPLICA-LABEL ...
-                                Label to treat as a replica indicator of blocks
-                                that can be deduplicated (repeated flag). This
-                                will merge multiple replica blocks into one.
-                                This process is irreversible.Experimental.
-                                When one or more labels are set, compactor
-                                will ignore the given labels so that vertical
-                                compaction can merge the blocks.Please note
-                                that by default this uses a NAIVE algorithm
+                                Experimental. Label to treat as a replica
+                                indicator of blocks that can be deduplicated
+                                (repeated flag). This will merge multiple
+                                replica blocks into one. This process is
+                                irreversible. Flag may be specified multiple
+                                times as well as a comma separated list of
+                                labels. When one or more labels are set,
+                                compactor will ignore the given labels so that
+                                vertical compaction can merge the blocks.Please
+                                note that by default this uses a NAIVE algorithm
                                 for merging which works well for deduplication
                                 of blocks with **precisely the same samples**
                                 like produced by Receiver replication.If you

docs/components/query.md

@@ -449,6 +449,8 @@ Flags:
                                  be able to query without deduplication using
                                  'dedup=false' parameter. Data includes time
                                  series, recording rules, and alerting rules.
+                                 Flag may be specified multiple times as well as
+                                 a comma separated list of labels.
       --query.telemetry.request-duration-seconds-quantiles=0.1... ...
                                  The quantiles for exporting metrics about the
                                  request duration quantiles.

go.mod

@@ -112,7 +112,7 @@ require (
 )
 
 require (
-	capnproto.org/go/capnp/v3 v3.0.1-alpha.1
+	capnproto.org/go/capnp/v3 v3.0.1-alpha.2.0.20240830165715-46ccd63a72af
 	github.com/cortexproject/promqlsmith v0.0.0-20240506042652-6cfdd9739a5e
 	github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1
 	github.com/hashicorp/golang-lru/v2 v2.0.7

go.sum

@@ -1,5 +1,5 @@
-capnproto.org/go/capnp/v3 v3.0.1-alpha.1 h1:hYEclwXEKsnu+PdHASdx3nLP0fC9kZnR+x1CEvMp9ck=
-capnproto.org/go/capnp/v3 v3.0.1-alpha.1/go.mod h1:B+ZjwFmHwTYv201x6CdIo7MmDC/TROJDa00kbjTnv1s=
+capnproto.org/go/capnp/v3 v3.0.1-alpha.2.0.20240830165715-46ccd63a72af h1:A5wxH0ZidOtYYUGjhtBaRuB87M73bGfc06uWB8sHpg0=
+capnproto.org/go/capnp/v3 v3.0.1-alpha.2.0.20240830165715-46ccd63a72af/go.mod h1:2vT5D2dtG8sJGEoEKU17e+j7shdaYp1Myl8X03B3hmc=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
@@ -2098,8 +2098,8 @@ github.com/ovh/go-ovh v1.6.0 h1:ixLOwxQdzYDx296sXcgS35TOPEahJkpjMGtzPadCjQI=
 github.com/ovh/go-ovh v1.6.0/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC7c=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
-github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
-github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
+github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
 github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
 github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
@@ -2255,8 +2255,8 @@ github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 h1:xPaP58g
 github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00=
 github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU=
 github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY=
-github.com/tinylib/msgp v1.1.5 h1:2gXmtWueD2HefZHQe1QOy9HVzmFrLOVvsXwXBQ0ayy0=
-github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=
+github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU=
+github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k=
 github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
 github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
 github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=

pkg/compact/clean_test.go

@@ -25,6 +25,8 @@ import (
 )
 
 func TestBestEffortCleanAbortedPartialUploads(t *testing.T) {
+	t.Parallel()
+
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 

pkg/compact/compact_test.go

@@ -32,6 +32,8 @@ import (
 )
 
 func TestHaltError(t *testing.T) {
+	t.Parallel()
+
 	err := errors.New("test")
 	testutil.Assert(t, !IsHaltError(err), "halt error")
 
@@ -46,6 +48,8 @@ func TestHaltError(t *testing.T) {
 }
 
 func TestHaltMultiError(t *testing.T) {
+	t.Parallel()
+
 	haltErr := halt(errors.New("halt error"))
 	nonHaltErr := errors.New("not a halt error")
 
@@ -59,6 +63,8 @@ func TestHaltMultiError(t *testing.T) {
 }
 
 func TestRetryMultiError(t *testing.T) {
+	t.Parallel()
+
 	retryErr := retry(errors.New("retry error"))
 	nonRetryErr := errors.New("not a retry error")
 
@@ -75,6 +81,8 @@ func TestRetryMultiError(t *testing.T) {
 }
 
 func TestRetryError(t *testing.T) {
+	t.Parallel()
+
 	err := errors.New("test")
 	testutil.Assert(t, !IsRetryError(err), "retry error")
 
@@ -92,6 +100,8 @@ func TestRetryError(t *testing.T) {
 }
 
 func TestGroupKey(t *testing.T) {
+	t.Parallel()
+
 	for _, tcase := range []struct {
 		input    metadata.Thanos
 		expected string
@@ -131,6 +141,8 @@ func TestGroupKey(t *testing.T) {
 }
 
 func TestGroupMaxMinTime(t *testing.T) {
+	t.Parallel()
+
 	g := &Group{
 		metasByMinTime: []*metadata.Meta{
 			{BlockMeta: tsdb.BlockMeta{MinTime: 0, MaxTime: 10}},
@@ -207,6 +219,8 @@ func createBlockMeta(id uint64, minTime, maxTime int64, labels map[string]string
 }
 
 func TestRetentionProgressCalculate(t *testing.T) {
+	t.Parallel()
+
 	logger := log.NewNopLogger()
 	reg := prometheus.NewRegistry()
 
@@ -328,6 +342,8 @@ func TestRetentionProgressCalculate(t *testing.T) {
 }
 
 func TestCompactProgressCalculate(t *testing.T) {
+	t.Parallel()
+
 	type planResult struct {
 		compactionBlocks, compactionRuns float64
 	}
@@ -433,6 +449,8 @@ func TestCompactProgressCalculate(t *testing.T) {
 }
 
 func TestDownsampleProgressCalculate(t *testing.T) {
+	t.Parallel()
+
 	reg := prometheus.NewRegistry()
 	logger := log.NewNopLogger()
 
@@ -532,6 +550,8 @@ func TestDownsampleProgressCalculate(t *testing.T) {
 }
 
 func TestNoMarkFilterAtomic(t *testing.T) {
+	t.Parallel()
+
 	ctx := context.TODO()
 	logger := log.NewLogfmtLogger(io.Discard)