diff --git a/CHANGELOG.md b/CHANGELOG.md index 87e9decfc3..9e2b84a2b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,9 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re ### Fixed - ### Changed -- + +- [#4223](https://github.com/thanos-io/thanos/pull/4223) Query: federated exemplars API only add replica labels to series labels, not to exemplar labels. + ### Removed - diff --git a/pkg/exemplars/exemplars.go b/pkg/exemplars/exemplars.go index 9c36ca4625..a5145a9b0f 100644 --- a/pkg/exemplars/exemplars.go +++ b/pkg/exemplars/exemplars.go @@ -79,54 +79,48 @@ func (rr *GRPCClient) Exemplars(ctx context.Context, req *exemplarspb.ExemplarsR return nil, nil, errors.Wrap(err, "proxy Exemplars") } - resp.data = dedupExemplarsData(resp.data, rr.replicaLabels) - for _, d := range resp.data { - d.Exemplars = dedupExemplars(d.Exemplars, rr.replicaLabels) - } - + resp.data = dedupExemplarsResponse(resp.data, rr.replicaLabels) return resp.data, resp.warnings, nil } -func dedupExemplarsData(exemplarsData []*exemplarspb.ExemplarData, replicaLabels map[string]struct{}) []*exemplarspb.ExemplarData { +func dedupExemplarsResponse(exemplarsData []*exemplarspb.ExemplarData, replicaLabels map[string]struct{}) []*exemplarspb.ExemplarData { if len(exemplarsData) == 0 { return exemplarsData } - // Sort each exemplar's label names such that they are comparable. - for _, d := range exemplarsData { - sort.Slice(d.SeriesLabels.Labels, func(i, j int) bool { - return d.SeriesLabels.Labels[i].Name < d.SeriesLabels.Labels[j].Name - }) - } - - // Sort exemplars data such that they appear next to each other. - sort.Slice(exemplarsData, func(i, j int) bool { - return exemplarsData[i].Compare(exemplarsData[j]) < 0 - }) - - i := 0 - exemplarsData[i].SeriesLabels.Labels = removeReplicaLabels(exemplarsData[i].SeriesLabels.Labels, replicaLabels) - for j := 1; j < len(exemplarsData); j++ { - exemplarsData[j].SeriesLabels.Labels = removeReplicaLabels(exemplarsData[j].SeriesLabels.Labels, replicaLabels) - if exemplarsData[i].Compare(exemplarsData[j]) != 0 { - // Effectively retain exemplarsData[j] in the resulting slice. - i++ - exemplarsData[i] = exemplarsData[j] + // Deduplicate series labels. + hashToExemplar := make(map[uint64]*exemplarspb.ExemplarData) + for _, e := range exemplarsData { + if len(e.Exemplars) == 0 { continue } + e.SeriesLabels.Labels = removeReplicaLabels(e.SeriesLabels.Labels, replicaLabels) + h := labelpb.ZLabelsToPromLabels(e.SeriesLabels.Labels).Hash() + if ref, ok := hashToExemplar[h]; ok { + ref.Exemplars = append(ref.Exemplars, e.Exemplars...) + } else { + hashToExemplar[h] = e + } } - return exemplarsData[:i+1] -} - -func dedupExemplars(exemplars []*exemplarspb.Exemplar, replicaLabels map[string]struct{}) []*exemplarspb.Exemplar { - if len(exemplars) == 0 { - return exemplars + res := make([]*exemplarspb.ExemplarData, 0, len(hashToExemplar)) + for _, e := range hashToExemplar { + // Dedup exemplars with the same series labels. + e.Exemplars = dedupExemplars(e.Exemplars) + res = append(res, e) } + // Sort by series labels. + sort.Slice(res, func(i, j int) bool { + return res[i].Compare(res[j]) < 0 + }) + return res +} + +func dedupExemplars(exemplars []*exemplarspb.Exemplar) []*exemplarspb.Exemplar { for _, e := range exemplars { sort.Slice(e.Labels.Labels, func(i, j int) bool { - return e.Labels.Labels[i].Name < e.Labels.Labels[j].Name + return e.Labels.Labels[i].Compare(e.Labels.Labels[j]) < 0 }) } @@ -135,9 +129,7 @@ func dedupExemplars(exemplars []*exemplarspb.Exemplar, replicaLabels map[string] }) i := 0 - exemplars[i].Labels.Labels = removeReplicaLabels(exemplars[i].Labels.Labels, replicaLabels) for j := 1; j < len(exemplars); j++ { - exemplars[j].Labels.Labels = removeReplicaLabels(exemplars[j].Labels.Labels, replicaLabels) if exemplars[i].Compare(exemplars[j]) != 0 { // Effectively retain exemplars[j] in the resulting slice. i++ @@ -149,6 +141,9 @@ func dedupExemplars(exemplars []*exemplarspb.Exemplar, replicaLabels map[string] } func removeReplicaLabels(labels []labelpb.ZLabel, replicaLabels map[string]struct{}) []labelpb.ZLabel { + if len(replicaLabels) == 0 { + return labels + } newLabels := make([]labelpb.ZLabel, 0, len(labels)) for _, l := range labels { if _, ok := replicaLabels[l.Name]; !ok { diff --git a/pkg/exemplars/exemplars_test.go b/pkg/exemplars/exemplars_test.go index 1597a021ee..dbebf131ba 100644 --- a/pkg/exemplars/exemplars_test.go +++ b/pkg/exemplars/exemplars_test.go @@ -15,7 +15,7 @@ func TestMain(m *testing.M) { testutil.TolerantVerifyLeakMain(m) } -func TestDedupExemplarsData(t *testing.T) { +func TestDedupExemplarsResponse(t *testing.T) { for _, tc := range []struct { name string exemplars, want []*exemplarspb.ExemplarData @@ -43,16 +43,7 @@ func TestDedupExemplarsData(t *testing.T) { }}, }, }, - want: []*exemplarspb.ExemplarData{ - { - SeriesLabels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ - {Name: "__name__", Value: "test_exemplar_metric_total"}, - {Name: "instance", Value: "localhost:8090"}, - {Name: "job", Value: "prometheus"}, - {Name: "service", Value: "bar"}, - }}, - }, - }, + want: []*exemplarspb.ExemplarData{}, }, { name: "multiple series", @@ -118,135 +109,111 @@ func TestDedupExemplarsData(t *testing.T) { Value: 19, Ts: 1600096955479, }, - { - Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ - {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, - }}, - Value: 19, - Ts: 1600096955479, - }, }, }, }, }, - } { - t.Run(tc.name, func(t *testing.T) { - replicaLabels := make(map[string]struct{}) - for _, lbl := range tc.replicaLabels { - replicaLabels[lbl] = struct{}{} - } - testutil.Equals(t, tc.want, dedupExemplarsData(tc.exemplars, replicaLabels)) - }) - } -} - -func TestDedupExemplars(t *testing.T) { - for _, tc := range []struct { - name string - exemplars, want []*exemplarspb.Exemplar - replicaLabels []string - }{ - { - name: "nil slice", - exemplars: nil, - want: nil, - }, - { - name: "empty exemplars slice", - exemplars: []*exemplarspb.Exemplar{}, - want: []*exemplarspb.Exemplar{}, - }, - { - name: "duplicate exemplars", - exemplars: []*exemplarspb.Exemplar{ - { - Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ - {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, - }}, - Value: 19, - Ts: 1600096955479, - }, - { - Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ - {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, - }}, - Value: 19, - Ts: 1600096955479, - }, - }, - want: []*exemplarspb.Exemplar{ - { - Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ - {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, - }}, - Value: 19, - Ts: 1600096955479, - }, - }, - }, - { - name: "distinct exemplars", - exemplars: []*exemplarspb.Exemplar{ - { - Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ - {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, - }}, - Value: 19, - Ts: 1600096955479, - }, - { - Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ - {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, - }}, - Value: 20, - Ts: 1600096955479, - }, - }, - want: []*exemplarspb.Exemplar{ - { - Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ - {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, - }}, - Value: 19, - Ts: 1600096955479, - }, - { - Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ - {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, - }}, - Value: 20, - Ts: 1600096955479, - }, - }, - }, { - name: "exemplars with replica labels", + name: "multiple series with multiple exemplars data", replicaLabels: []string{"replica"}, - exemplars: []*exemplarspb.Exemplar{ + exemplars: []*exemplarspb.ExemplarData{ { - Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ - {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, + SeriesLabels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "__name__", Value: "test_exemplar_metric_total"}, + {Name: "instance", Value: "localhost:8090"}, + {Name: "job", Value: "prometheus"}, + {Name: "service", Value: "bar"}, {Name: "replica", Value: "0"}, }}, - Value: 19, - Ts: 1600096955479, + Exemplars: []*exemplarspb.Exemplar{ + { + Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, + }}, + Value: 19, + Ts: 1600096955479, + }, + { + Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "traceID", Value: "foo"}, + }}, + Value: 19, + Ts: 1600096955470, + }, + }, }, { - Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ - {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, + SeriesLabels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "__name__", Value: "test_exemplar_metric_total"}, + {Name: "instance", Value: "localhost:8090"}, + {Name: "job", Value: "prometheus"}, + {Name: "service", Value: "bar"}, {Name: "replica", Value: "1"}, }}, - Value: 19, - Ts: 1600096955479, + Exemplars: []*exemplarspb.Exemplar{ + { + Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "traceID", Value: "bar"}, + }}, + Value: 19, + Ts: 1600096955579, + }, + { + Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, + }}, + Value: 19, + Ts: 1600096955479, + }, + // Same ts but different labels, cannot dedup. + { + Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "traceID", Value: "test"}, + }}, + Value: 19, + Ts: 1600096955479, + }, + }, }, }, - want: []*exemplarspb.Exemplar{ + want: []*exemplarspb.ExemplarData{ { - Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ - {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, + SeriesLabels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "__name__", Value: "test_exemplar_metric_total"}, + {Name: "instance", Value: "localhost:8090"}, + {Name: "job", Value: "prometheus"}, + {Name: "service", Value: "bar"}, }}, - Value: 19, - Ts: 1600096955479, + Exemplars: []*exemplarspb.Exemplar{ + { + Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "traceID", Value: "foo"}, + }}, + Value: 19, + Ts: 1600096955470, + }, + { + Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "traceID", Value: "EpTxMJ40fUus7aGY"}, + }}, + Value: 19, + Ts: 1600096955479, + }, + { + Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "traceID", Value: "test"}, + }}, + Value: 19, + Ts: 1600096955479, + }, + { + Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "traceID", Value: "bar"}, + }}, + Value: 19, + Ts: 1600096955579, + }, + }, }, }, }, @@ -256,7 +223,7 @@ func TestDedupExemplars(t *testing.T) { for _, lbl := range tc.replicaLabels { replicaLabels[lbl] = struct{}{} } - testutil.Equals(t, tc.want, dedupExemplars(tc.exemplars, replicaLabels)) + testutil.Equals(t, tc.want, dedupExemplarsResponse(tc.exemplars, replicaLabels)) }) } } diff --git a/pkg/exemplars/exemplarspb/custom.go b/pkg/exemplars/exemplarspb/custom.go index ac17cb2157..0c331e5dd4 100644 --- a/pkg/exemplars/exemplarspb/custom.go +++ b/pkg/exemplars/exemplarspb/custom.go @@ -66,6 +66,7 @@ func NewWarningExemplarsResponse(warning error) *ExemplarsResponse { } } +// Compare only compares the series labels of two exemplar data. func (s1 *ExemplarData) Compare(s2 *ExemplarData) int { return labels.Compare(s1.SeriesLabels.PromLabels(), s2.SeriesLabels.PromLabels()) } @@ -80,26 +81,17 @@ func (s *ExemplarData) SetSeriesLabels(ls labels.Labels) { s.SeriesLabels = result } -func (e *Exemplar) SetLabels(ls labels.Labels) { - var result labelpb.ZLabelSet - - if len(ls) > 0 { - result = labelpb.ZLabelSet{Labels: labelpb.ZLabelsFromPromLabels(ls)} - } - - e.Labels = result -} - +// Compare is used for sorting and comparing exemplars. Start from timestamp, then labels, finally values. func (e1 *Exemplar) Compare(e2 *Exemplar) int { - if d := labels.Compare(e1.Labels.PromLabels(), e2.Labels.PromLabels()); d != 0 { - return d - } if e1.Ts < e2.Ts { - return 1 + return -1 } if e1.Ts > e2.Ts { - return -1 + return 1 } + if d := labels.Compare(e1.Labels.PromLabels(), e2.Labels.PromLabels()); d != 0 { + return d + } return big.NewFloat(e1.Value).Cmp(big.NewFloat(e2.Value)) } diff --git a/pkg/exemplars/prometheus.go b/pkg/exemplars/prometheus.go index a76d8e6fa2..2ab3b5f648 100644 --- a/pkg/exemplars/prometheus.go +++ b/pkg/exemplars/prometheus.go @@ -37,22 +37,13 @@ func (p *Prometheus) Exemplars(r *exemplarspb.ExemplarsRequest, s exemplarspb.Ex } // Prometheus does not add external labels, so we need to add on our own. - enrichExemplarsWithExtLabels(exemplars, p.extLabels()) - + extLset := p.extLabels() for _, e := range exemplars { + // Make sure the returned series labels are sorted. + e.SetSeriesLabels(labelpb.ExtendSortedLabels(e.SeriesLabels.PromLabels(), extLset)) if err := s.Send(&exemplarspb.ExemplarsResponse{Result: &exemplarspb.ExemplarsResponse_Data{Data: e}}); err != nil { return err } } return nil } - -func enrichExemplarsWithExtLabels(exemplars []*exemplarspb.ExemplarData, extLset labels.Labels) { - for _, d := range exemplars { - d.SetSeriesLabels(labelpb.ExtendSortedLabels(d.SeriesLabels.PromLabels(), extLset)) - for i, e := range d.Exemplars { - e.SetLabels(labelpb.ExtendSortedLabels(e.Labels.PromLabels(), extLset)) - d.Exemplars[i] = e - } - } -}