From 977b0b241328dc2311adacd536461f9dab480a75 Mon Sep 17 00:00:00 2001 From: Arianna Vespri Date: Sat, 15 Jun 2024 18:12:43 +0200 Subject: [PATCH 01/11] Start on native histogram parser Signed-off-by: Arianna Vespri --- prometheus_client/core.py | 4 +- prometheus_client/metrics.py | 13 +- prometheus_client/metrics_core.py | 13 +- prometheus_client/openmetrics/exposition.py | 4 +- prometheus_client/openmetrics/parser.py | 144 +++++++++++++++----- prometheus_client/samples.py | 23 +++- tests/openmetrics/test_parser.py | 16 ++- 7 files changed, 166 insertions(+), 51 deletions(-) diff --git a/prometheus_client/core.py b/prometheus_client/core.py index ad3a4542..4f563966 100644 --- a/prometheus_client/core.py +++ b/prometheus_client/core.py @@ -5,9 +5,10 @@ SummaryMetricFamily, UnknownMetricFamily, UntypedMetricFamily, ) from .registry import CollectorRegistry, REGISTRY -from .samples import Exemplar, Sample, Timestamp +from .samples import BucketSpan, Exemplar, NativeHistStructValue, Sample, Timestamp __all__ = ( + 'BucketSpan', 'CollectorRegistry', 'Counter', 'CounterMetricFamily', @@ -21,6 +22,7 @@ 'Info', 'InfoMetricFamily', 'Metric', + 'NativeHistStructValue', 'REGISTRY', 'Sample', 'StateSetMetricFamily', diff --git a/prometheus_client/metrics.py b/prometheus_client/metrics.py index af512115..700c0259 100644 --- a/prometheus_client/metrics.py +++ b/prometheus_client/metrics.py @@ -1,3 +1,4 @@ +from datetime import timedelta import os from threading import Lock import time @@ -15,7 +16,7 @@ RESERVED_METRIC_LABEL_NAME_RE, ) from .registry import Collector, CollectorRegistry, REGISTRY -from .samples import Exemplar, Sample +from .samples import BucketSpan, Exemplar, NativeHistStructValue, Sample from .utils import floatToGoString, INF T = TypeVar('T', bound='MetricWrapperBase') @@ -595,6 +596,14 @@ def __init__(self, registry: Optional[CollectorRegistry] = REGISTRY, _labelvalues: Optional[Sequence[str]] = None, buckets: Sequence[Union[float, str]] = DEFAULT_BUCKETS, + # native_hist_schema: Optional[int] = None, # create this dynamically? + # native_hist_bucket_fact: Optional[float] = None, + # native_hist_zero_threshold: Optional[float] = None, + # native_hist_max_bucket_num: Optional[int] = None, + # native_hist_min_reset_dur: Optional[timedelta] = None, + # native_hist_max_zero_threshold: Optional[float] = None, + # native_hist_max_exemplars: Optional[int] = None, + # native_hist_exemplar_TTL: Optional[timedelta] = None, ): self._prepare_buckets(buckets) super().__init__( @@ -773,4 +782,4 @@ def _child_samples(self) -> Iterable[Sample]: Sample('', {self._name: s}, 1 if i == self._value else 0, None, None) for i, s in enumerate(self._states) - ] + ] \ No newline at end of file diff --git a/prometheus_client/metrics_core.py b/prometheus_client/metrics_core.py index 7226d920..95c79a1e 100644 --- a/prometheus_client/metrics_core.py +++ b/prometheus_client/metrics_core.py @@ -1,7 +1,8 @@ +from datetime import timedelta import re from typing import Dict, List, Optional, Sequence, Tuple, Union -from .samples import Exemplar, Sample, Timestamp +from .samples import BucketSpan, Exemplar, NativeHistStructValue, Sample, Timestamp METRIC_TYPES = ( 'counter', 'gauge', 'summary', 'histogram', @@ -36,11 +37,14 @@ def __init__(self, name: str, documentation: str, typ: str, unit: str = ''): self.type: str = typ self.samples: List[Sample] = [] - def add_sample(self, name: str, labels: Dict[str, str], value: float, timestamp: Optional[Union[Timestamp, float]] = None, exemplar: Optional[Exemplar] = None) -> None: + def add_sample(self, name: str, labels: Union[Dict[str, str], None], value: Union[float, NativeHistStructValue], timestamp: Optional[Union[Timestamp, float]] = None, exemplar: Optional[Exemplar] = None) -> None: """Add a sample to the metric. Internal-only, do not use.""" - self.samples.append(Sample(name, labels, value, timestamp, exemplar)) + if not isinstance(value, NativeHistStructValue): + self.samples.append(Sample(name, labels, value, timestamp, exemplar)) + else: + self.samples.append(Sample(name, labels, value)) def __eq__(self, other: object) -> bool: return (isinstance(other, Metric) @@ -236,6 +240,7 @@ def __init__(self, sum_value: Optional[float] = None, labels: Optional[Sequence[str]] = None, unit: str = '', + native_hist_bucket_factor: Optional[float] = None ): Metric.__init__(self, name, documentation, 'histogram', unit) if sum_value is not None and buckets is None: @@ -283,8 +288,6 @@ def add_metric(self, self.samples.append( Sample(self.name + '_sum', dict(zip(self._labelnames, labels)), sum_value, timestamp)) - - class GaugeHistogramMetricFamily(Metric): """A single gauge histogram and its samples. diff --git a/prometheus_client/openmetrics/exposition.py b/prometheus_client/openmetrics/exposition.py index 26f3109f..1959847b 100644 --- a/prometheus_client/openmetrics/exposition.py +++ b/prometheus_client/openmetrics/exposition.py @@ -10,7 +10,9 @@ def _is_valid_exemplar_metric(metric, sample): if metric.type == 'counter' and sample.name.endswith('_total'): return True - if metric.type in ('histogram', 'gaugehistogram') and sample.name.endswith('_bucket'): + if metric.type in ('gaugehistogram') and sample.name.endswith('_bucket'): + return True + if metric.type in ('histogram') and sample.name.endswith('_bucket') or sample.name == metric.name: return True return False diff --git a/prometheus_client/openmetrics/parser.py b/prometheus_client/openmetrics/parser.py index 6128a0d3..2a28cb02 100644 --- a/prometheus_client/openmetrics/parser.py +++ b/prometheus_client/openmetrics/parser.py @@ -1,12 +1,13 @@ #!/usr/bin/env python +from ast import literal_eval import io as StringIO import math import re from ..metrics_core import Metric, METRIC_LABEL_NAME_RE -from ..samples import Exemplar, Sample, Timestamp +from ..samples import BucketSpan, Exemplar, NativeHistStructValue, Sample, Timestamp from ..utils import floatToGoString @@ -277,7 +278,6 @@ def _parse_sample(text): value, timestamp, exemplar = _parse_remaining_text(remaining_text) return Sample(name, labels, value, timestamp, exemplar) - def _parse_remaining_text(text): split_text = text.split(" ", 1) val = _parse_value(split_text[0]) @@ -363,6 +363,70 @@ def _parse_remaining_text(text): return val, ts, exemplar +def _parse_nh_sample(text, suffixes): + label_start = text.find("{") + # check if it's a native histogram with labels + re_nh_without_labels = re.compile(r'^[^{} ]+ {[^{}]+}$') + re_nh_with_labels = re.compile(r'[^{} ]+{[^{}]+} {[^{}]+}$') + ph = text.find("}") + 2 + print('we are matching \'{}\''.format(text)) + if re_nh_with_labels.match(text): + is_nh_with_labels = True + label_end = text.find("}") + label = text[label_start + 1:label_end] + labels = _parse_labels(label) + name_end = label_start - 1 + name = text[:name_end] + if name.endswith(suffixes): + raise ValueError("the sample name of a native histogram with labels should have no suffixes", name) + nh_value_start = text.rindex("{") + nh_value = text[nh_value_start:] + value = _parse_nh_struct(nh_value) + return Sample(name, labels, value) + # check if it's a native histogram + if re_nh_without_labels.match(text): + is_nh_with_labels = False + nh_value_start = label_start + nh_value = text[nh_value_start:] + name_end = nh_value_start - 1 + name = text[:name_end] + if name.endswith(suffixes): + raise ValueError("the sample name of a native histogram should have no suffixes", name) + value = _parse_nh_struct(nh_value) + return Sample(name, None, value) + else: + # it's not a native histogram + return + +def _parse_nh_struct(text): + + pattern = r'(\w+):\s*([^,}]+)' + + items = dict(re.findall(pattern, text)) + + count_value = float(items['count']) + sum_value = float(items['sum']) + schema = int(items['schema']) + zero_threshold = float(items['zero_threshold']) + zero_count = float(items['zero_count']) + + pos_spans = tuple(BucketSpan(*map(int, span.split(':'))) for span in literal_eval(items['positive_spans'])) if 'positive_spans' in items else None + neg_spans = tuple(BucketSpan(*map(int, span.split(':'))) for span in literal_eval(items['negative_spans'])) if 'negative_spans' in items else None + pos_deltas = literal_eval(items['positive_deltas']) if 'positive_deltas' in items else None + neg_deltas = literal_eval(items['negative_deltas']) if 'negative_deltas' in items else None + + return NativeHistStructValue( + count_value=count_value, + sum_value=sum_value, + schema=schema, + zero_threshold=zero_threshold, + zero_count=zero_count, + pos_spans=pos_spans, + neg_spans=neg_spans, + pos_deltas=pos_deltas, + neg_deltas=neg_deltas + ) + def _group_for_sample(sample, name, typ): if typ == 'info': @@ -406,37 +470,38 @@ def do_checks(): for s in samples: suffix = s.name[len(name):] g = _group_for_sample(s, name, 'histogram') - if g != group or s.timestamp != timestamp: - if group is not None: - do_checks() - count = None - bucket = None - has_negative_buckets = False - has_sum = False - has_gsum = False - has_negative_gsum = False - value = 0 - group = g - timestamp = s.timestamp - - if suffix == '_bucket': - b = float(s.labels['le']) - if b < 0: - has_negative_buckets = True - if bucket is not None and b <= bucket: - raise ValueError("Buckets out of order: " + name) - if s.value < value: - raise ValueError("Bucket values out of order: " + name) - bucket = b - value = s.value - elif suffix in ['_count', '_gcount']: - count = s.value - elif suffix in ['_sum']: - has_sum = True - elif suffix in ['_gsum']: - has_gsum = True - if s.value < 0: - has_negative_gsum = True + if len(suffix) != 0: + if g != group or s.timestamp != timestamp: + if group is not None: + do_checks() + count = None + bucket = None + has_negative_buckets = False + has_sum = False + has_gsum = False + has_negative_gsum = False + value = 0 + group = g + timestamp = s.timestamp + + if suffix == '_bucket': + b = float(s.labels['le']) + if b < 0: + has_negative_buckets = True + if bucket is not None and b <= bucket: + raise ValueError("Buckets out of order: " + name) + if s.value < value: + raise ValueError("Bucket values out of order: " + name) + bucket = b + value = s.value + elif suffix in ['_count', '_gcount']: + count = s.value + elif suffix in ['_sum']: + has_sum = True + elif suffix in ['_gsum']: + has_gsum = True + if s.value < 0: + has_negative_gsum = True if group is not None: do_checks() @@ -486,6 +551,7 @@ def build_metric(name, documentation, typ, unit, samples): metric.samples = samples return metric + typ = None for line in fd: if line[-1] == '\n': line = line[:-1] @@ -498,6 +564,7 @@ def build_metric(name, documentation, typ, unit, samples): if line == '# EOF': eof = True + elif line.startswith('#'): parts = line.split(' ', 3) if len(parts) < 4: @@ -518,7 +585,7 @@ def build_metric(name, documentation, typ, unit, samples): group_timestamp_samples = set() samples = [] allowed_names = [parts[2]] - + if parts[1] == 'HELP': if documentation is not None: raise ValueError("More than one HELP for metric: " + line) @@ -537,7 +604,14 @@ def build_metric(name, documentation, typ, unit, samples): else: raise ValueError("Invalid line: " + line) else: - sample = _parse_sample(line) + if typ == 'histogram': + print("THis is typ",typ) + sample = _parse_nh_sample(line, tuple(type_suffixes['histogram'])) + print("THIS IS THE SAMPLE", sample) + else: + sample = None + if sample is None: + sample = _parse_sample(line) if sample.name not in allowed_names: if name is not None: yield build_metric(name, documentation, typ, unit, samples) diff --git a/prometheus_client/samples.py b/prometheus_client/samples.py index 53c47264..4550efe0 100644 --- a/prometheus_client/samples.py +++ b/prometheus_client/samples.py @@ -1,4 +1,4 @@ -from typing import Dict, NamedTuple, Optional, Union +from typing import Dict, NamedTuple, Optional, Sequence, Tuple, Union class Timestamp: @@ -33,6 +33,20 @@ def __gt__(self, other: "Timestamp") -> bool: def __lt__(self, other: "Timestamp") -> bool: return self.nsec < other.nsec if self.sec == other.sec else self.sec < other.sec +class BucketSpan(NamedTuple): + offset: int + length: int + +class NativeHistStructValue(NamedTuple): + count_value: float + sum_value: float + schema: int + zero_threshold: float + zero_count: float + pos_spans: Optional[Tuple[BucketSpan, BucketSpan ]] = None + neg_spans: Optional[Tuple[BucketSpan, BucketSpan ]] = None + pos_deltas: Optional[Sequence[int]] = None + neg_deltas: Optional[Sequence[int]] = None # Timestamp and exemplar are optional. # Value can be an int or a float. @@ -44,10 +58,11 @@ class Exemplar(NamedTuple): value: float timestamp: Optional[Union[float, Timestamp]] = None - class Sample(NamedTuple): name: str - labels: Dict[str, str] - value: float + labels: Union[Dict[str, str], None] + value: Union[float, NativeHistStructValue] timestamp: Optional[Union[float, Timestamp]] = None exemplar: Optional[Exemplar] = None + + diff --git a/tests/openmetrics/test_parser.py b/tests/openmetrics/test_parser.py index 937aef5c..a4219f8a 100644 --- a/tests/openmetrics/test_parser.py +++ b/tests/openmetrics/test_parser.py @@ -2,9 +2,9 @@ import unittest from prometheus_client.core import ( - CollectorRegistry, CounterMetricFamily, Exemplar, + BucketSpan, CollectorRegistry, CounterMetricFamily, Exemplar, GaugeHistogramMetricFamily, GaugeMetricFamily, HistogramMetricFamily, - InfoMetricFamily, Metric, Sample, StateSetMetricFamily, + InfoMetricFamily, Metric, NativeHistStructValue, Sample, StateSetMetricFamily, SummaryMetricFamily, Timestamp, ) from prometheus_client.openmetrics.exposition import generate_latest @@ -175,7 +175,17 @@ def test_histogram_exemplars(self): Exemplar({"a": "2345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678"}, 4, Timestamp(123, 0))) self.assertEqual([hfm], list(families)) - + def test_native_histogram(self): + families = text_string_to_metric_families("""# TYPE nativehistogram histogram +# HELP nativehistogram Is a basic example of a native histogram. +nativehistogram {count:24,sum:100,schema:0,zero_threshold:0.001,zero_count:4,positive_spans:[0:2,1:2],negative_spans:[0:2,1:2],positive_deltas:[2,1,-3,3],negative_deltas:[2,1,-2,3]} +# EOF +""") + + hfm = HistogramMetricFamily("nativehistogram", "Is a basic example of a native histogram") + hfm.add_sample("nativehistogram", None, NativeHistStructValue(24, 100, 0, 000.1, 4, BucketSpan({0:2},{1:2}),BucketSpan({0:2},{1:2}),(2,1,-3,3),(2,1,-2,3))) + self.assertEqual([hfm], list(families)) + def test_simple_gaugehistogram(self): families = text_string_to_metric_families("""# TYPE a gaugehistogram # HELP a help From fd1b56360fb45728916efe6d78ad81c9bb4478f4 Mon Sep 17 00:00:00 2001 From: Arianna Vespri Date: Sun, 16 Jun 2024 11:05:33 +0200 Subject: [PATCH 02/11] Fix regex for nh sample Signed-off-by: Arianna Vespri --- prometheus_client/openmetrics/parser.py | 48 ++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/prometheus_client/openmetrics/parser.py b/prometheus_client/openmetrics/parser.py index 2a28cb02..a0251a7b 100644 --- a/prometheus_client/openmetrics/parser.py +++ b/prometheus_client/openmetrics/parser.py @@ -399,10 +399,15 @@ def _parse_nh_sample(text, suffixes): return def _parse_nh_struct(text): - pattern = r'(\w+):\s*([^,}]+)' + re_spans = re.compile(r'(positive_spans|negative_spans):\[(\d+:\d+,\d+:\d+)\]') + re_deltas = re.compile(r'(positive_deltas|negative_deltas):\[(-?\d+(?:,-?\d+)*)\]') + + print('Matching text {}'.format(text)) items = dict(re.findall(pattern, text)) + spans = dict(re_spans.findall(text)) + deltas = dict(re_deltas.findall(text)) count_value = float(items['count']) sum_value = float(items['sum']) @@ -410,10 +415,43 @@ def _parse_nh_struct(text): zero_threshold = float(items['zero_threshold']) zero_count = float(items['zero_count']) - pos_spans = tuple(BucketSpan(*map(int, span.split(':'))) for span in literal_eval(items['positive_spans'])) if 'positive_spans' in items else None - neg_spans = tuple(BucketSpan(*map(int, span.split(':'))) for span in literal_eval(items['negative_spans'])) if 'negative_spans' in items else None - pos_deltas = literal_eval(items['positive_deltas']) if 'positive_deltas' in items else None - neg_deltas = literal_eval(items['negative_deltas']) if 'negative_deltas' in items else None + try: + pos_spans_text = spans['positive_spans'] + except KeyError: + pos_spans = None + else: + elems = pos_spans_text.split(',') + arg1 = [int(x) for x in elems[0].split(':')] + arg2 = [int(x) for x in elems[1].split(':')] + pos_spans = (BucketSpan(arg1[0], arg1[1]), BucketSpan(arg2[0], arg2[1])) + try: + neg_spans_text = spans['negative_spans'] + except KeyError: + neg_spans = None + else: + elems = neg_spans_text.split(',') + arg1 = [int(x) for x in elems[0].split(':')] + arg2 = [int(x) for x in elems[1].split(':')] + neg_spans = (BucketSpan(arg1[0], arg1[1]), BucketSpan(arg2[0], arg2[1])) + print('Created pos_spans: {}'.format(pos_spans)) + print('Created neg_spans: {}'.format(neg_spans)) + + try: + pos_deltas_text = deltas['positive_deltas'] + except KeyError: + pos_deltas = None + else: + elems = pos_deltas_text.split(',') + pos_deltas = [int(x) for x in elems] + try: + neg_deltas_text = deltas['negative_deltas'] + except KeyError: + neg_deltas = None + else: + elems = neg_deltas_text.split(',') + neg_deltas = [int(x) for x in elems] + print('Positive deltas: {}'.format(pos_deltas)) + print('Negative deltas: {}'.format(neg_deltas)) return NativeHistStructValue( count_value=count_value, From e32d2a820da1b3125e95f8d6d6e169a23077bd00 Mon Sep 17 00:00:00 2001 From: Arianna Vespri Date: Tue, 18 Jun 2024 09:43:43 +0200 Subject: [PATCH 03/11] Get nh sample appended Signed-off-by: Arianna Vespri --- prometheus_client/openmetrics/parser.py | 57 ++++++++++++++----------- tests/openmetrics/test_parser.py | 12 +++++- 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/prometheus_client/openmetrics/parser.py b/prometheus_client/openmetrics/parser.py index a0251a7b..3ab54aa0 100644 --- a/prometheus_client/openmetrics/parser.py +++ b/prometheus_client/openmetrics/parser.py @@ -371,7 +371,6 @@ def _parse_nh_sample(text, suffixes): ph = text.find("}") + 2 print('we are matching \'{}\''.format(text)) if re_nh_with_labels.match(text): - is_nh_with_labels = True label_end = text.find("}") label = text[label_start + 1:label_end] labels = _parse_labels(label) @@ -385,7 +384,6 @@ def _parse_nh_sample(text, suffixes): return Sample(name, labels, value) # check if it's a native histogram if re_nh_without_labels.match(text): - is_nh_with_labels = False nh_value_start = label_start nh_value = text[nh_value_start:] name_end = nh_value_start - 1 @@ -482,6 +480,7 @@ def _group_for_sample(sample, name, typ): d = sample.labels.copy() del d['le'] return d + print('Returning sample labels: {}'.format(sample.labels)) return sample.labels @@ -544,7 +543,6 @@ def do_checks(): if group is not None: do_checks() - def text_fd_to_metric_families(fd): """Parse Prometheus text format from a file descriptor. @@ -589,6 +587,7 @@ def build_metric(name, documentation, typ, unit, samples): metric.samples = samples return metric + is_nh = True typ = None for line in fd: if line[-1] == '\n': @@ -602,7 +601,6 @@ def build_metric(name, documentation, typ, unit, samples): if line == '# EOF': eof = True - elif line.startswith('#'): parts = line.split(' ', 3) if len(parts) < 4: @@ -647,10 +645,12 @@ def build_metric(name, documentation, typ, unit, samples): sample = _parse_nh_sample(line, tuple(type_suffixes['histogram'])) print("THIS IS THE SAMPLE", sample) else: + # It's not a native histogram sample = None if sample is None: + is_nh = False sample = _parse_sample(line) - if sample.name not in allowed_names: + if sample.name not in allowed_names and not is_nh: if name is not None: yield build_metric(name, documentation, typ, unit, samples) # Start an unknown metric. @@ -669,7 +669,7 @@ def build_metric(name, documentation, typ, unit, samples): raise ValueError("Stateset missing label: " + line) if (name + '_bucket' == sample.name and (sample.labels.get('le', "NaN") == "NaN" - or _isUncanonicalNumber(sample.labels['le']))): + or _isUncanonicalNumber(sample.labels['le']))): raise ValueError("Invalid le label: " + line) if (name + '_bucket' == sample.name and (not isinstance(sample.value, int) and not sample.value.is_integer())): @@ -679,29 +679,33 @@ def build_metric(name, documentation, typ, unit, samples): raise ValueError("Count value must be an integer: " + line) if (typ == 'summary' and name == sample.name and (not (0 <= float(sample.labels.get('quantile', -1)) <= 1) - or _isUncanonicalNumber(sample.labels['quantile']))): + or _isUncanonicalNumber(sample.labels['quantile']))): raise ValueError("Invalid quantile label: " + line) - g = tuple(sorted(_group_for_sample(sample, name, typ).items())) - if group is not None and g != group and g in seen_groups: - raise ValueError("Invalid metric grouping: " + line) - if group is not None and g == group: - if (sample.timestamp is None) != (group_timestamp is None): - raise ValueError("Mix of timestamp presence within a group: " + line) - if group_timestamp is not None and group_timestamp > sample.timestamp and typ != 'info': - raise ValueError("Timestamps went backwards within a group: " + line) + print("is nh?", is_nh) + if not is_nh: + g = tuple(sorted(_group_for_sample(sample, name, typ).items())) + if group is not None and g != group and g in seen_groups: + raise ValueError("Invalid metric grouping: " + line) + if group is not None and g == group: + if (sample.timestamp is None) != (group_timestamp is None): + raise ValueError("Mix of timestamp presence within a group: " + line) + if group_timestamp is not None and group_timestamp > sample.timestamp and typ != 'info': + raise ValueError("Timestamps went backwards within a group: " + line) + else: + group_timestamp_samples = set() + + series_id = (sample.name, tuple(sorted(sample.labels.items()))) + if sample.timestamp != group_timestamp or series_id not in group_timestamp_samples: + # Not a duplicate due to timestamp truncation. + samples.append(sample) + group_timestamp_samples.add(series_id) + + group = g + group_timestamp = sample.timestamp + seen_groups.add(g) else: - group_timestamp_samples = set() - - series_id = (sample.name, tuple(sorted(sample.labels.items()))) - if sample.timestamp != group_timestamp or series_id not in group_timestamp_samples: - # Not a duplicate due to timestamp truncation. samples.append(sample) - group_timestamp_samples.add(series_id) - - group = g - group_timestamp = sample.timestamp - seen_groups.add(g) if typ == 'stateset' and sample.value not in [0, 1]: raise ValueError("Stateset samples can only have values zero and one: " + line) @@ -718,8 +722,9 @@ def build_metric(name, documentation, typ, unit, samples): (typ in ['histogram', 'gaugehistogram'] and sample.name.endswith('_bucket')) or (typ in ['counter'] and sample.name.endswith('_total'))): raise ValueError("Invalid line only histogram/gaugehistogram buckets and counters can have exemplars: " + line) - + if name is not None: + print('Building metric {}, {} samples'.format(name, len(samples))) yield build_metric(name, documentation, typ, unit, samples) if not eof: diff --git a/tests/openmetrics/test_parser.py b/tests/openmetrics/test_parser.py index a4219f8a..71602efe 100644 --- a/tests/openmetrics/test_parser.py +++ b/tests/openmetrics/test_parser.py @@ -177,14 +177,22 @@ def test_histogram_exemplars(self): self.assertEqual([hfm], list(families)) def test_native_histogram(self): families = text_string_to_metric_families("""# TYPE nativehistogram histogram -# HELP nativehistogram Is a basic example of a native histogram. +# HELP nativehistogram Is a basic example of a native histogram nativehistogram {count:24,sum:100,schema:0,zero_threshold:0.001,zero_count:4,positive_spans:[0:2,1:2],negative_spans:[0:2,1:2],positive_deltas:[2,1,-3,3],negative_deltas:[2,1,-2,3]} # EOF """) + families = list(families) hfm = HistogramMetricFamily("nativehistogram", "Is a basic example of a native histogram") hfm.add_sample("nativehistogram", None, NativeHistStructValue(24, 100, 0, 000.1, 4, BucketSpan({0:2},{1:2}),BucketSpan({0:2},{1:2}),(2,1,-3,3),(2,1,-2,3))) - self.assertEqual([hfm], list(families)) + from pprint import pprint + print('\nDumping hfm') + pprint(hfm) + print('\nDumping family') + + pprint(families[0]) + print('Done dumping hfm\n') + self.assertEqual([hfm], families) def test_simple_gaugehistogram(self): families = text_string_to_metric_families("""# TYPE a gaugehistogram From cb013d89a8ae468fcc2a9ce04965f23ccbdffc68 Mon Sep 17 00:00:00 2001 From: Arianna Vespri Date: Tue, 18 Jun 2024 11:53:43 +0200 Subject: [PATCH 04/11] Complete parsing for simple native histogram Signed-off-by: Arianna Vespri --- prometheus_client/openmetrics/parser.py | 22 +++++----------------- prometheus_client/samples.py | 6 +++--- tests/openmetrics/test_parser.py | 12 +++--------- 3 files changed, 11 insertions(+), 29 deletions(-) diff --git a/prometheus_client/openmetrics/parser.py b/prometheus_client/openmetrics/parser.py index 3ab54aa0..8f0d091d 100644 --- a/prometheus_client/openmetrics/parser.py +++ b/prometheus_client/openmetrics/parser.py @@ -402,16 +402,15 @@ def _parse_nh_struct(text): re_spans = re.compile(r'(positive_spans|negative_spans):\[(\d+:\d+,\d+:\d+)\]') re_deltas = re.compile(r'(positive_deltas|negative_deltas):\[(-?\d+(?:,-?\d+)*)\]') - print('Matching text {}'.format(text)) items = dict(re.findall(pattern, text)) spans = dict(re_spans.findall(text)) deltas = dict(re_deltas.findall(text)) - count_value = float(items['count']) - sum_value = float(items['sum']) + count_value = int(items['count']) + sum_value = int(items['sum']) schema = int(items['schema']) zero_threshold = float(items['zero_threshold']) - zero_count = float(items['zero_count']) + zero_count = int(items['zero_count']) try: pos_spans_text = spans['positive_spans'] @@ -431,26 +430,20 @@ def _parse_nh_struct(text): arg1 = [int(x) for x in elems[0].split(':')] arg2 = [int(x) for x in elems[1].split(':')] neg_spans = (BucketSpan(arg1[0], arg1[1]), BucketSpan(arg2[0], arg2[1])) - print('Created pos_spans: {}'.format(pos_spans)) - print('Created neg_spans: {}'.format(neg_spans)) - try: pos_deltas_text = deltas['positive_deltas'] except KeyError: pos_deltas = None else: elems = pos_deltas_text.split(',') - pos_deltas = [int(x) for x in elems] + pos_deltas = tuple([int(x) for x in elems]) try: neg_deltas_text = deltas['negative_deltas'] except KeyError: neg_deltas = None else: elems = neg_deltas_text.split(',') - neg_deltas = [int(x) for x in elems] - print('Positive deltas: {}'.format(pos_deltas)) - print('Negative deltas: {}'.format(neg_deltas)) - + neg_deltas = tuple([int(x) for x in elems]) return NativeHistStructValue( count_value=count_value, sum_value=sum_value, @@ -480,7 +473,6 @@ def _group_for_sample(sample, name, typ): d = sample.labels.copy() del d['le'] return d - print('Returning sample labels: {}'.format(sample.labels)) return sample.labels @@ -641,9 +633,7 @@ def build_metric(name, documentation, typ, unit, samples): raise ValueError("Invalid line: " + line) else: if typ == 'histogram': - print("THis is typ",typ) sample = _parse_nh_sample(line, tuple(type_suffixes['histogram'])) - print("THIS IS THE SAMPLE", sample) else: # It's not a native histogram sample = None @@ -682,7 +672,6 @@ def build_metric(name, documentation, typ, unit, samples): or _isUncanonicalNumber(sample.labels['quantile']))): raise ValueError("Invalid quantile label: " + line) - print("is nh?", is_nh) if not is_nh: g = tuple(sorted(_group_for_sample(sample, name, typ).items())) if group is not None and g != group and g in seen_groups: @@ -724,7 +713,6 @@ def build_metric(name, documentation, typ, unit, samples): raise ValueError("Invalid line only histogram/gaugehistogram buckets and counters can have exemplars: " + line) if name is not None: - print('Building metric {}, {} samples'.format(name, len(samples))) yield build_metric(name, documentation, typ, unit, samples) if not eof: diff --git a/prometheus_client/samples.py b/prometheus_client/samples.py index 4550efe0..7ef3b6e8 100644 --- a/prometheus_client/samples.py +++ b/prometheus_client/samples.py @@ -1,4 +1,4 @@ -from typing import Dict, NamedTuple, Optional, Sequence, Tuple, Union +from typing import Dict, List, NamedTuple, Optional, Sequence, Tuple, Union class Timestamp: @@ -43,8 +43,8 @@ class NativeHistStructValue(NamedTuple): schema: int zero_threshold: float zero_count: float - pos_spans: Optional[Tuple[BucketSpan, BucketSpan ]] = None - neg_spans: Optional[Tuple[BucketSpan, BucketSpan ]] = None + pos_spans: Optional[Tuple[BucketSpan, BucketSpan]] = None + neg_spans: Optional[Tuple[BucketSpan, BucketSpan]] = None pos_deltas: Optional[Sequence[int]] = None neg_deltas: Optional[Sequence[int]] = None diff --git a/tests/openmetrics/test_parser.py b/tests/openmetrics/test_parser.py index 71602efe..1bed49c7 100644 --- a/tests/openmetrics/test_parser.py +++ b/tests/openmetrics/test_parser.py @@ -175,6 +175,7 @@ def test_histogram_exemplars(self): Exemplar({"a": "2345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678"}, 4, Timestamp(123, 0))) self.assertEqual([hfm], list(families)) + def test_native_histogram(self): families = text_string_to_metric_families("""# TYPE nativehistogram histogram # HELP nativehistogram Is a basic example of a native histogram @@ -184,16 +185,9 @@ def test_native_histogram(self): families = list(families) hfm = HistogramMetricFamily("nativehistogram", "Is a basic example of a native histogram") - hfm.add_sample("nativehistogram", None, NativeHistStructValue(24, 100, 0, 000.1, 4, BucketSpan({0:2},{1:2}),BucketSpan({0:2},{1:2}),(2,1,-3,3),(2,1,-2,3))) - from pprint import pprint - print('\nDumping hfm') - pprint(hfm) - print('\nDumping family') - - pprint(families[0]) - print('Done dumping hfm\n') + hfm.add_sample("nativehistogram", None, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0,2),BucketSpan(1,2)),(BucketSpan(0,2), BucketSpan(1,2)),(2,1,-3,3),(2,1,-2,3))) self.assertEqual([hfm], families) - + def test_simple_gaugehistogram(self): families = text_string_to_metric_families("""# TYPE a gaugehistogram # HELP a help From 4b1f527b5f1d953a4e3187a963f01e8ab6a270bb Mon Sep 17 00:00:00 2001 From: Arianna Vespri Date: Thu, 20 Jun 2024 16:47:26 +0200 Subject: [PATCH 05/11] Add parsing for native histograms with labels, fix linting Signed-off-by: Arianna Vespri --- prometheus_client/metrics.py | 7 +++---- prometheus_client/metrics_core.py | 4 ++-- prometheus_client/openmetrics/parser.py | 26 +++++++++++++------------ prometheus_client/samples.py | 8 +++++--- tests/openmetrics/test_parser.py | 12 ++++++++++++ 5 files changed, 36 insertions(+), 21 deletions(-) diff --git a/prometheus_client/metrics.py b/prometheus_client/metrics.py index 700c0259..448e3af9 100644 --- a/prometheus_client/metrics.py +++ b/prometheus_client/metrics.py @@ -1,4 +1,3 @@ -from datetime import timedelta import os from threading import Lock import time @@ -16,7 +15,7 @@ RESERVED_METRIC_LABEL_NAME_RE, ) from .registry import Collector, CollectorRegistry, REGISTRY -from .samples import BucketSpan, Exemplar, NativeHistStructValue, Sample +from .samples import Exemplar, Sample from .utils import floatToGoString, INF T = TypeVar('T', bound='MetricWrapperBase') @@ -598,7 +597,7 @@ def __init__(self, buckets: Sequence[Union[float, str]] = DEFAULT_BUCKETS, # native_hist_schema: Optional[int] = None, # create this dynamically? # native_hist_bucket_fact: Optional[float] = None, - # native_hist_zero_threshold: Optional[float] = None, + # native_hist_zero_threshold: Optional[float] = None, # native_hist_max_bucket_num: Optional[int] = None, # native_hist_min_reset_dur: Optional[timedelta] = None, # native_hist_max_zero_threshold: Optional[float] = None, @@ -782,4 +781,4 @@ def _child_samples(self) -> Iterable[Sample]: Sample('', {self._name: s}, 1 if i == self._value else 0, None, None) for i, s in enumerate(self._states) - ] \ No newline at end of file + ] diff --git a/prometheus_client/metrics_core.py b/prometheus_client/metrics_core.py index 95c79a1e..77433546 100644 --- a/prometheus_client/metrics_core.py +++ b/prometheus_client/metrics_core.py @@ -1,8 +1,7 @@ -from datetime import timedelta import re from typing import Dict, List, Optional, Sequence, Tuple, Union -from .samples import BucketSpan, Exemplar, NativeHistStructValue, Sample, Timestamp +from .samples import Exemplar, NativeHistStructValue, Sample, Timestamp METRIC_TYPES = ( 'counter', 'gauge', 'summary', 'histogram', @@ -288,6 +287,7 @@ def add_metric(self, self.samples.append( Sample(self.name + '_sum', dict(zip(self._labelnames, labels)), sum_value, timestamp)) + class GaugeHistogramMetricFamily(Metric): """A single gauge histogram and its samples. diff --git a/prometheus_client/openmetrics/parser.py b/prometheus_client/openmetrics/parser.py index 8f0d091d..ca7a24ed 100644 --- a/prometheus_client/openmetrics/parser.py +++ b/prometheus_client/openmetrics/parser.py @@ -1,7 +1,6 @@ #!/usr/bin/env python -from ast import literal_eval import io as StringIO import math import re @@ -278,6 +277,7 @@ def _parse_sample(text): value, timestamp, exemplar = _parse_remaining_text(remaining_text) return Sample(name, labels, value, timestamp, exemplar) + def _parse_remaining_text(text): split_text = text.split(" ", 1) val = _parse_value(split_text[0]) @@ -363,28 +363,28 @@ def _parse_remaining_text(text): return val, ts, exemplar + def _parse_nh_sample(text, suffixes): - label_start = text.find("{") + labels_start = text.find("{") # check if it's a native histogram with labels re_nh_without_labels = re.compile(r'^[^{} ]+ {[^{}]+}$') re_nh_with_labels = re.compile(r'[^{} ]+{[^{}]+} {[^{}]+}$') - ph = text.find("}") + 2 print('we are matching \'{}\''.format(text)) if re_nh_with_labels.match(text): - label_end = text.find("}") - label = text[label_start + 1:label_end] - labels = _parse_labels(label) - name_end = label_start - 1 + nh_value_start = text.rindex("{") + labels_end = nh_value_start - 2 + labelstext = text[labels_start + 1:labels_end] + labels = _parse_labels(labelstext) + name_end = labels_start name = text[:name_end] if name.endswith(suffixes): - raise ValueError("the sample name of a native histogram with labels should have no suffixes", name) - nh_value_start = text.rindex("{") + raise ValueError("the sample name of a native histogram with labels should have no suffixes", name) nh_value = text[nh_value_start:] value = _parse_nh_struct(nh_value) return Sample(name, labels, value) # check if it's a native histogram if re_nh_without_labels.match(text): - nh_value_start = label_start + nh_value_start = labels_start nh_value = text[nh_value_start:] name_end = nh_value_start - 1 name = text[:name_end] @@ -396,6 +396,7 @@ def _parse_nh_sample(text, suffixes): # it's not a native histogram return + def _parse_nh_struct(text): pattern = r'(\w+):\s*([^,}]+)' @@ -535,6 +536,7 @@ def do_checks(): if group is not None: do_checks() + def text_fd_to_metric_families(fd): """Parse Prometheus text format from a file descriptor. @@ -659,7 +661,7 @@ def build_metric(name, documentation, typ, unit, samples): raise ValueError("Stateset missing label: " + line) if (name + '_bucket' == sample.name and (sample.labels.get('le', "NaN") == "NaN" - or _isUncanonicalNumber(sample.labels['le']))): + or _isUncanonicalNumber(sample.labels['le']))): raise ValueError("Invalid le label: " + line) if (name + '_bucket' == sample.name and (not isinstance(sample.value, int) and not sample.value.is_integer())): @@ -669,7 +671,7 @@ def build_metric(name, documentation, typ, unit, samples): raise ValueError("Count value must be an integer: " + line) if (typ == 'summary' and name == sample.name and (not (0 <= float(sample.labels.get('quantile', -1)) <= 1) - or _isUncanonicalNumber(sample.labels['quantile']))): + or _isUncanonicalNumber(sample.labels['quantile']))): raise ValueError("Invalid quantile label: " + line) if not is_nh: diff --git a/prometheus_client/samples.py b/prometheus_client/samples.py index 7ef3b6e8..e8cdfcd3 100644 --- a/prometheus_client/samples.py +++ b/prometheus_client/samples.py @@ -1,4 +1,4 @@ -from typing import Dict, List, NamedTuple, Optional, Sequence, Tuple, Union +from typing import Dict, NamedTuple, Optional, Sequence, Tuple, Union class Timestamp: @@ -33,10 +33,12 @@ def __gt__(self, other: "Timestamp") -> bool: def __lt__(self, other: "Timestamp") -> bool: return self.nsec < other.nsec if self.sec == other.sec else self.sec < other.sec + class BucketSpan(NamedTuple): offset: int length: int + class NativeHistStructValue(NamedTuple): count_value: float sum_value: float @@ -48,6 +50,7 @@ class NativeHistStructValue(NamedTuple): pos_deltas: Optional[Sequence[int]] = None neg_deltas: Optional[Sequence[int]] = None + # Timestamp and exemplar are optional. # Value can be an int or a float. # Timestamp can be a float containing a unixtime in seconds, @@ -58,11 +61,10 @@ class Exemplar(NamedTuple): value: float timestamp: Optional[Union[float, Timestamp]] = None + class Sample(NamedTuple): name: str labels: Union[Dict[str, str], None] value: Union[float, NativeHistStructValue] timestamp: Optional[Union[float, Timestamp]] = None exemplar: Optional[Exemplar] = None - - diff --git a/tests/openmetrics/test_parser.py b/tests/openmetrics/test_parser.py index 1bed49c7..ab6b41c0 100644 --- a/tests/openmetrics/test_parser.py +++ b/tests/openmetrics/test_parser.py @@ -188,6 +188,18 @@ def test_native_histogram(self): hfm.add_sample("nativehistogram", None, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0,2),BucketSpan(1,2)),(BucketSpan(0,2), BucketSpan(1,2)),(2,1,-3,3),(2,1,-2,3))) self.assertEqual([hfm], families) + def test_native_histogram_with_labels(self): + families = text_string_to_metric_families("""# TYPE hist_w_labels histogram +# HELP hist_w_labels Is a basic example of a native histogram with labels +hist_w_labels{foo="bar",baz="qux"} {count:24,sum:100,schema:0,zero_threshold:0.001,zero_count:4,positive_spans:[0:2,1:2],negative_spans:[0:2,1:2],positive_deltas:[2,1,-3,3],negative_deltas:[2,1,-2,3]} +# EOF +""") + families = list(families) + + hfm = HistogramMetricFamily("hist_w_labels", "Is a basic example of a native histogram with labels") + hfm.add_sample("hist_w_labels", {"foo": "bar", "baz": "qux"}, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0,2),BucketSpan(1,2)),(BucketSpan(0,2), BucketSpan(1,2)),(2,1,-3,3),(2,1,-2,3))) + self.assertEqual([hfm], families) + def test_simple_gaugehistogram(self): families = text_string_to_metric_families("""# TYPE a gaugehistogram # HELP a help From eb6d9de5951e99ca42b0fd8604e2d06aab03f421 Mon Sep 17 00:00:00 2001 From: Arianna Vespri Date: Tue, 2 Jul 2024 10:09:06 +0200 Subject: [PATCH 06/11] Mitigate type and style errors Signed-off-by: Arianna Vespri --- prometheus_client/bridge/graphite.py | 12 +++++++++++- prometheus_client/core.py | 4 +++- prometheus_client/metrics_core.py | 2 +- prometheus_client/openmetrics/parser.py | 4 +++- prometheus_client/registry.py | 3 ++- prometheus_client/samples.py | 2 +- tests/openmetrics/test_parser.py | 4 ++-- 7 files changed, 23 insertions(+), 8 deletions(-) diff --git a/prometheus_client/bridge/graphite.py b/prometheus_client/bridge/graphite.py index 8cadbedc..515a3d97 100755 --- a/prometheus_client/bridge/graphite.py +++ b/prometheus_client/bridge/graphite.py @@ -20,6 +20,13 @@ def _sanitize(s): return _INVALID_GRAPHITE_CHARS.sub('_', s) +def safe_float_convert(value): + try: + return float(value) + except ValueError: + return None + + class _RegularPush(threading.Thread): def __init__(self, pusher, interval, prefix): super().__init__() @@ -82,7 +89,9 @@ def push(self, prefix: str = '') -> None: for k, v in sorted(s.labels.items())]) else: labelstr = '' - output.append(f'{prefixstr}{_sanitize(s.name)}{labelstr} {float(s.value)} {now}\n') + # using a safe float convert on s.value as a temporary workaround while figuring out what to do + # in case value is a native histogram structured value, if that's ever a possibility + output.append(f'{prefixstr}{_sanitize(s.name)}{labelstr} {safe_float_convert(s.value)} {now}\n') conn = socket.create_connection(self._address, self._timeout) conn.sendall(''.join(output).encode('ascii')) @@ -92,3 +101,4 @@ def start(self, interval: float = 60.0, prefix: str = '') -> None: t = _RegularPush(self, interval, prefix) t.daemon = True t.start() + diff --git a/prometheus_client/core.py b/prometheus_client/core.py index 4f563966..55b35221 100644 --- a/prometheus_client/core.py +++ b/prometheus_client/core.py @@ -5,7 +5,9 @@ SummaryMetricFamily, UnknownMetricFamily, UntypedMetricFamily, ) from .registry import CollectorRegistry, REGISTRY -from .samples import BucketSpan, Exemplar, NativeHistStructValue, Sample, Timestamp +from .samples import ( + BucketSpan, Exemplar, NativeHistStructValue, Sample, Timestamp, +) __all__ = ( 'BucketSpan', diff --git a/prometheus_client/metrics_core.py b/prometheus_client/metrics_core.py index 77433546..25ba5f00 100644 --- a/prometheus_client/metrics_core.py +++ b/prometheus_client/metrics_core.py @@ -36,7 +36,7 @@ def __init__(self, name: str, documentation: str, typ: str, unit: str = ''): self.type: str = typ self.samples: List[Sample] = [] - def add_sample(self, name: str, labels: Union[Dict[str, str], None], value: Union[float, NativeHistStructValue], timestamp: Optional[Union[Timestamp, float]] = None, exemplar: Optional[Exemplar] = None) -> None: + def add_sample(self, name: str, labels: Dict[str, str], value: Union[float, NativeHistStructValue], timestamp: Optional[Union[Timestamp, float]] = None, exemplar: Optional[Exemplar] = None) -> None: """Add a sample to the metric. Internal-only, do not use.""" diff --git a/prometheus_client/openmetrics/parser.py b/prometheus_client/openmetrics/parser.py index ca7a24ed..8de5a4e0 100644 --- a/prometheus_client/openmetrics/parser.py +++ b/prometheus_client/openmetrics/parser.py @@ -6,7 +6,9 @@ import re from ..metrics_core import Metric, METRIC_LABEL_NAME_RE -from ..samples import BucketSpan, Exemplar, NativeHistStructValue, Sample, Timestamp +from ..samples import ( + BucketSpan, Exemplar, NativeHistStructValue, Sample, Timestamp, +) from ..utils import floatToGoString diff --git a/prometheus_client/registry.py b/prometheus_client/registry.py index 694e4bd8..e23cba2e 100644 --- a/prometheus_client/registry.py +++ b/prometheus_client/registry.py @@ -4,6 +4,7 @@ from typing import Dict, Iterable, List, Optional from .metrics_core import Metric +from .samples import NativeHistStructValue # Ideally this would be a Protocol, but Protocols are only available in Python >= 3.8. @@ -128,7 +129,7 @@ def _target_info_metric(self): m.add_sample('target_info', self._target_info, 1) return m - def get_sample_value(self, name: str, labels: Optional[Dict[str, str]] = None) -> Optional[float]: + def get_sample_value(self, name: str, labels: Optional[Dict[str, str]] = None) -> Optional[float|NativeHistStructValue]: """Returns the sample value, or None if not found. This is inefficient, and intended only for use in unittests. diff --git a/prometheus_client/samples.py b/prometheus_client/samples.py index e8cdfcd3..3fcc8855 100644 --- a/prometheus_client/samples.py +++ b/prometheus_client/samples.py @@ -64,7 +64,7 @@ class Exemplar(NamedTuple): class Sample(NamedTuple): name: str - labels: Union[Dict[str, str], None] + labels: Dict[str, str] value: Union[float, NativeHistStructValue] timestamp: Optional[Union[float, Timestamp]] = None exemplar: Optional[Exemplar] = None diff --git a/tests/openmetrics/test_parser.py b/tests/openmetrics/test_parser.py index ab6b41c0..6d2ba73c 100644 --- a/tests/openmetrics/test_parser.py +++ b/tests/openmetrics/test_parser.py @@ -185,7 +185,7 @@ def test_native_histogram(self): families = list(families) hfm = HistogramMetricFamily("nativehistogram", "Is a basic example of a native histogram") - hfm.add_sample("nativehistogram", None, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0,2),BucketSpan(1,2)),(BucketSpan(0,2), BucketSpan(1,2)),(2,1,-3,3),(2,1,-2,3))) + hfm.add_sample("nativehistogram", None, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) self.assertEqual([hfm], families) def test_native_histogram_with_labels(self): @@ -197,7 +197,7 @@ def test_native_histogram_with_labels(self): families = list(families) hfm = HistogramMetricFamily("hist_w_labels", "Is a basic example of a native histogram with labels") - hfm.add_sample("hist_w_labels", {"foo": "bar", "baz": "qux"}, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0,2),BucketSpan(1,2)),(BucketSpan(0,2), BucketSpan(1,2)),(2,1,-3,3),(2,1,-2,3))) + hfm.add_sample("hist_w_labels", {"foo": "bar", "baz": "qux"}, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) self.assertEqual([hfm], families) def test_simple_gaugehistogram(self): From 86f165a213e9705d3fb0dd578d20ebddb9f0bbe9 Mon Sep 17 00:00:00 2001 From: Arianna Vespri Date: Tue, 2 Jul 2024 15:48:57 +0200 Subject: [PATCH 07/11] Add test for parsing coexisting native and classic hist with simple label set Signed-off-by: Arianna Vespri --- tests/openmetrics/test_parser.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/openmetrics/test_parser.py b/tests/openmetrics/test_parser.py index 6d2ba73c..2199df19 100644 --- a/tests/openmetrics/test_parser.py +++ b/tests/openmetrics/test_parser.py @@ -200,6 +200,27 @@ def test_native_histogram_with_labels(self): hfm.add_sample("hist_w_labels", {"foo": "bar", "baz": "qux"}, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) self.assertEqual([hfm], families) + def test_native_histogram_with_classic_histogram(self): + families = text_string_to_metric_families("""# TYPE hist_w_classic histogram +# HELP hist_w_classic Is a basic example of a native histogram coexisting with a classic histogram +hist_w_classic{foo="bar"} {count:24,sum:100,schema:0,zero_threshold:0.001,zero_count:4,positive_spans:[0:2,1:2],negative_spans:[0:2,1:2],positive_deltas:[2,1,-3,3],negative_deltas:[2,1,-2,3]} +hist_w_classic_bucket{foo="bar",le="0.001"} 4 +hist_w_classic_bucket{foo="bar",le="+Inf"} 24 +hist_w_classic_count{foo="bar"} 24 +hist_w_classic_sum{foo="bar"} 100 +# EOF +""") + families = list(families) + + hfm = HistogramMetricFamily("hist_w_classic", "Is a basic example of a native histogram coexisting with a classic histogram") + hfm.add_sample("hist_w_classic", {"foo": "bar"}, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) + hfm.add_sample("hist_w_classic_bucket", {"foo": "bar", "le": "0.001"}, 4.0, None, None) + hfm.add_sample("hist_w_classic_bucket", {"foo": "bar", "le": "+Inf"}, 24.0, None, None) + hfm.add_sample("hist_w_classic_count", {"foo": "bar"}, 24.0, None, None) + hfm.add_sample("hist_w_classic_sum", {"foo": "bar"}, 100.0, None, None) + self.assertEqual([hfm], families) + + def test_simple_gaugehistogram(self): families = text_string_to_metric_families("""# TYPE a gaugehistogram # HELP a help From c69a500a5bc312b8e4a4eb59631acd583a21f335 Mon Sep 17 00:00:00 2001 From: Arianna Vespri Date: Sun, 7 Jul 2024 15:51:27 +0200 Subject: [PATCH 08/11] Solve error in Python 3.9 tests Signed-off-by: Arianna Vespri --- prometheus_client/registry.py | 4 ++-- tests/openmetrics/test_parser.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/prometheus_client/registry.py b/prometheus_client/registry.py index e23cba2e..6e4f5176 100644 --- a/prometheus_client/registry.py +++ b/prometheus_client/registry.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod import copy from threading import Lock -from typing import Dict, Iterable, List, Optional +from typing import Dict, Iterable, List, Optional, Union from .metrics_core import Metric from .samples import NativeHistStructValue @@ -129,7 +129,7 @@ def _target_info_metric(self): m.add_sample('target_info', self._target_info, 1) return m - def get_sample_value(self, name: str, labels: Optional[Dict[str, str]] = None) -> Optional[float|NativeHistStructValue]: + def get_sample_value(self, name: str, labels: Optional[Dict[str, str]] = None) -> Optional[Union[float, NativeHistStructValue]]: """Returns the sample value, or None if not found. This is inefficient, and intended only for use in unittests. diff --git a/tests/openmetrics/test_parser.py b/tests/openmetrics/test_parser.py index 2199df19..4b8c1e3d 100644 --- a/tests/openmetrics/test_parser.py +++ b/tests/openmetrics/test_parser.py @@ -4,8 +4,8 @@ from prometheus_client.core import ( BucketSpan, CollectorRegistry, CounterMetricFamily, Exemplar, GaugeHistogramMetricFamily, GaugeMetricFamily, HistogramMetricFamily, - InfoMetricFamily, Metric, NativeHistStructValue, Sample, StateSetMetricFamily, - SummaryMetricFamily, Timestamp, + InfoMetricFamily, Metric, NativeHistStructValue, Sample, + StateSetMetricFamily, SummaryMetricFamily, Timestamp, ) from prometheus_client.openmetrics.exposition import generate_latest from prometheus_client.openmetrics.parser import text_string_to_metric_families From c06db3fa6ada8f04645c8cc36edf57093f27a32c Mon Sep 17 00:00:00 2001 From: Arianna Vespri Date: Mon, 8 Jul 2024 14:58:48 +0200 Subject: [PATCH 09/11] Add test for native + classic histograms with more than a label set and adapt logic accordigly Signed-off-by: Arianna Vespri --- prometheus_client/metrics.py | 2 +- prometheus_client/openmetrics/parser.py | 11 +++++---- tests/openmetrics/test_parser.py | 31 ++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/prometheus_client/metrics.py b/prometheus_client/metrics.py index 448e3af9..8918f592 100644 --- a/prometheus_client/metrics.py +++ b/prometheus_client/metrics.py @@ -595,7 +595,7 @@ def __init__(self, registry: Optional[CollectorRegistry] = REGISTRY, _labelvalues: Optional[Sequence[str]] = None, buckets: Sequence[Union[float, str]] = DEFAULT_BUCKETS, - # native_hist_schema: Optional[int] = None, # create this dynamically? + # native_hist_schema: Optional[int] = None, # native_hist_bucket_fact: Optional[float] = None, # native_hist_zero_threshold: Optional[float] = None, # native_hist_max_bucket_num: Optional[int] = None, diff --git a/prometheus_client/openmetrics/parser.py b/prometheus_client/openmetrics/parser.py index 8de5a4e0..ac499566 100644 --- a/prometheus_client/openmetrics/parser.py +++ b/prometheus_client/openmetrics/parser.py @@ -583,7 +583,7 @@ def build_metric(name, documentation, typ, unit, samples): metric.samples = samples return metric - is_nh = True + is_nh = False typ = None for line in fd: if line[-1] == '\n': @@ -637,11 +637,14 @@ def build_metric(name, documentation, typ, unit, samples): raise ValueError("Invalid line: " + line) else: if typ == 'histogram': + # set to true to account for native histograms naming exceptions/sanitizing differences + is_nh = True sample = _parse_nh_sample(line, tuple(type_suffixes['histogram'])) - else: # It's not a native histogram - sample = None - if sample is None: + if sample is None: + is_nh = False + sample = _parse_sample(line) + else: is_nh = False sample = _parse_sample(line) if sample.name not in allowed_names and not is_nh: diff --git a/tests/openmetrics/test_parser.py b/tests/openmetrics/test_parser.py index 4b8c1e3d..b183f702 100644 --- a/tests/openmetrics/test_parser.py +++ b/tests/openmetrics/test_parser.py @@ -219,7 +219,36 @@ def test_native_histogram_with_classic_histogram(self): hfm.add_sample("hist_w_classic_count", {"foo": "bar"}, 24.0, None, None) hfm.add_sample("hist_w_classic_sum", {"foo": "bar"}, 100.0, None, None) self.assertEqual([hfm], families) - + + def test_native_plus_classic_histogram_two_labelsets(self): + families = text_string_to_metric_families("""# TYPE hist_w_classic_two_sets histogram +# HELP hist_w_classic_two_sets Is an example of a native histogram plus a classic histogram with two label sets +hist_w_classic_two_sets{foo="bar"} {count:24,sum:100,schema:0,zero_threshold:0.001,zero_count:4,positive_spans:[0:2,1:2],negative_spans:[0:2,1:2],positive_deltas:[2,1,-3,3],negative_deltas:[2,1,-2,3]} +hist_w_classic_two_sets_bucket{foo="bar",le="0.001"} 4 +hist_w_classic_two_sets_bucket{foo="bar",le="+Inf"} 24 +hist_w_classic_two_sets_count{foo="bar"} 24 +hist_w_classic_two_sets_sum{foo="bar"} 100 +hist_w_classic_two_sets{foo="baz"} {count:24,sum:100,schema:0,zero_threshold:0.001,zero_count:4,positive_spans:[0:2,1:2],negative_spans:[0:2,1:2],positive_deltas:[2,1,-3,3],negative_deltas:[2,1,-2,3]} +hist_w_classic_two_sets_bucket{foo="baz",le="0.001"} 4 +hist_w_classic_two_sets_bucket{foo="baz",le="+Inf"} 24 +hist_w_classic_two_sets_count{foo="baz"} 24 +hist_w_classic_two_sets_sum{foo="baz"} 100 +# EOF +""") + families = list(families) + + hfm = HistogramMetricFamily("hist_w_classic_two_sets", "Is an example of a native histogram plus a classic histogram with two label sets") + hfm.add_sample("hist_w_classic_two_sets", {"foo": "bar"}, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) + hfm.add_sample("hist_w_classic_two_sets_bucket", {"foo": "bar", "le": "0.001"}, 4.0, None, None) + hfm.add_sample("hist_w_classic_two_sets_bucket", {"foo": "bar", "le": "+Inf"}, 24.0, None, None) + hfm.add_sample("hist_w_classic_two_sets_count", {"foo": "bar"}, 24.0, None, None) + hfm.add_sample("hist_w_classic_two_sets_sum", {"foo": "bar"}, 100.0, None, None) + hfm.add_sample("hist_w_classic_two_sets", {"foo": "baz"}, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) + hfm.add_sample("hist_w_classic_two_sets_bucket", {"foo": "baz", "le": "0.001"}, 4.0, None, None) + hfm.add_sample("hist_w_classic_two_sets_bucket", {"foo": "baz", "le": "+Inf"}, 24.0, None, None) + hfm.add_sample("hist_w_classic_two_sets_count", {"foo": "baz"}, 24.0, None, None) + hfm.add_sample("hist_w_classic_two_sets_sum", {"foo": "baz"}, 100.0, None, None) + self.assertEqual([hfm], families) def test_simple_gaugehistogram(self): families = text_string_to_metric_families("""# TYPE a gaugehistogram From d394c71e08e667f5050ed3fd981638d340175229 Mon Sep 17 00:00:00 2001 From: Arianna Vespri Date: Fri, 23 Aug 2024 15:32:05 +0200 Subject: [PATCH 10/11] Separate native histogram from value field, improve conditional/try blocks Signed-off-by: Arianna Vespri --- prometheus_client/bridge/graphite.py | 11 +-- prometheus_client/core.py | 6 +- prometheus_client/metrics.py | 16 +--- prometheus_client/metrics_core.py | 9 +- prometheus_client/multiprocess.py | 2 +- prometheus_client/openmetrics/parser.py | 104 ++++++++++++------------ prometheus_client/registry.py | 5 +- prometheus_client/samples.py | 5 +- tests/openmetrics/test_parser.py | 39 ++++----- 9 files changed, 88 insertions(+), 109 deletions(-) diff --git a/prometheus_client/bridge/graphite.py b/prometheus_client/bridge/graphite.py index 515a3d97..b365f303 100755 --- a/prometheus_client/bridge/graphite.py +++ b/prometheus_client/bridge/graphite.py @@ -20,13 +20,6 @@ def _sanitize(s): return _INVALID_GRAPHITE_CHARS.sub('_', s) -def safe_float_convert(value): - try: - return float(value) - except ValueError: - return None - - class _RegularPush(threading.Thread): def __init__(self, pusher, interval, prefix): super().__init__() @@ -89,9 +82,7 @@ def push(self, prefix: str = '') -> None: for k, v in sorted(s.labels.items())]) else: labelstr = '' - # using a safe float convert on s.value as a temporary workaround while figuring out what to do - # in case value is a native histogram structured value, if that's ever a possibility - output.append(f'{prefixstr}{_sanitize(s.name)}{labelstr} {safe_float_convert(s.value)} {now}\n') + output.append(f'{prefixstr}{_sanitize(s.name)}{labelstr} {float(s.value)} {now}\n') conn = socket.create_connection(self._address, self._timeout) conn.sendall(''.join(output).encode('ascii')) diff --git a/prometheus_client/core.py b/prometheus_client/core.py index 55b35221..60f93ce1 100644 --- a/prometheus_client/core.py +++ b/prometheus_client/core.py @@ -5,9 +5,7 @@ SummaryMetricFamily, UnknownMetricFamily, UntypedMetricFamily, ) from .registry import CollectorRegistry, REGISTRY -from .samples import ( - BucketSpan, Exemplar, NativeHistStructValue, Sample, Timestamp, -) +from .samples import BucketSpan, Exemplar, NativeHistogram, Sample, Timestamp __all__ = ( 'BucketSpan', @@ -24,7 +22,7 @@ 'Info', 'InfoMetricFamily', 'Metric', - 'NativeHistStructValue', + 'NativeHistogram', 'REGISTRY', 'Sample', 'StateSetMetricFamily', diff --git a/prometheus_client/metrics.py b/prometheus_client/metrics.py index 8918f592..03e1e66a 100644 --- a/prometheus_client/metrics.py +++ b/prometheus_client/metrics.py @@ -111,8 +111,8 @@ def describe(self) -> Iterable[Metric]: def collect(self) -> Iterable[Metric]: metric = self._get_metric() - for suffix, labels, value, timestamp, exemplar in self._samples(): - metric.add_sample(self._name + suffix, labels, value, timestamp, exemplar) + for suffix, labels, value, timestamp, exemplar, native_histogram_value in self._samples(): + metric.add_sample(self._name + suffix, labels, value, timestamp, exemplar, native_histogram_value) return [metric] def __str__(self) -> str: @@ -246,8 +246,8 @@ def _multi_samples(self) -> Iterable[Sample]: metrics = self._metrics.copy() for labels, metric in metrics.items(): series_labels = list(zip(self._labelnames, labels)) - for suffix, sample_labels, value, timestamp, exemplar in metric._samples(): - yield Sample(suffix, dict(series_labels + list(sample_labels.items())), value, timestamp, exemplar) + for suffix, sample_labels, value, timestamp, exemplar, native_histogram_value in metric._samples(): + yield Sample(suffix, dict(series_labels + list(sample_labels.items())), value, timestamp, exemplar, native_histogram_value) def _child_samples(self) -> Iterable[Sample]: # pragma: no cover raise NotImplementedError('_child_samples() must be implemented by %r' % self) @@ -595,14 +595,6 @@ def __init__(self, registry: Optional[CollectorRegistry] = REGISTRY, _labelvalues: Optional[Sequence[str]] = None, buckets: Sequence[Union[float, str]] = DEFAULT_BUCKETS, - # native_hist_schema: Optional[int] = None, - # native_hist_bucket_fact: Optional[float] = None, - # native_hist_zero_threshold: Optional[float] = None, - # native_hist_max_bucket_num: Optional[int] = None, - # native_hist_min_reset_dur: Optional[timedelta] = None, - # native_hist_max_zero_threshold: Optional[float] = None, - # native_hist_max_exemplars: Optional[int] = None, - # native_hist_exemplar_TTL: Optional[timedelta] = None, ): self._prepare_buckets(buckets) super().__init__( diff --git a/prometheus_client/metrics_core.py b/prometheus_client/metrics_core.py index 25ba5f00..06e0e61d 100644 --- a/prometheus_client/metrics_core.py +++ b/prometheus_client/metrics_core.py @@ -1,7 +1,7 @@ import re from typing import Dict, List, Optional, Sequence, Tuple, Union -from .samples import Exemplar, NativeHistStructValue, Sample, Timestamp +from .samples import Exemplar, NativeHistogram, Sample, Timestamp METRIC_TYPES = ( 'counter', 'gauge', 'summary', 'histogram', @@ -36,14 +36,11 @@ def __init__(self, name: str, documentation: str, typ: str, unit: str = ''): self.type: str = typ self.samples: List[Sample] = [] - def add_sample(self, name: str, labels: Dict[str, str], value: Union[float, NativeHistStructValue], timestamp: Optional[Union[Timestamp, float]] = None, exemplar: Optional[Exemplar] = None) -> None: + def add_sample(self, name: str, labels: Dict[str, str], value: float, timestamp: Optional[Union[Timestamp, float]] = None, exemplar: Optional[Exemplar] = None, native_histogram: Optional[NativeHistogram] = None) -> None: """Add a sample to the metric. Internal-only, do not use.""" - if not isinstance(value, NativeHistStructValue): - self.samples.append(Sample(name, labels, value, timestamp, exemplar)) - else: - self.samples.append(Sample(name, labels, value)) + self.samples.append(Sample(name, labels, value, timestamp, exemplar, native_histogram)) def __eq__(self, other: object) -> bool: return (isinstance(other, Metric) diff --git a/prometheus_client/multiprocess.py b/prometheus_client/multiprocess.py index 7021b49a..2682190a 100644 --- a/prometheus_client/multiprocess.py +++ b/prometheus_client/multiprocess.py @@ -93,7 +93,7 @@ def _accumulate_metrics(metrics, accumulate): buckets = defaultdict(lambda: defaultdict(float)) samples_setdefault = samples.setdefault for s in metric.samples: - name, labels, value, timestamp, exemplar = s + name, labels, value, timestamp, exemplar, native_histogram_value = s if metric.type == 'gauge': without_pid_key = (name, tuple(l for l in labels if l[0] != 'pid')) if metric._multiprocess_mode in ('min', 'livemin'): diff --git a/prometheus_client/openmetrics/parser.py b/prometheus_client/openmetrics/parser.py index ac499566..a976016e 100644 --- a/prometheus_client/openmetrics/parser.py +++ b/prometheus_client/openmetrics/parser.py @@ -6,9 +6,7 @@ import re from ..metrics_core import Metric, METRIC_LABEL_NAME_RE -from ..samples import ( - BucketSpan, Exemplar, NativeHistStructValue, Sample, Timestamp, -) +from ..samples import BucketSpan, Exemplar, NativeHistogram, Sample, Timestamp from ..utils import floatToGoString @@ -373,6 +371,7 @@ def _parse_nh_sample(text, suffixes): re_nh_with_labels = re.compile(r'[^{} ]+{[^{}]+} {[^{}]+}$') print('we are matching \'{}\''.format(text)) if re_nh_with_labels.match(text): + print('nh without labels matches') nh_value_start = text.rindex("{") labels_end = nh_value_start - 2 labelstext = text[labels_start + 1:labels_end] @@ -382,8 +381,8 @@ def _parse_nh_sample(text, suffixes): if name.endswith(suffixes): raise ValueError("the sample name of a native histogram with labels should have no suffixes", name) nh_value = text[nh_value_start:] - value = _parse_nh_struct(nh_value) - return Sample(name, labels, value) + nat_hist_value = _parse_nh_struct(nh_value) + return Sample(name, labels, None, None, None, nat_hist_value) # check if it's a native histogram if re_nh_without_labels.match(text): nh_value_start = labels_start @@ -392,8 +391,8 @@ def _parse_nh_sample(text, suffixes): name = text[:name_end] if name.endswith(suffixes): raise ValueError("the sample name of a native histogram should have no suffixes", name) - value = _parse_nh_struct(nh_value) - return Sample(name, None, value) + nat_hist_value = _parse_nh_struct(nh_value) + return Sample(name, None, None, None, None, nat_hist_value) else: # it's not a native histogram return @@ -417,37 +416,37 @@ def _parse_nh_struct(text): try: pos_spans_text = spans['positive_spans'] - except KeyError: - pos_spans = None - else: elems = pos_spans_text.split(',') arg1 = [int(x) for x in elems[0].split(':')] arg2 = [int(x) for x in elems[1].split(':')] pos_spans = (BucketSpan(arg1[0], arg1[1]), BucketSpan(arg2[0], arg2[1])) + except KeyError: + pos_spans = None + try: neg_spans_text = spans['negative_spans'] - except KeyError: - neg_spans = None - else: elems = neg_spans_text.split(',') arg1 = [int(x) for x in elems[0].split(':')] arg2 = [int(x) for x in elems[1].split(':')] neg_spans = (BucketSpan(arg1[0], arg1[1]), BucketSpan(arg2[0], arg2[1])) + except KeyError: + neg_spans = None + try: pos_deltas_text = deltas['positive_deltas'] - except KeyError: - pos_deltas = None - else: elems = pos_deltas_text.split(',') pos_deltas = tuple([int(x) for x in elems]) + except KeyError: + pos_deltas = None + try: neg_deltas_text = deltas['negative_deltas'] - except KeyError: - neg_deltas = None - else: elems = neg_deltas_text.split(',') neg_deltas = tuple([int(x) for x in elems]) - return NativeHistStructValue( + except KeyError: + neg_deltas = None + + return NativeHistogram( count_value=count_value, sum_value=sum_value, schema=schema, @@ -502,38 +501,39 @@ def do_checks(): for s in samples: suffix = s.name[len(name):] g = _group_for_sample(s, name, 'histogram') - if len(suffix) != 0: - if g != group or s.timestamp != timestamp: - if group is not None: - do_checks() - count = None - bucket = None - has_negative_buckets = False - has_sum = False - has_gsum = False - has_negative_gsum = False - value = 0 - group = g - timestamp = s.timestamp - - if suffix == '_bucket': - b = float(s.labels['le']) - if b < 0: - has_negative_buckets = True - if bucket is not None and b <= bucket: - raise ValueError("Buckets out of order: " + name) - if s.value < value: - raise ValueError("Bucket values out of order: " + name) - bucket = b - value = s.value - elif suffix in ['_count', '_gcount']: - count = s.value - elif suffix in ['_sum']: - has_sum = True - elif suffix in ['_gsum']: - has_gsum = True - if s.value < 0: - has_negative_gsum = True + if len(suffix) == 0: + continue + if g != group or s.timestamp != timestamp: + if group is not None: + do_checks() + count = None + bucket = None + has_negative_buckets = False + has_sum = False + has_gsum = False + has_negative_gsum = False + value = 0 + group = g + timestamp = s.timestamp + + if suffix == '_bucket': + b = float(s.labels['le']) + if b < 0: + has_negative_buckets = True + if bucket is not None and b <= bucket: + raise ValueError("Buckets out of order: " + name) + if s.value < value: + raise ValueError("Bucket values out of order: " + name) + bucket = b + value = s.value + elif suffix in ['_count', '_gcount']: + count = s.value + elif suffix in ['_sum']: + has_sum = True + elif suffix in ['_gsum']: + has_gsum = True + if s.value < 0: + has_negative_gsum = True if group is not None: do_checks() diff --git a/prometheus_client/registry.py b/prometheus_client/registry.py index 6e4f5176..694e4bd8 100644 --- a/prometheus_client/registry.py +++ b/prometheus_client/registry.py @@ -1,10 +1,9 @@ from abc import ABC, abstractmethod import copy from threading import Lock -from typing import Dict, Iterable, List, Optional, Union +from typing import Dict, Iterable, List, Optional from .metrics_core import Metric -from .samples import NativeHistStructValue # Ideally this would be a Protocol, but Protocols are only available in Python >= 3.8. @@ -129,7 +128,7 @@ def _target_info_metric(self): m.add_sample('target_info', self._target_info, 1) return m - def get_sample_value(self, name: str, labels: Optional[Dict[str, str]] = None) -> Optional[Union[float, NativeHistStructValue]]: + def get_sample_value(self, name: str, labels: Optional[Dict[str, str]] = None) -> Optional[float]: """Returns the sample value, or None if not found. This is inefficient, and intended only for use in unittests. diff --git a/prometheus_client/samples.py b/prometheus_client/samples.py index 3fcc8855..a7860f95 100644 --- a/prometheus_client/samples.py +++ b/prometheus_client/samples.py @@ -39,7 +39,7 @@ class BucketSpan(NamedTuple): length: int -class NativeHistStructValue(NamedTuple): +class NativeHistogram(NamedTuple): count_value: float sum_value: float schema: int @@ -65,6 +65,7 @@ class Exemplar(NamedTuple): class Sample(NamedTuple): name: str labels: Dict[str, str] - value: Union[float, NativeHistStructValue] + value: float timestamp: Optional[Union[float, Timestamp]] = None exemplar: Optional[Exemplar] = None + native_histogram: Optional[NativeHistogram] = None diff --git a/tests/openmetrics/test_parser.py b/tests/openmetrics/test_parser.py index b183f702..dc5e9916 100644 --- a/tests/openmetrics/test_parser.py +++ b/tests/openmetrics/test_parser.py @@ -4,8 +4,8 @@ from prometheus_client.core import ( BucketSpan, CollectorRegistry, CounterMetricFamily, Exemplar, GaugeHistogramMetricFamily, GaugeMetricFamily, HistogramMetricFamily, - InfoMetricFamily, Metric, NativeHistStructValue, Sample, - StateSetMetricFamily, SummaryMetricFamily, Timestamp, + InfoMetricFamily, Metric, NativeHistogram, Sample, StateSetMetricFamily, + SummaryMetricFamily, Timestamp, ) from prometheus_client.openmetrics.exposition import generate_latest from prometheus_client.openmetrics.parser import text_string_to_metric_families @@ -185,7 +185,7 @@ def test_native_histogram(self): families = list(families) hfm = HistogramMetricFamily("nativehistogram", "Is a basic example of a native histogram") - hfm.add_sample("nativehistogram", None, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) + hfm.add_sample("nativehistogram", None, None, None, None, NativeHistogram(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) self.assertEqual([hfm], families) def test_native_histogram_with_labels(self): @@ -197,7 +197,7 @@ def test_native_histogram_with_labels(self): families = list(families) hfm = HistogramMetricFamily("hist_w_labels", "Is a basic example of a native histogram with labels") - hfm.add_sample("hist_w_labels", {"foo": "bar", "baz": "qux"}, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) + hfm.add_sample("hist_w_labels", {"foo": "bar", "baz": "qux"}, None, None, None, NativeHistogram(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) self.assertEqual([hfm], families) def test_native_histogram_with_classic_histogram(self): @@ -213,11 +213,11 @@ def test_native_histogram_with_classic_histogram(self): families = list(families) hfm = HistogramMetricFamily("hist_w_classic", "Is a basic example of a native histogram coexisting with a classic histogram") - hfm.add_sample("hist_w_classic", {"foo": "bar"}, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) - hfm.add_sample("hist_w_classic_bucket", {"foo": "bar", "le": "0.001"}, 4.0, None, None) - hfm.add_sample("hist_w_classic_bucket", {"foo": "bar", "le": "+Inf"}, 24.0, None, None) - hfm.add_sample("hist_w_classic_count", {"foo": "bar"}, 24.0, None, None) - hfm.add_sample("hist_w_classic_sum", {"foo": "bar"}, 100.0, None, None) + hfm.add_sample("hist_w_classic", {"foo": "bar"}, None, None, None, NativeHistogram(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) + hfm.add_sample("hist_w_classic_bucket", {"foo": "bar", "le": "0.001"}, 4.0, None, None, None) + hfm.add_sample("hist_w_classic_bucket", {"foo": "bar", "le": "+Inf"}, 24.0, None, None, None) + hfm.add_sample("hist_w_classic_count", {"foo": "bar"}, 24.0, None, None, None) + hfm.add_sample("hist_w_classic_sum", {"foo": "bar"}, 100.0, None, None, None) self.assertEqual([hfm], families) def test_native_plus_classic_histogram_two_labelsets(self): @@ -238,16 +238,16 @@ def test_native_plus_classic_histogram_two_labelsets(self): families = list(families) hfm = HistogramMetricFamily("hist_w_classic_two_sets", "Is an example of a native histogram plus a classic histogram with two label sets") - hfm.add_sample("hist_w_classic_two_sets", {"foo": "bar"}, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) - hfm.add_sample("hist_w_classic_two_sets_bucket", {"foo": "bar", "le": "0.001"}, 4.0, None, None) - hfm.add_sample("hist_w_classic_two_sets_bucket", {"foo": "bar", "le": "+Inf"}, 24.0, None, None) - hfm.add_sample("hist_w_classic_two_sets_count", {"foo": "bar"}, 24.0, None, None) - hfm.add_sample("hist_w_classic_two_sets_sum", {"foo": "bar"}, 100.0, None, None) - hfm.add_sample("hist_w_classic_two_sets", {"foo": "baz"}, NativeHistStructValue(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) - hfm.add_sample("hist_w_classic_two_sets_bucket", {"foo": "baz", "le": "0.001"}, 4.0, None, None) - hfm.add_sample("hist_w_classic_two_sets_bucket", {"foo": "baz", "le": "+Inf"}, 24.0, None, None) - hfm.add_sample("hist_w_classic_two_sets_count", {"foo": "baz"}, 24.0, None, None) - hfm.add_sample("hist_w_classic_two_sets_sum", {"foo": "baz"}, 100.0, None, None) + hfm.add_sample("hist_w_classic_two_sets", {"foo": "bar"}, None, None, None, NativeHistogram(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) + hfm.add_sample("hist_w_classic_two_sets_bucket", {"foo": "bar", "le": "0.001"}, 4.0, None, None, None) + hfm.add_sample("hist_w_classic_two_sets_bucket", {"foo": "bar", "le": "+Inf"}, 24.0, None, None, None) + hfm.add_sample("hist_w_classic_two_sets_count", {"foo": "bar"}, 24.0, None, None, None) + hfm.add_sample("hist_w_classic_two_sets_sum", {"foo": "bar"}, 100.0, None, None, None) + hfm.add_sample("hist_w_classic_two_sets", {"foo": "baz"}, None, None, None, NativeHistogram(24, 100, 0, 0.001, 4, (BucketSpan(0, 2), BucketSpan(1, 2)), (BucketSpan(0, 2), BucketSpan(1, 2)), (2, 1, -3, 3), (2, 1, -2, 3))) + hfm.add_sample("hist_w_classic_two_sets_bucket", {"foo": "baz", "le": "0.001"}, 4.0, None, None, None) + hfm.add_sample("hist_w_classic_two_sets_bucket", {"foo": "baz", "le": "+Inf"}, 24.0, None, None, None) + hfm.add_sample("hist_w_classic_two_sets_count", {"foo": "baz"}, 24.0, None, None, None) + hfm.add_sample("hist_w_classic_two_sets_sum", {"foo": "baz"}, 100.0, None, None, None) self.assertEqual([hfm], families) def test_simple_gaugehistogram(self): @@ -879,6 +879,7 @@ def test_invalid_input(self): ('# TYPE a histogram\na_bucket{le="+INF"} 0\n# EOF\n'), ('# TYPE a histogram\na_bucket{le="2"} 0\na_bucket{le="1"} 0\na_bucket{le="+Inf"} 0\n# EOF\n'), ('# TYPE a histogram\na_bucket{le="1"} 1\na_bucket{le="2"} 1\na_bucket{le="+Inf"} 0\n# EOF\n'), + ('# TYPE a histogram\na_bucket {} {}'), # Bad grouping or ordering. ('# TYPE a histogram\na_sum{a="1"} 0\na_sum{a="2"} 0\na_count{a="1"} 0\n# EOF\n'), ('# TYPE a histogram\na_bucket{a="1",le="1"} 0\na_bucket{a="2",le="+Inf""} ' From 90cd08ef74a23eb98a79ccf340fb220c7ac7f041 Mon Sep 17 00:00:00 2001 From: Arianna Vespri Date: Sun, 8 Sep 2024 11:39:09 +0200 Subject: [PATCH 11/11] Clean up debug lines, add warnings, delete unnecessary lines Signed-off-by: Arianna Vespri --- prometheus_client/bridge/graphite.py | 1 - prometheus_client/metrics_core.py | 1 - prometheus_client/openmetrics/parser.py | 2 -- prometheus_client/samples.py | 2 ++ 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/prometheus_client/bridge/graphite.py b/prometheus_client/bridge/graphite.py index b365f303..8cadbedc 100755 --- a/prometheus_client/bridge/graphite.py +++ b/prometheus_client/bridge/graphite.py @@ -92,4 +92,3 @@ def start(self, interval: float = 60.0, prefix: str = '') -> None: t = _RegularPush(self, interval, prefix) t.daemon = True t.start() - diff --git a/prometheus_client/metrics_core.py b/prometheus_client/metrics_core.py index 06e0e61d..19166e1d 100644 --- a/prometheus_client/metrics_core.py +++ b/prometheus_client/metrics_core.py @@ -236,7 +236,6 @@ def __init__(self, sum_value: Optional[float] = None, labels: Optional[Sequence[str]] = None, unit: str = '', - native_hist_bucket_factor: Optional[float] = None ): Metric.__init__(self, name, documentation, 'histogram', unit) if sum_value is not None and buckets is None: diff --git a/prometheus_client/openmetrics/parser.py b/prometheus_client/openmetrics/parser.py index a976016e..39a44dc2 100644 --- a/prometheus_client/openmetrics/parser.py +++ b/prometheus_client/openmetrics/parser.py @@ -369,9 +369,7 @@ def _parse_nh_sample(text, suffixes): # check if it's a native histogram with labels re_nh_without_labels = re.compile(r'^[^{} ]+ {[^{}]+}$') re_nh_with_labels = re.compile(r'[^{} ]+{[^{}]+} {[^{}]+}$') - print('we are matching \'{}\''.format(text)) if re_nh_with_labels.match(text): - print('nh without labels matches') nh_value_start = text.rindex("{") labels_end = nh_value_start - 2 labelstext = text[labels_start + 1:labels_end] diff --git a/prometheus_client/samples.py b/prometheus_client/samples.py index a7860f95..b57a5d48 100644 --- a/prometheus_client/samples.py +++ b/prometheus_client/samples.py @@ -34,11 +34,13 @@ def __lt__(self, other: "Timestamp") -> bool: return self.nsec < other.nsec if self.sec == other.sec else self.sec < other.sec +# BucketSpan is experimental and subject to change at any time. class BucketSpan(NamedTuple): offset: int length: int +# NativeHistogram is experimental and subject to change at any time. class NativeHistogram(NamedTuple): count_value: float sum_value: float