diff --git a/dashboards/lodestar_general.json b/dashboards/lodestar_general.json index 2841eaffcb49..35a8300f6a44 100644 --- a/dashboards/lodestar_general.json +++ b/dashboards/lodestar_general.json @@ -3875,7 +3875,7 @@ "h": 8, "w": 12, "x": 0, - "y": 7 + "y": 21 }, "id": 429, "options": { @@ -3952,10 +3952,10 @@ "overrides": [] }, "gridPos": { - "h": 3, - "w": 2, + "h": 4, + "w": 3, "x": 12, - "y": 7 + "y": 21 }, "id": 426, "options": { @@ -4004,10 +4004,10 @@ "overrides": [] }, "gridPos": { - "h": 3, - "w": 2, - "x": 14, - "y": 7 + "h": 4, + "w": 3, + "x": 15, + "y": 21 }, "id": 427, "options": { @@ -4052,17 +4052,18 @@ "value": null } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 3, - "w": 2, - "x": 16, - "y": 7 + "h": 4, + "w": 3, + "x": 18, + "y": 21 }, - "id": 435, + "id": 411, "options": { "colorMode": "value", "graphMode": "area", @@ -4082,14 +4083,14 @@ "targets": [ { "exemplar": false, - "expr": "lodestar_eth1_http_client_active_requests", + "expr": "lodestar_eth1_follow_distance_seconds_config", "hide": false, "interval": "", "legendFormat": "", "refId": "A" } ], - "title": "Actv Calls", + "title": "Follow Config", "type": "stat" }, { @@ -4106,17 +4107,17 @@ } ] }, - "unit": "s" + "unit": "dateTimeFromNow" }, "overrides": [] }, "gridPos": { - "h": 3, + "h": 4, "w": 3, - "x": 18, - "y": 7 + "x": 21, + "y": 21 }, - "id": 411, + "id": 431, "options": { "colorMode": "value", "graphMode": "area", @@ -4126,7 +4127,7 @@ "calcs": [ "lastNotNull" ], - "fields": "", + "fields": "/^Time$/", "values": false }, "text": {}, @@ -4136,20 +4137,54 @@ "targets": [ { "exemplar": false, - "expr": "lodestar_eth1_follow_distance_seconds_config", + "expr": "lodestar_eth1_last_fetched_block_timestamp", + "format": "time_series", "hide": false, + "instant": false, "interval": "", - "legendFormat": "", + "legendFormat": "eth1_last_fetched_block_timestamp", "refId": "A" } ], - "title": "Follow Config", + "title": "Last fetched", "type": "stat" }, { "datasource": null, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], "mappings": [], "thresholds": { "mode": "absolute", @@ -4157,50 +4192,48 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] }, - "unit": "dateTimeFromNow" + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 3, - "w": 3, - "x": 21, - "y": 7 + "h": 4, + "w": 12, + "x": 12, + "y": 25 }, - "id": 431, + "id": 423, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^Time$/", - "values": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" }, - "text": {}, - "textMode": "auto" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "8.0.6", + "pluginVersion": "8.2.2", "targets": [ { "exemplar": false, - "expr": "lodestar_eth1_last_fetched_block_timestamp", - "format": "time_series", + "expr": "lodestar_eth1_http_client_active_requests", "hide": false, - "instant": false, "interval": "", - "legendFormat": "eth1_last_fetched_block_timestamp", + "legendFormat": "{{routeId}}", "refId": "A" } ], - "title": "Last fetched", - "type": "stat" + "title": "Active requests", + "type": "timeseries" }, { "datasource": null, @@ -4257,12 +4290,12 @@ "overrides": [] }, "gridPos": { - "h": 5, + "h": 8, "w": 12, - "x": 12, - "y": 10 + "x": 0, + "y": 29 }, - "id": 423, + "id": 428, "options": { "legend": { "calcs": [], @@ -4278,92 +4311,264 @@ "targets": [ { "exemplar": false, - "expr": "lodestar_eth1_follow_distance_dynamic", + "expr": "rate(lodestar_eth1_blocks_fetched_total[$__rate_interval])", "hide": false, "interval": "", - "legendFormat": "eth1_follow_distance_dynamic", + "legendFormat": "blocks fetched", "refId": "A" } ], - "title": "Eth1 Follow Distance Dynamic", + "title": "Eth1 blocks fetched rate", "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": null, "fieldConfig": { "defaults": { - "unit": "s" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "fill": 1, - "fillGradient": 1, "gridPos": { - "h": 8, + "h": 4, "w": 12, - "x": 0, - "y": 15 - }, - "hiddenSeries": false, - "id": 384, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "x": 12, + "y": 29 }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", + "id": 474, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 0.5, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "8.2.2", "targets": [ { "exemplar": false, - "expr": "delta(lodestar_eth1_http_client_request_time_seconds_sum[$__rate_interval])/delta(lodestar_eth1_http_client_request_time_seconds_count[$__rate_interval])", - "format": "time_series", + "expr": "lodestar_eth1_follow_distance_dynamic", "hide": false, "interval": "", - "legendFormat": "{{routeId}}", + "legendFormat": "eth1_follow_distance_dynamic", "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Average response times", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ + "title": "Eth1 Follow Distance Dynamic", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 424, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_deposit_events_fetched_total", + "hide": false, + "interval": "", + "legendFormat": "Eth1_deposit_events_fetched", + "refId": "A" + } + ], + "title": "Eth1 Deposit Events Fetched", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "hiddenSeries": false, + "id": 384, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 0.5, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": false, + "expr": "delta(lodestar_eth1_http_client_request_time_seconds_sum[$__rate_interval])/delta(lodestar_eth1_http_client_request_time_seconds_count[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{routeId}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average response times", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ { "$$hashKey": "object:87", "format": "s", @@ -4377,9 +4582,127 @@ "show": false } ], - "yaxis": { - "align": false - } + "yaxis": { + "align": false + } + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "getBlockNumber" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 413, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_eth1_http_client_request_errors_total", + "interval": "", + "legendFormat": "{{routeId}}", + "refId": "A" + }, + { + "exemplar": false, + "expr": "rate(lodestar_eth1_deposit_tracker_update_errors_total[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "eth1_deposit_tracker_update_errors_total", + "refId": "B" + } + ], + "title": "Error rate", + "type": "timeseries" }, { "aliasColors": {}, @@ -4392,8 +4715,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 15 + "x": 0, + "y": 45 }, "hiddenSeries": false, "id": 434, @@ -4520,13 +4843,260 @@ }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 414, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ + { + "exemplar": false, + "expr": "rate(lodestar_eth1_http_client_request_used_fallback_url_total[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "{{routeId}}", + "refId": "A" + } + ], + "title": "Fallback url request rate", + "type": "timeseries" + } + ], + "title": "Eth1 Stats", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 388, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 23 + "y": 2 + }, + "hiddenSeries": false, + "id": 390, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 0.5, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": false, + "expr": "delta(lodestar_execution_engine_http_client_request_time_seconds_sum[$__rate_interval])/delta(lodestar_execution_engine_http_client_request_time_seconds_count[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "{{routeId}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Avg request durations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:87", + "format": "s", + "logBase": 2, + "show": true + }, + { + "$$hashKey": "object:88", + "format": "s", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 2 + }, + "id": 430, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "exemplar": false, + "expr": "lodestar_execution_engine_http_client_config_urls_count", + "hide": false, + "interval": "", + "legendFormat": "execution_engine_http_client_config_urls_count", + "refId": "A" + } + ], + "title": "Urls", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 9, + "x": 15, + "y": 2 }, - "id": 428, + "id": 420, "options": { "legend": { "calcs": [], @@ -4542,14 +5112,14 @@ "targets": [ { "exemplar": false, - "expr": "rate(lodestar_eth1_blocks_fetched_total[$__rate_interval])", + "expr": "lodestar_execution_engine_http_client_active_requests", "hide": false, "interval": "", - "legendFormat": "eth1_blocks_fetched", + "legendFormat": "{{routeId}}", "refId": "A" } ], - "title": "Eth1 Blocks Fetched", + "title": "Active requests", "type": "timeseries" }, { @@ -4589,6 +5159,7 @@ }, "links": [], "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ @@ -4604,40 +5175,15 @@ }, "unit": "short" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "getBlockNumber" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 23 + "y": 6 }, - "id": 413, + "id": 419, "options": { "legend": { "calcs": [], @@ -4650,25 +5196,191 @@ } }, "pluginVersion": "8.2.2", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 433, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 0.5, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "exemplar": false, - "expr": "lodestar_eth1_http_client_request_errors_total", + "expr": "12*rate(lodestar_execution_engine_http_client_request_time_seconds_count[$__rate_interval])", + "hide": false, "interval": "", "legendFormat": "{{routeId}}", "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests / slot", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:195", + "format": "none", + "logBase": 2, + "show": true + }, + { + "$$hashKey": "object:196", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [], + "unit": "short" }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "hiddenSeries": false, + "id": 468, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { "exemplar": false, - "expr": "rate(lodestar_eth1_deposit_tracker_update_errors_total[$__rate_interval])", + "expr": "rate(lodestar_execution_engine_http_client_request_retries_total[$__rate_interval])", "hide": false, "interval": "", - "legendFormat": "eth1_deposit_tracker_update_errors_total", - "refId": "B" + "legendFormat": "{{routeId}}", + "refId": "A" } ], - "title": "Error rate", - "type": "timeseries" + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request retries rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "datasource": null, @@ -4728,9 +5440,9 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 18 }, - "id": 414, + "id": 418, "options": { "legend": { "calcs": [], @@ -4746,14 +5458,14 @@ "targets": [ { "exemplar": false, - "expr": "lodestar_eth1_http_client_request_used_fallback_url_total", + "expr": "rate(lodestar_execution_engine_http_client_request_errors_total[$__rate_interval])", "hide": false, "interval": "", - "legendFormat": "count_of_requests_on_fallback_url(s)", + "legendFormat": "{{routeId}}", "refId": "A" } ], - "title": "count of requests on fallback url(s)", + "title": "Rate of Request errors", "type": "timeseries" }, { @@ -4793,6 +5505,7 @@ }, "links": [], "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ @@ -4814,9 +5527,9 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 18 }, - "id": 424, + "id": 473, "options": { "legend": { "calcs": [], @@ -4832,18 +5545,18 @@ "targets": [ { "exemplar": false, - "expr": "lodestar_eth1_deposit_events_fetched_total", + "expr": "rate(lodestar_execution_engine_http_client_request_used_fallback_url_total[$__rate_interval])", "hide": false, "interval": "", - "legendFormat": "Eth1_deposit_events_fetched", + "legendFormat": "{{routeId}}", "refId": "A" } ], - "title": "Eth1 Deposit Events Fetched", + "title": "Fallback Url requests(s)", "type": "timeseries" } ], - "title": "Eth1 Stats", + "title": "Execution Metrics", "type": "row" }, { @@ -4853,7 +5566,7 @@ "h": 1, "w": 24, "x": 0, - "y": 21 + "y": 22 }, "id": 437, "panels": [ @@ -5769,7 +6482,7 @@ "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 23 }, "id": 208, "panels": [ @@ -6771,7 +7484,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 24 }, "id": 108, "panels": [ @@ -7320,7 +8033,7 @@ "h": 1, "w": 24, "x": 0, - "y": 24 + "y": 25 }, "id": 92, "panels": [ @@ -8366,7 +9079,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 26 }, "id": 25, "panels": [ @@ -8887,7 +9600,7 @@ "h": 1, "w": 24, "x": 0, - "y": 26 + "y": 27 }, "id": 110, "panels": [ @@ -9399,7 +10112,7 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 28 }, "id": 136, "panels": [ @@ -9991,7 +10704,7 @@ "h": 1, "w": 24, "x": 0, - "y": 28 + "y": 29 }, "id": 75, "panels": [ @@ -10703,7 +11416,7 @@ "h": 1, "w": 24, "x": 0, - "y": 29 + "y": 30 }, "id": 86, "panels": [ @@ -10878,7 +11591,7 @@ "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 31 }, "id": 28, "panels": [ @@ -11628,7 +12341,7 @@ "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 32 }, "id": 66, "panels": [ @@ -12297,7 +13010,7 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 33 }, "id": 232, "panels": [ @@ -13524,7 +14237,7 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 34 }, "id": 164, "panels": [ @@ -13706,7 +14419,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 35 }, "id": 166, "panels": [ @@ -13936,7 +14649,7 @@ "h": 1, "w": 24, "x": 0, - "y": 35 + "y": 36 }, "id": 374, "panels": [ @@ -14130,7 +14843,7 @@ "h": 1, "w": 24, "x": 0, - "y": 36 + "y": 37 }, "id": 188, "panels": [ @@ -14569,7 +15282,7 @@ "h": 1, "w": 24, "x": 0, - "y": 37 + "y": 38 }, "id": 214, "panels": [ @@ -15096,7 +15809,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 39 }, "id": 270, "panels": [ @@ -15797,7 +16510,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 40 }, "id": 337, "panels": [ @@ -16367,7 +17080,7 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 41 }, "id": 252, "panels": [ @@ -16796,7 +17509,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 42 }, "id": 309, "panels": [ @@ -17218,7 +17931,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 43 }, "id": 313, "panels": [ diff --git a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts index e3f06b893b0e..770ffcd09453 100644 --- a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts +++ b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts @@ -38,10 +38,10 @@ export type ReqOpts = { export type JsonRpcHttpClientMetrics = { requestTime: IHistogram<"routeId">; requestErrors: IGauge<"routeId">; - requestUsedFallbackUrl: IGauge; - activeRequests: IGauge; + requestUsedFallbackUrl: IGauge<"routeId">; + activeRequests: IGauge<"routeId">; configUrlsCount: IGauge; - retryCount: IGauge; + retryCount: IGauge<"routeId">; }; export interface IJsonRpcHttpClient { @@ -52,7 +52,6 @@ export interface IJsonRpcHttpClient { export class JsonRpcHttpClient implements IJsonRpcHttpClient { private id = 1; - private activeRequests = 0; /** * Optional: If provided, use this jwt secret to HS256 encode and add a jwt token in the * request header which can be authenticated by the RPC server to provide access. @@ -101,7 +100,6 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { this.metrics = opts?.metrics ?? null; this.metrics?.configUrlsCount.set(urls.length); - this.metrics?.activeRequests.addCollect(() => this.metrics?.activeRequests.set(this.activeRequests)); } /** @@ -117,6 +115,7 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { */ async fetchWithRetries(payload: IRpcPayload

, opts?: ReqOpts): Promise { const routeId = opts?.routeId ?? "unknown"; + const res = await retry>( async (attempt) => { /** If this is a retry, increment the retry counter for this method */ @@ -149,11 +148,12 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { } private async fetchJson(json: T, opts?: ReqOpts): Promise { + const routeId = opts?.routeId ?? "unknown"; let lastError: Error | null = null; for (let i = 0; i < this.urls.length; i++) { if (i > 0) { - this.metrics?.requestUsedFallbackUrl.inc(1); + this.metrics?.requestUsedFallbackUrl.inc({routeId}); } try { @@ -196,7 +196,7 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { // Default to "unknown" to prevent mixing metrics with others. const routeId = opts?.routeId ?? "unknown"; const timer = this.metrics?.requestTime.startTimer({routeId}); - this.activeRequests++; + this.metrics?.activeRequests.inc({routeId}, 1); try { const headers: Record = {"Content-Type": "application/json"}; @@ -243,7 +243,7 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { } } finally { timer?.(); - this.activeRequests--; + this.metrics?.activeRequests.dec({routeId}, 1); clearTimeout(timeout); this.opts?.signal?.removeEventListener("abort", onParentSignalAbort); diff --git a/packages/beacon-node/src/metrics/interface.ts b/packages/beacon-node/src/metrics/interface.ts index d8ff22f9885a..6a55f03eefcf 100644 --- a/packages/beacon-node/src/metrics/interface.ts +++ b/packages/beacon-node/src/metrics/interface.ts @@ -1,6 +1,6 @@ import {Gauge, Histogram} from "prom-client"; -export type IGauge = Pick, "inc" | "set"> & { +export type IGauge = Pick, "inc" | "dec" | "set"> & { addCollect: (collectFn: () => void) => void; }; diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 3c710168d178..597a3c989a62 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1123,10 +1123,12 @@ export function createLodestarMetrics( requestUsedFallbackUrl: register.gauge({ name: "lodestar_eth1_http_client_request_used_fallback_url_total", help: "eth1 JsonHttpClient - total count of requests on fallback url(s)", + labelNames: ["routeId"], }), activeRequests: register.gauge({ name: "lodestar_eth1_http_client_active_requests", help: "eth1 JsonHttpClient - current count of active requests", + labelNames: ["routeId"], }), configUrlsCount: register.gauge({ name: "lodestar_eth1_http_client_config_urls_count", @@ -1155,10 +1157,12 @@ export function createLodestarMetrics( requestUsedFallbackUrl: register.gauge({ name: "lodestar_execution_engine_http_client_request_used_fallback_url_total", help: "ExecutionEngineHttp client - total count of requests on fallback url(s)", + labelNames: ["routeId"], }), activeRequests: register.gauge({ name: "lodestar_execution_engine_http_client_active_requests", help: "ExecutionEngineHttp client - current count of active requests", + labelNames: ["routeId"], }), configUrlsCount: register.gauge({ name: "lodestar_execution_engine_http_client_config_urls_count", diff --git a/packages/beacon-node/src/metrics/utils/gauge.ts b/packages/beacon-node/src/metrics/utils/gauge.ts index 3d8d964a8118..9f599f041a4c 100644 --- a/packages/beacon-node/src/metrics/utils/gauge.ts +++ b/packages/beacon-node/src/metrics/utils/gauge.ts @@ -54,6 +54,16 @@ export class GaugeChild implements IGauge { } } + dec(value?: number): void; + dec(labels: Labels, value?: number): void; + dec(arg1?: Labels | number, arg2?: number): void { + if (typeof arg1 === "object") { + this.gauge.dec({...this.labelsParent, ...arg1}, arg2 ?? 1); + } else { + this.gauge.dec(this.labelsParent, arg1 ?? 1); + } + } + set(value: number): void; set(labels: Labels, value: number): void; set(arg1?: Labels | number, arg2?: number): void { diff --git a/packages/validator/src/metrics.ts b/packages/validator/src/metrics.ts index 3c7704f3a701..31c07803691d 100644 --- a/packages/validator/src/metrics.ts +++ b/packages/validator/src/metrics.ts @@ -13,6 +13,10 @@ interface Gauge { inc(labels: Labels, value?: number): void; inc(arg1?: Labels | number, arg2?: number): void; + dec(value?: number): void; + dec(labels: Labels, value?: number): void; + dec(arg1?: Labels | number, arg2?: number): void; + set(value: number): void; set(labels: Labels, value: number): void; set(arg1?: Labels | number, arg2?: number): void;