diff --git a/cmd/lotus-stats/README.md b/cmd/lotus-stats/README.md index 04220aa3bdc..4311e1aa7bc 100644 --- a/cmd/lotus-stats/README.md +++ b/cmd/lotus-stats/README.md @@ -7,22 +7,24 @@ Influx configuration can be configured through env variables. ``` -INFLUX_ADDR="http://localhost:8086" -INFLUX_USER="" -INFLUX_PASS="" +LOTUS_STATS_INFLUX_ADDR="http://localhost:8086" +LOTUS_STATS_INFLUX_USER="" +LOTUS_STATS_INFLUX_PASS="" ``` ## Usage -lotus-stats will be default look in `~/.lotus` to connect to a running daemon and resume collecting stats from last record block height. +lotus-stats will look in `~/.lotus` to connect to a running daemon and resume collecting stats from last record block height. For other usage see `./lotus-stats --help` ``` go build -o lotus-stats *.go -. env.stats && ./lotus-stats +. env.stats && ./lotus-stats run ``` +For large networks there is an additional query in the `Top Miner Power` table, which can be toggled on to only show miners larger +than 1 PiB. This is a good option to enable to reduce the number of miners listed when viewing mainnet stats. ## Development @@ -37,3 +39,8 @@ docker-compose up -d ``` The default username and password for grafana are both `admin`. + +## Updating the dashboard + +After importing the provided dashboard in `chain.dashboard.json`, you may make changes to the dashboard. To export +the dashboard to be commited back to the project, make sure the option "sharing externally" is toggled on. diff --git a/cmd/lotus-stats/chain.dashboard.json b/cmd/lotus-stats/chain.dashboard.json index 8083c96b183..f8f54597814 100644 --- a/cmd/lotus-stats/chain.dashboard.json +++ b/cmd/lotus-stats/chain.dashboard.json @@ -30,6 +30,12 @@ "id": "table-old", "name": "Table (old)", "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" } ], "annotations": { @@ -47,11 +53,217 @@ }, "editable": true, "gnetId": null, - "graphTooltip": 0, + "graphTooltip": 1, "id": null, - "iteration": 1604018016916, + "iteration": 1630020824868, "links": [], "panels": [ + { + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 56, + "options": { + "content": "
\n \n
\n", + "mode": "html" + }, + "pluginVersion": "7.3.0", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 16, + "x": 5, + "y": 0 + }, + "id": 58, + "options": { + "content": "
\n The stats dashboard is undergoing some maintance. Please check back later.\n
", + "mode": "html" + }, + "pluginVersion": "7.3.0", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": "$network", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 54, + "options": { + "content": "
\n
\n \n
\n
", + "mode": "html" + }, + "pluginVersion": "7.3.0", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + }, { "aliasColors": {}, "bars": true, @@ -72,7 +284,7 @@ "h": 9, "w": 24, "x": 0, - "y": 0 + "y": 3 }, "hiddenSeries": false, "hideTimeOverride": false, @@ -186,7 +398,7 @@ ], "hide": false, "orderByTime": "ASC", - "policy": "defult", + "policy": "default", "query": "SELECT TRIPLE_EXPONENTIAL_MOVING_AVERAGE(sum(\"value\"), 40) FROM \"chain.election\" WHERE $timeFilter -$blockInterval*40 AND time < now() - $blockInterval*3 GROUP BY time($blockInterval) fill(0)", "rawQuery": true, "refId": "B", @@ -271,7 +483,7 @@ "h": 4, "w": 8, "x": 0, - "y": 9 + "y": 12 }, "hiddenSeries": false, "id": 22, @@ -396,7 +608,7 @@ "h": 4, "w": 4, "x": 8, - "y": 9 + "y": 12 }, "id": 12, "interval": null, @@ -493,7 +705,7 @@ }, "overrides": [] }, - "format": "bytes", + "format": "gbytes", "gauge": { "maxValue": 100, "minValue": 0, @@ -505,7 +717,7 @@ "h": 4, "w": 4, "x": 12, - "y": 9 + "y": 12 }, "id": 42, "interval": "", @@ -626,7 +838,7 @@ "h": 4, "w": 8, "x": 16, - "y": 9 + "y": 12 }, "id": 6, "interval": null, @@ -741,7 +953,7 @@ "h": 3, "w": 4, "x": 0, - "y": 13 + "y": 16 }, "id": 4, "interval": null, @@ -844,7 +1056,7 @@ "h": 3, "w": 4, "x": 4, - "y": 13 + "y": 16 }, "id": 14, "interval": null, @@ -951,7 +1163,7 @@ "h": 3, "w": 4, "x": 8, - "y": 13 + "y": 16 }, "id": 32, "interval": null, @@ -1071,7 +1283,7 @@ "h": 3, "w": 4, "x": 12, - "y": 13 + "y": 16 }, "id": 20, "interval": null, @@ -1191,7 +1403,7 @@ "h": 3, "w": 4, "x": 16, - "y": 13 + "y": 16 }, "id": 8, "interval": null, @@ -1311,7 +1523,7 @@ "h": 3, "w": 4, "x": 20, - "y": 13 + "y": 16 }, "id": 10, "interval": null, @@ -1435,7 +1647,7 @@ "h": 3, "w": 4, "x": 0, - "y": 16 + "y": 19 }, "id": 16, "interval": "", @@ -1536,7 +1748,7 @@ "h": 3, "w": 16, "x": 4, - "y": 16 + "y": 19 }, "hiddenSeries": false, "id": 2, @@ -1706,7 +1918,7 @@ "h": 3, "w": 4, "x": 20, - "y": 16 + "y": 19 }, "id": 30, "interval": null, @@ -1795,7 +2007,7 @@ "h": 21, "w": 4, "x": 0, - "y": 19 + "y": 22 }, "id": 28, "pageSize": null, @@ -1827,7 +2039,7 @@ "pattern": "power", "thresholds": [], "type": "number", - "unit": "bytes" + "unit": "gbytes" }, { "alias": "", @@ -1894,7 +2106,7 @@ "h": 8, "w": 12, "x": 4, - "y": 19 + "y": 22 }, "hiddenSeries": false, "id": 40, @@ -1979,6 +2191,44 @@ ], "slimit": "", "tags": [] + }, + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": true, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM \"chain.miner_power\" WHERE $timeFilter AND value > 1125899906842624 GROUP BY time($__interval), \"miner\" fill(null)", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] } ], "thresholds": [], @@ -2002,7 +2252,7 @@ "yaxes": [ { "decimals": 2, - "format": "bytes", + "format": "gbytes", "label": "Power", "logBase": 1, "max": "100", @@ -2037,7 +2287,7 @@ "h": 21, "w": 8, "x": 16, - "y": 19 + "y": 22 }, "id": 18, "pageSize": null, @@ -2142,7 +2392,7 @@ "h": 7, "w": 12, "x": 4, - "y": 27 + "y": 30 }, "hiddenSeries": false, "id": 50, @@ -2182,7 +2432,7 @@ }, { "params": [ - "null" + "previous" ], "type": "fill" } @@ -2221,7 +2471,7 @@ }, { "params": [ - "null" + "previous" ], "type": "fill" } @@ -2260,7 +2510,7 @@ }, { "params": [ - "null" + "previous" ], "type": "fill" } @@ -2361,7 +2611,7 @@ "h": 6, "w": 12, "x": 4, - "y": 34 + "y": 37 }, "hiddenSeries": false, "id": 44, @@ -2491,7 +2741,7 @@ "h": 9, "w": 12, "x": 0, - "y": 40 + "y": 43 }, "hiddenSeries": false, "id": 34, @@ -2632,7 +2882,7 @@ "h": 9, "w": 12, "x": 12, - "y": 40 + "y": 43 }, "hiddenSeries": false, "id": 36, @@ -2785,7 +3035,7 @@ "h": 8, "w": 12, "x": 0, - "y": 49 + "y": 52 }, "hiddenSeries": false, "id": 48, @@ -2915,7 +3165,7 @@ "h": 8, "w": 12, "x": 12, - "y": 49 + "y": 52 }, "hiddenSeries": false, "id": 46, @@ -3046,7 +3296,7 @@ "h": 8, "w": 12, "x": 0, - "y": 57 + "y": 60 }, "hiddenSeries": false, "id": 51, @@ -3218,7 +3468,7 @@ "h": 8, "w": 12, "x": 12, - "y": 57 + "y": 60 }, "hiddenSeries": false, "id": 52, @@ -3417,8 +3667,8 @@ { "current": { "selected": false, - "text": "filecoin-ntwk-testnet", - "value": "filecoin-ntwk-testnet" + "text": "ntwk-localstats", + "value": "ntwk-localstats" }, "error": null, "hide": 0, @@ -3430,7 +3680,7 @@ "query": "influxdb", "queryValue": "", "refresh": 1, - "regex": "/^filecoin-ntwk-/", + "regex": "/^ntwk-/", "skipUrlSync": false, "type": "datasource" }, @@ -3466,23 +3716,13 @@ "to": "now" }, "timepicker": { + "hidden": true, "refresh_intervals": [ - "5s", - "10s", - "25s", - "30s", - "45s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" + "30s" ] }, "timezone": "", "title": "Filecoin Chain Stats", "uid": "z6FtI92Zz", - "version": 4 + "version": 2 } diff --git a/cmd/lotus-stats/env.stats b/cmd/lotus-stats/env.stats index ad5ec1619ee..bc4e44ca908 100644 --- a/cmd/lotus-stats/env.stats +++ b/cmd/lotus-stats/env.stats @@ -1,3 +1,3 @@ -export INFLUX_ADDR="http://localhost:18086" -export INFLUX_USER="" -export INFLUX_PASS="" +export LOTUS_STATS_INFLUX_ADDR="http://localhost:18086" +export LOTUS_STATS_INFLUX_USER="" +export LOTUS_STATS_INFLUX_PASS="" diff --git a/cmd/lotus-stats/setup.bash b/cmd/lotus-stats/setup.bash index 6510c2fc6d7..5aacad4723e 100755 --- a/cmd/lotus-stats/setup.bash +++ b/cmd/lotus-stats/setup.bash @@ -1,10 +1,10 @@ #!/usr/bin/env bash -GRAFANA_HOST="http://localhost:13000" +GRAFANA_HOST="localhost:13000" -curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/datasources -H 'Content-Type: text/json' --data-binary @- > /dev/null << EOF +curl -XPOST http://admin:admin@$GRAFANA_HOST/api/datasources -H 'Content-Type: text/json' --data-binary @- > /dev/null << EOF { - "name":"filecoin-ntwk-localstats", + "name":"ntwk-localstats", "type":"influxdb", "database":"lotus", "url": "http://influxdb:8086", @@ -13,7 +13,7 @@ curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/datasources -H 'Content-Type } EOF -curl -s -XPOST http://admin:admin@$GRAFANA_HOST/api/dashboards/import -H 'Content-Type: text/json' --data-binary @- << EOF | jq -r "\"http://$GRAFANA_HOST\" + .importedUrl" +curl -XPOST http://admin:admin@$GRAFANA_HOST/api/dashboards/import -H 'Content-Type: text/json' --data-binary @- << EOF | jq -r "\"http://$GRAFANA_HOST\" + .importedUrl" { "dashboard": $(cat ./chain.dashboard.json), "overwrite": true, diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index 7764c4bcabd..ca3f26336fa 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -267,7 +267,11 @@ func RecordTipsetStatePoints(ctx context.Context, api v0api.FullNode, pl *PointL return err } - p = NewPoint("chain.power", totalPower.TotalPower.QualityAdjPower.Int64()) + // We divide the power into gibibytes because 2^63 bytes is 8 exbibytes which is smaller than the Filecoin Mainnet. + // Dividing by a gibibyte gives us more room to work with. This will allow the dashboard to report network and miner + // sizes up to 8192 yobibytes. + gibi := types.NewInt(1024 * 1024 * 1024) + p = NewPoint("chain.power", types.BigDiv(totalPower.TotalPower.QualityAdjPower, gibi).Int64()) pl.AddPoint(p) powerActor, err := api.StateGetActor(ctx, power.Address, tipset.Key()) @@ -281,11 +285,12 @@ func RecordTipsetStatePoints(ctx context.Context, api v0api.FullNode, pl *PointL } return powerActorState.ForEachClaim(func(addr address.Address, claim power.Claim) error { - if claim.QualityAdjPower.Int64() == 0 { + // BigCmp returns 0 if values are equal + if types.BigCmp(claim.QualityAdjPower, types.NewInt(0)) == 0 { return nil } - p = NewPoint("chain.miner_power", claim.QualityAdjPower.Int64()) + p = NewPoint("chain.miner_power", types.BigDiv(claim.QualityAdjPower, gibi).Int64()) p.AddTag("miner", addr.String()) pl.AddPoint(p)