monitoring dashboard

cmd/algorand-indexer/daemon.go

@@ -213,8 +213,14 @@ func handleBlock(block *rpcs.EncodedBlockCert, imp importer.Importer) error {
 	// Ignore round 0 (which is empty).
 	if block.Block.Round() > 0 {
 		metrics.BlockImportTimeSeconds.Observe(dt.Seconds())
-		metrics.ImportedTxnsPerBlock.Observe(float64(len(block.Block.Payset)))
 		metrics.ImportedRoundGauge.Set(float64(block.Block.Round()))
+		txnCountByType := make(map[string]int)
+		for _, txn := range block.Block.Payset {
+			txnCountByType[string(txn.Txn.Type)]++
+		}
+		for k, v := range txnCountByType {
+			metrics.ImportedTxnsPerBlock.WithLabelValues(k).Set(float64(v))
+		}
 	}
 
 	logger.Infof("round r=%d (%d txn) imported in %s", block.Block.Round(), len(block.Block.Payset), dt.String())

fetcher/fetcher.go

@@ -13,6 +13,7 @@ import (
 	"github.com/algorand/go-algorand-sdk/client/v2/algod"
 	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/rpcs"
+	"github.com/algorand/indexer/util/metrics"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -111,7 +112,13 @@ func (bot *fetcherImpl) catchupLoop(ctx context.Context) error {
 	var blockbytes []byte
 	aclient := bot.Algod()
 	for {
+		start := time.Now()
+
 		blockbytes, err = aclient.BlockRaw(bot.nextRound).Do(ctx)
+
+		dt := time.Since(start)
+		metrics.AlgodRawBlockTimeSeconds.Observe(dt.Seconds())
+
 		if err != nil {
 			// If context has expired.
 			if ctx.Err() != nil {
@@ -150,7 +157,13 @@ func (bot *fetcherImpl) followLoop(ctx context.Context) error {
 					"r=%d error getting status %d", retries, bot.nextRound)
 				continue
 			}
+			start := time.Now()
+
 			blockbytes, err = aclient.BlockRaw(bot.nextRound).Do(ctx)
+
+			dt := time.Since(start)
+			metrics.AlgodRawBlockTimeSeconds.Observe(dt.Seconds())
+
 			if err == nil {
 				break
 			} else if ctx.Err() != nil { // if context has expired

monitoring/.env

@@ -0,0 +1,2 @@
+DB_CONNECTION = "host=indexer-db port=5432 user=algorand password=algorand dbname=indexer_db sslmode=disable"
+ALGOD_NET = https://node-archival-testnet.internal.aws.algodev.network:443

monitoring/Dockerfile-indexer

@@ -0,0 +1,31 @@
+FROM alpine:latest
+
+RUN apk add --no-cache git make musl-dev go bash libtool autoconf automake build-base libsodium-dev boost-dev
+
+# Configure Go
+ENV GOROOT /usr/lib/go
+ENV GOPATH /go
+ENV PATH /go/bin:$PATH
+
+RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin
+
+# Build and run indexer
+RUN mkdir /work
+WORKDIR /work
+ADD . ./
+RUN make
+WORKDIR /work/cmd/algorand-indexer
+ENV CGO_ENABLED="1"
+RUN go build
+
+# The sleep is to wait until postgres starts
+CMD ["/bin/sh", "-c", "\
+  echo $ALGOD_NET && \
+  echo $CONNECTION_STRING &&\
+  sleep 5 && \
+  ./algorand-indexer daemon \
+    --server \":${PORT}\" \
+    -P \"${CONNECTION_STRING}\" \
+    --metrics-mode VERBOSE \
+    --algod-net \"${ALGOD_NET}\" \
+    --algod-token YddOUGbAjHLr1uPZtZwHOvMDmXvR1Zvw1f3Roj2PT1ufenXbNyIxIz0IeznrLbDsF"]

monitoring/README.md

@@ -0,0 +1,57 @@
+## Indexer Monitoring Dashboard
+A monitoring dashboard displaying indexer performance metrics. 
+
+### To start a monitoring dashboard
+Configurable env vars, `DB_CONNECTION` and `ALGOD_NET`. The default values are set in `.env`. 
+
+if indexer is not already running, start all services in `docker-compose.yml`
+
+`docker-compose up -d`
+
+if indexer is already running, start monitoring services only.
+
+`docker-compose up -d prometheus grafana`
+
+prometheus target should be updated to listen to a correct indexer address (host:port), 
+
+```json
+static_configs:
+- targets: ['indexer:8888'] 
+```
+
+By default, 
+ - indexer is running on http://localhost:8888 
+ - grafana is running on http://localhost:3000; default login (admin/admin)
+
+### View metrics on grafana
+
+- In grafana configurations, add a PostgreSQL datasource. See example below. 
+- Go to Import and upload dashboard.json
+- Run `create extension pg_stat_statements;` sql on db to enable query stats from Postgres
+
+![](examples/postgresql_conn.png)
+![](examples/widgets.png)
+
+
+**Default widgets**:
+
+Importing
+- Block import time
+- Transactions per block by type
+- TPS
+- Round #
+
+Query
+
+- Request duration
+- Request duraton by url
+- Postgres Eval Time
+
+System
+
+- average CPU usage
+- average Heap Alloc
+- Postgres table info
+- Postgres queries info
+
+