Add ip bloom metrics (#2614)

network/ip_tracker.go

@@ -7,12 +7,15 @@ import (
 	"crypto/rand"
 	"sync"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"go.uber.org/zap"
 
 	"golang.org/x/exp/maps"
 
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/snow/validators"
+	"github.com/ava-labs/avalanchego/utils"
 	"github.com/ava-labs/avalanchego/utils/bloom"
 	"github.com/ava-labs/avalanchego/utils/crypto/bls"
 	"github.com/ava-labs/avalanchego/utils/ips"
@@ -34,19 +37,77 @@ const (
 
 var _ validators.SetCallbackListener = (*ipTracker)(nil)
 
-func newIPTracker(log logging.Logger) (*ipTracker, error) {
+func newIPTracker(
+	log logging.Logger,
+	namespace string,
+	registerer prometheus.Registerer,
+) (*ipTracker, error) {
 	tracker := &ipTracker{
-		log:                    log,
+		log: log,
+		numValidatorIPs: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "validator_ips",
+			Help:      "Number of known validator IPs",
+		}),
+		numGossipable: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "gossipable_ips",
+			Help:      "Number of IPs this node is willing to gossip",
+		}),
+		bloomCount: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "ip_bloom_count",
+			Help:      "Number of IP entries added to the bloom",
+		}),
+		bloomNumHashes: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "ip_bloom_hashes",
+			Help:      "Number of hashes in the IP bloom",
+		}),
+		bloomNumEntries: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "ip_bloom_entries",
+			Help:      "Number of entry slots in the IP bloom",
+		}),
+		bloomMaxCount: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      "ip_bloom_max_count",
+			Help:      "Maximum number of IP entries that can be added to the bloom before resetting",
+		}),
+		bloomResetCount: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      "ip_bloom_reset_count",
+			Help:      "Number times the IP bloom has been reset",
+		}),
 		connected:              make(map[ids.NodeID]*ips.ClaimedIPPort),
 		mostRecentValidatorIPs: make(map[ids.NodeID]*ips.ClaimedIPPort),
 		gossipableIndicies:     make(map[ids.NodeID]int),
 		bloomAdditions:         make(map[ids.NodeID]int),
 	}
+	err := utils.Err(
+		registerer.Register(tracker.numValidatorIPs),
+		registerer.Register(tracker.numGossipable),
+		registerer.Register(tracker.bloomCount),
+		registerer.Register(tracker.bloomNumHashes),
+		registerer.Register(tracker.bloomNumEntries),
+		registerer.Register(tracker.bloomMaxCount),
+		registerer.Register(tracker.bloomResetCount),
+	)
+	if err != nil {
+		return nil, err
+	}
 	return tracker, tracker.resetBloom()
 }
 
 type ipTracker struct {
-	log logging.Logger
+	log             logging.Logger
+	numValidatorIPs prometheus.Gauge
+	numGossipable   prometheus.Gauge
+	bloomCount      prometheus.Gauge
+	bloomNumHashes  prometheus.Gauge
+	bloomNumEntries prometheus.Gauge
+	bloomMaxCount   prometheus.Gauge
+	bloomResetCount prometheus.Counter
 
 	lock sync.RWMutex
 	// Manually tracked nodes are always treated like validators
@@ -220,12 +281,16 @@ func (i *ipTracker) OnValidatorRemoved(nodeID ids.NodeID, _ uint64) {
 	}
 
 	delete(i.mostRecentValidatorIPs, nodeID)
+	i.numValidatorIPs.Set(float64(len(i.mostRecentValidatorIPs)))
+
 	i.validators.Remove(nodeID)
 	i.removeGossipableIP(nodeID)
 }
 
 func (i *ipTracker) updateMostRecentValidatorIP(ip *ips.ClaimedIPPort) {
 	i.mostRecentValidatorIPs[ip.NodeID] = ip
+	i.numValidatorIPs.Set(float64(len(i.mostRecentValidatorIPs)))
+
 	oldCount := i.bloomAdditions[ip.NodeID]
 	if oldCount >= maxIPEntriesPerValidator {
 		return
@@ -250,11 +315,13 @@ func (i *ipTracker) updateMostRecentValidatorIP(ip *ips.ClaimedIPPort) {
 
 	i.bloomAdditions[ip.NodeID] = oldCount + 1
 	bloom.Add(i.bloom, ip.GossipID[:], i.bloomSalt)
+	i.bloomCount.Inc()
 }
 
 func (i *ipTracker) addGossipableIP(ip *ips.ClaimedIPPort) {
 	i.gossipableIndicies[ip.NodeID] = len(i.gossipableIPs)
 	i.gossipableIPs = append(i.gossipableIPs, ip)
+	i.numGossipable.Inc()
 }
 
 func (i *ipTracker) removeGossipableIP(nodeID ids.NodeID) {
@@ -273,6 +340,7 @@ func (i *ipTracker) removeGossipableIP(nodeID ids.NodeID) {
 	delete(i.gossipableIndicies, nodeID)
 	i.gossipableIPs[newNumGossipable] = nil
 	i.gossipableIPs = i.gossipableIPs[:newNumGossipable]
+	i.numGossipable.Dec()
 }
 
 // GetGossipableIPs returns the latest IPs of connected validators. The returned
@@ -360,5 +428,10 @@ func (i *ipTracker) resetBloom() error {
 		bloom.Add(newFilter, ip.GossipID[:], newSalt)
 		i.bloomAdditions[nodeID] = 1
 	}
+	i.bloomCount.Set(float64(len(i.mostRecentValidatorIPs)))
+	i.bloomNumHashes.Set(float64(numHashes))
+	i.bloomNumEntries.Set(float64(numEntries))
+	i.bloomMaxCount.Set(float64(i.maxBloomCount))
+	i.bloomResetCount.Inc()
 	return nil
 }

network/ip_tracker_test.go

@@ -6,6 +6,9 @@ package network
 import (
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/ava-labs/avalanchego/ids"
@@ -15,7 +18,7 @@ import (
 )
 
 func newTestIPTracker(t *testing.T) *ipTracker {
-	tracker, err := newIPTracker(logging.NoLog{})
+	tracker, err := newIPTracker(logging.NoLog{}, "", prometheus.NewRegistry())
 	require.NoError(t, err)
 	return tracker
 }
@@ -41,6 +44,14 @@ func requireEqual(t *testing.T, expected, actual *ipTracker) {
 	require.Equal(expected.maxBloomCount, actual.maxBloomCount)
 }
 
+func requireMetricsConsistent(t *testing.T, tracker *ipTracker) {
+	require := require.New(t)
+	require.Equal(float64(len(tracker.mostRecentValidatorIPs)), testutil.ToFloat64(tracker.numValidatorIPs))
+	require.Equal(float64(len(tracker.gossipableIPs)), testutil.ToFloat64(tracker.numGossipable))
+	require.Equal(float64(tracker.bloom.Count()), testutil.ToFloat64(tracker.bloomCount))
+	require.Equal(float64(tracker.maxBloomCount), testutil.ToFloat64(tracker.bloomMaxCount))
+}
+
 func TestIPTracker_ManuallyTrack(t *testing.T) {
 	tests := []struct {
 		name          string
@@ -118,6 +129,7 @@ func TestIPTracker_ManuallyTrack(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			test.initialState.ManuallyTrack(test.nodeID)
 			requireEqual(t, test.expectedState, test.initialState)
+			requireMetricsConsistent(t, test.initialState)
 		})
 	}
 }
@@ -235,6 +247,7 @@ func TestIPTracker_AddIP(t *testing.T) {
 			updated := test.initialState.AddIP(test.ip)
 			require.Equal(t, test.expectedUpdated, updated)
 			requireEqual(t, test.expectedState, test.initialState)
+			requireMetricsConsistent(t, test.initialState)
 		})
 	}
 }
@@ -344,6 +357,7 @@ func TestIPTracker_Connected(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			test.initialState.Connected(test.ip)
 			requireEqual(t, test.expectedState, test.initialState)
+			requireMetricsConsistent(t, test.initialState)
 		})
 	}
 }
@@ -416,6 +430,7 @@ func TestIPTracker_Disconnected(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			test.initialState.Disconnected(test.nodeID)
 			requireEqual(t, test.expectedState, test.initialState)
+			requireMetricsConsistent(t, test.initialState)
 		})
 	}
 }
@@ -477,6 +492,7 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			test.initialState.OnValidatorAdded(test.nodeID, nil, ids.Empty, 0)
 			requireEqual(t, test.expectedState, test.initialState)
+			requireMetricsConsistent(t, test.initialState)
 		})
 	}
 }
@@ -577,6 +593,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			test.initialState.OnValidatorRemoved(test.nodeID, 0)
 			requireEqual(t, test.expectedState, test.initialState)
+			requireMetricsConsistent(t, test.initialState)
 		})
 	}
 }
@@ -637,9 +654,11 @@ func TestIPTracker_BloomGrowsWithValidatorSet(t *testing.T) {
 	for i := 0; i < 2048; i++ {
 		tracker.onValidatorAdded(ids.GenerateTestNodeID())
 	}
+	requireMetricsConsistent(t, tracker)
 
 	require.NoError(tracker.ResetBloom())
 	require.Greater(tracker.maxBloomCount, initialMaxBloomCount)
+	requireMetricsConsistent(t, tracker)
 }
 
 func TestIPTracker_BloomResetsDynamically(t *testing.T) {
@@ -652,6 +671,7 @@ func TestIPTracker_BloomResetsDynamically(t *testing.T) {
 	tracker.maxBloomCount = 1
 	tracker.Connected(otherIP)
 	tracker.onValidatorAdded(otherIP.NodeID)
+	requireMetricsConsistent(t, tracker)
 
 	bloomBytes, salt := tracker.Bloom()
 	readFilter, err := bloom.Parse(bloomBytes)
@@ -673,6 +693,7 @@ func TestIPTracker_PreventBloomFilterAddition(t *testing.T) {
 	require.True(tracker.AddIP(newerIP))
 	require.True(tracker.AddIP(newestIP))
 	require.Equal(maxIPEntriesPerValidator, tracker.bloomAdditions[ip.NodeID])
+	requireMetricsConsistent(t, tracker)
 }
 
 func TestIPTracker_ShouldVerifyIP(t *testing.T) {

network/network.go

@@ -237,7 +237,7 @@ func NewNetwork(
 		return nil, fmt.Errorf("initializing network metrics failed with: %w", err)
 	}
 
-	ipTracker, err := newIPTracker(log)
+	ipTracker, err := newIPTracker(log, config.Namespace, metricsRegisterer)
 	if err != nil {
 		return nil, fmt.Errorf("initializing ip tracker failed with: %w", err)
 	}