Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ip bloom metrics #2614

Merged
merged 2 commits into from
Jan 16, 2024
Merged

Add ip bloom metrics #2614

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e54601e
Add ip bloom metrics
StephenButtolph Jan 16, 2024
9e4f153
Add some tests for metrics consistency
StephenButtolph Jan 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 76 additions & 3 deletions network/ip_tracker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,15 @@ import (
"crypto/rand"
"sync"

"github.com/prometheus/client_golang/prometheus"

"go.uber.org/zap"

"golang.org/x/exp/maps"

"github.com/ava-labs/avalanchego/ids"
"github.com/ava-labs/avalanchego/snow/validators"
"github.com/ava-labs/avalanchego/utils"
"github.com/ava-labs/avalanchego/utils/bloom"
"github.com/ava-labs/avalanchego/utils/crypto/bls"
"github.com/ava-labs/avalanchego/utils/ips"
Expand All @@ -34,19 +37,77 @@ const (

var _ validators.SetCallbackListener = (*ipTracker)(nil)

func newIPTracker(log logging.Logger) (*ipTracker, error) {
func newIPTracker(
log logging.Logger,
namespace string,
registerer prometheus.Registerer,
) (*ipTracker, error) {
tracker := &ipTracker{
log: log,
log: log,
numValidatorIPs: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Name: "validator_ips",
Help: "Number of known validator IPs",
}),
numGossipable: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Name: "gossipable_ips",
Help: "Number of IPs this node is willing to gossip",
}),
bloomCount: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Name: "ip_bloom_count",
Help: "Number of IP entries added to the bloom",
}),
bloomNumHashes: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Name: "ip_bloom_hashes",
Help: "Number of hashes in the IP bloom",
}),
bloomNumEntries: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Name: "ip_bloom_entries",
Help: "Number of entry slots in the IP bloom",
}),
bloomMaxCount: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Name: "ip_bloom_max_count",
Help: "Maximum number of IP entries that can be added to the bloom before resetting",
}),
bloomResetCount: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Name: "ip_bloom_reset_count",
Help: "Number times the IP bloom has been reset",
}),
connected: make(map[ids.NodeID]*ips.ClaimedIPPort),
mostRecentValidatorIPs: make(map[ids.NodeID]*ips.ClaimedIPPort),
gossipableIndicies: make(map[ids.NodeID]int),
bloomAdditions: make(map[ids.NodeID]int),
}
err := utils.Err(
registerer.Register(tracker.numValidatorIPs),
registerer.Register(tracker.numGossipable),
registerer.Register(tracker.bloomCount),
registerer.Register(tracker.bloomNumHashes),
registerer.Register(tracker.bloomNumEntries),
registerer.Register(tracker.bloomMaxCount),
registerer.Register(tracker.bloomResetCount),
)
if err != nil {
return nil, err
}
return tracker, tracker.resetBloom()
}

type ipTracker struct {
log logging.Logger
log logging.Logger
numValidatorIPs prometheus.Gauge
numGossipable prometheus.Gauge
bloomCount prometheus.Gauge
bloomNumHashes prometheus.Gauge
bloomNumEntries prometheus.Gauge
bloomMaxCount prometheus.Gauge
bloomResetCount prometheus.Counter

lock sync.RWMutex
// Manually tracked nodes are always treated like validators
Expand Down Expand Up @@ -220,12 +281,16 @@ func (i *ipTracker) OnValidatorRemoved(nodeID ids.NodeID, _ uint64) {
}

delete(i.mostRecentValidatorIPs, nodeID)
i.numValidatorIPs.Set(float64(len(i.mostRecentValidatorIPs)))

i.validators.Remove(nodeID)
i.removeGossipableIP(nodeID)
}

func (i *ipTracker) updateMostRecentValidatorIP(ip *ips.ClaimedIPPort) {
i.mostRecentValidatorIPs[ip.NodeID] = ip
i.numValidatorIPs.Set(float64(len(i.mostRecentValidatorIPs)))

oldCount := i.bloomAdditions[ip.NodeID]
if oldCount >= maxIPEntriesPerValidator {
return
Expand All @@ -250,11 +315,13 @@ func (i *ipTracker) updateMostRecentValidatorIP(ip *ips.ClaimedIPPort) {

i.bloomAdditions[ip.NodeID] = oldCount + 1
bloom.Add(i.bloom, ip.GossipID[:], i.bloomSalt)
i.bloomCount.Inc()
}

func (i *ipTracker) addGossipableIP(ip *ips.ClaimedIPPort) {
i.gossipableIndicies[ip.NodeID] = len(i.gossipableIPs)
i.gossipableIPs = append(i.gossipableIPs, ip)
i.numGossipable.Inc()
}

func (i *ipTracker) removeGossipableIP(nodeID ids.NodeID) {
Expand All @@ -273,6 +340,7 @@ func (i *ipTracker) removeGossipableIP(nodeID ids.NodeID) {
delete(i.gossipableIndicies, nodeID)
i.gossipableIPs[newNumGossipable] = nil
i.gossipableIPs = i.gossipableIPs[:newNumGossipable]
i.numGossipable.Dec()
}

// GetGossipableIPs returns the latest IPs of connected validators. The returned
Expand Down Expand Up @@ -360,5 +428,10 @@ func (i *ipTracker) resetBloom() error {
bloom.Add(newFilter, ip.GossipID[:], newSalt)
i.bloomAdditions[nodeID] = 1
}
i.bloomCount.Set(float64(len(i.mostRecentValidatorIPs)))
i.bloomNumHashes.Set(float64(numHashes))
i.bloomNumEntries.Set(float64(numEntries))
i.bloomMaxCount.Set(float64(i.maxBloomCount))
i.bloomResetCount.Inc()
return nil
}
23 changes: 22 additions & 1 deletion network/ip_tracker_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ package network
import (
"testing"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"

"github.com/stretchr/testify/require"

"github.com/ava-labs/avalanchego/ids"
Expand All @@ -15,7 +18,7 @@ import (
)

func newTestIPTracker(t *testing.T) *ipTracker {
tracker, err := newIPTracker(logging.NoLog{})
tracker, err := newIPTracker(logging.NoLog{}, "", prometheus.NewRegistry())
require.NoError(t, err)
return tracker
}
Expand All @@ -41,6 +44,14 @@ func requireEqual(t *testing.T, expected, actual *ipTracker) {
require.Equal(expected.maxBloomCount, actual.maxBloomCount)
}

func requireMetricsConsistent(t *testing.T, tracker *ipTracker) {
require := require.New(t)
require.Equal(float64(len(tracker.mostRecentValidatorIPs)), testutil.ToFloat64(tracker.numValidatorIPs))
require.Equal(float64(len(tracker.gossipableIPs)), testutil.ToFloat64(tracker.numGossipable))
require.Equal(float64(tracker.bloom.Count()), testutil.ToFloat64(tracker.bloomCount))
require.Equal(float64(tracker.maxBloomCount), testutil.ToFloat64(tracker.bloomMaxCount))
}

func TestIPTracker_ManuallyTrack(t *testing.T) {
tests := []struct {
name string
Expand Down Expand Up @@ -118,6 +129,7 @@ func TestIPTracker_ManuallyTrack(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
test.initialState.ManuallyTrack(test.nodeID)
requireEqual(t, test.expectedState, test.initialState)
requireMetricsConsistent(t, test.initialState)
})
}
}
Expand Down Expand Up @@ -235,6 +247,7 @@ func TestIPTracker_AddIP(t *testing.T) {
updated := test.initialState.AddIP(test.ip)
require.Equal(t, test.expectedUpdated, updated)
requireEqual(t, test.expectedState, test.initialState)
requireMetricsConsistent(t, test.initialState)
})
}
}
Expand Down Expand Up @@ -344,6 +357,7 @@ func TestIPTracker_Connected(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
test.initialState.Connected(test.ip)
requireEqual(t, test.expectedState, test.initialState)
requireMetricsConsistent(t, test.initialState)
})
}
}
Expand Down Expand Up @@ -416,6 +430,7 @@ func TestIPTracker_Disconnected(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
test.initialState.Disconnected(test.nodeID)
requireEqual(t, test.expectedState, test.initialState)
requireMetricsConsistent(t, test.initialState)
})
}
}
Expand Down Expand Up @@ -477,6 +492,7 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
test.initialState.OnValidatorAdded(test.nodeID, nil, ids.Empty, 0)
requireEqual(t, test.expectedState, test.initialState)
requireMetricsConsistent(t, test.initialState)
})
}
}
Expand Down Expand Up @@ -577,6 +593,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
test.initialState.OnValidatorRemoved(test.nodeID, 0)
requireEqual(t, test.expectedState, test.initialState)
requireMetricsConsistent(t, test.initialState)
})
}
}
Expand Down Expand Up @@ -637,9 +654,11 @@ func TestIPTracker_BloomGrowsWithValidatorSet(t *testing.T) {
for i := 0; i < 2048; i++ {
tracker.onValidatorAdded(ids.GenerateTestNodeID())
}
requireMetricsConsistent(t, tracker)

require.NoError(tracker.ResetBloom())
require.Greater(tracker.maxBloomCount, initialMaxBloomCount)
requireMetricsConsistent(t, tracker)
}

func TestIPTracker_BloomResetsDynamically(t *testing.T) {
Expand All @@ -652,6 +671,7 @@ func TestIPTracker_BloomResetsDynamically(t *testing.T) {
tracker.maxBloomCount = 1
tracker.Connected(otherIP)
tracker.onValidatorAdded(otherIP.NodeID)
requireMetricsConsistent(t, tracker)

bloomBytes, salt := tracker.Bloom()
readFilter, err := bloom.Parse(bloomBytes)
Expand All @@ -673,6 +693,7 @@ func TestIPTracker_PreventBloomFilterAddition(t *testing.T) {
require.True(tracker.AddIP(newerIP))
require.True(tracker.AddIP(newestIP))
require.Equal(maxIPEntriesPerValidator, tracker.bloomAdditions[ip.NodeID])
requireMetricsConsistent(t, tracker)
}

func TestIPTracker_ShouldVerifyIP(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion network/network.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ func NewNetwork(
return nil, fmt.Errorf("initializing network metrics failed with: %w", err)
}

ipTracker, err := newIPTracker(log)
ipTracker, err := newIPTracker(log, config.Namespace, metricsRegisterer)
if err != nil {
return nil, fmt.Errorf("initializing ip tracker failed with: %w", err)
}
Expand Down
Loading