More metrics for p3 and p7 (#213)

* Add behaviourPenalty metrics

* Add duplicateMsgDelivery metric

* Observe topic and peer in duplicateMsgDelivery topic

* Remove peerId from duplicateMsgDelivery metric

* Use min meshMessageDeliveriesWindow

* Use topic label for duplicateMsgDelivery metric

* Record duplicateMsgDelivery in all cases

ts/index.ts

@@ -392,7 +392,13 @@ export default class Gossipsub extends EventEmitter {
       }
 
       const metrics = getMetrics(options.metricsRegister, options.metricsTopicStrToLabel, {
-        gossipPromiseExpireSec: constants.GossipsubIWantFollowupTime / 1000
+        gossipPromiseExpireSec: constants.GossipsubIWantFollowupTime / 1000,
+        behaviourPenaltyThreshold: opts.scoreParams.behaviourPenaltyThreshold,
+        // in theory, each topic has its own meshMessageDeliveriesWindow param
+        // however in lodestar, we configure it the same so just pick the min one
+        minMeshMessageDeliveriesWindow: Math.min(
+          ...Object.values(opts.scoreParams.topics).map((topicParam) => topicParam.meshMessageDeliveriesWindow)
+        )
       })
 
       metrics.mcacheSize.addCollect(() => this.onScrapeMetrics(metrics))
@@ -2522,6 +2528,7 @@ export default class Gossipsub extends EventEmitter {
       const score = this.score.score(peerIdStr)
       scores.push(score)
       scoreByPeer.set(peerIdStr, score)
+      metrics.behaviourPenalty.observe(this.score.peerStats.get(peerIdStr)?.behaviourPenalty ?? 0)
     }
 
     metrics.registerScores(scores, this.opts.scoreThresholds)

ts/metrics.ts

@@ -161,7 +161,7 @@ export type Metrics = ReturnType<typeof getMetrics>
 export function getMetrics(
   register: MetricsRegister,
   topicStrToLabel: TopicStrToLabel,
-  opts: { gossipPromiseExpireSec: number }
+  opts: { gossipPromiseExpireSec: number; behaviourPenaltyThreshold: number; minMeshMessageDeliveriesWindow: number }
 ) {
   // Using function style instead of class to prevent having to re-declare all MetricsPrometheus types.
 
@@ -340,6 +340,17 @@ export function getMetrics(
       help: 'Tracks specific reason of invalid',
       labelNames: ['topic', 'error']
     }),
+    /** Track duplicate message delivery time */
+    duplicateMsgDelivery: register.histogram<{ topic: TopicLabel }>({
+      name: 'gossisub_duplicate_msg_delivery_delay_seconds',
+      help: 'Time since the 1st duplicated message validated',
+      labelNames: ['topic'],
+      buckets: [
+        0.5 * opts.minMeshMessageDeliveriesWindow,
+        1 * opts.minMeshMessageDeliveriesWindow,
+        2 * opts.minMeshMessageDeliveriesWindow
+      ]
+    }),
 
     /* Metrics related to scoring */
     /** Total times score() is called */
@@ -388,6 +399,15 @@ export function getMetrics(
       help: 'A counter of the kind of penalties being applied to peers',
       labelNames: ['penalty']
     }),
+    behaviourPenalty: register.histogram({
+      name: 'gossipsub_peer_stat_behaviour_penalty',
+      help: 'Current peer stat behaviour_penalty at each scrape',
+      buckets: [
+        0.5 * opts.behaviourPenaltyThreshold,
+        1 * opts.behaviourPenaltyThreshold,
+        2 * opts.behaviourPenaltyThreshold
+      ]
+    }),
 
     // TODO:
     // - iasked per peer (on heartbeat)
@@ -572,6 +592,11 @@ export function getMetrics(
       this.msgReceivedInvalid.inc({ topic, error }, 1)
     },
 
+    onDuplicateMsgDelivery(topicStr: TopicStr, deliveryDelayMs: number): void {
+      const topic = this.toTopic(topicStr)
+      this.duplicateMsgDelivery.observe({ topic }, deliveryDelayMs / 1000)
+    },
+
     onRpcRecv(rpc: IRPC, rpcBytes: number): void {
       this.rpcRecvBytes.inc(rpcBytes)
       this.rpcRecvCount.inc(1)

ts/score/peer-score.ts

@@ -474,8 +474,13 @@ export class PeerScore {
         // check against the mesh delivery window -- if the validated time is passed as 0, then
         // the message was received before we finished validation and thus falls within the mesh
         // delivery window.
-        if (validatedTime && now > validatedTime + tparams.meshMessageDeliveriesWindow) {
-          return
+        if (validatedTime) {
+          const deliveryDelayMs = now - validatedTime
+          this.metrics?.onDuplicateMsgDelivery(topic, deliveryDelayMs)
+
+          if (deliveryDelayMs > tparams.meshMessageDeliveriesWindow) {
+            return
+          }
         }
 
         const cap = tparams.meshMessageDeliveriesCap