move gossipsub into a separate crate

Cargo.toml

@@ -8,6 +8,7 @@ members = [
     "beacon_node/builder_client",
     "beacon_node/client",
     "beacon_node/eth1",
+    "beacon_node/gossipsub",
     "beacon_node/lighthouse_network",
     "beacon_node/execution_layer",
     "beacon_node/http_api",

beacon_node/gossipsub/Cargo.toml

@@ -0,0 +1,53 @@
+[package]
+name = "gossipsub"
+edition = "2021"
+description = "Sigma prime's version of Gossipsub protocol for libp2p"
+version = "0.5.0"
+authors = ["Age Manning <[email protected]>"]
+license = "MIT"
+repository = "https://github.com/sigp/lighthouse/"
+keywords = ["peer-to-peer", "libp2p", "networking"]
+categories = ["network-programming", "asynchronous"]
+
+[features]
+wasm-bindgen = ["getrandom/js", "instant/wasm-bindgen"]
+
+[dependencies]
+async-channel = { workspace = true }
+asynchronous-codec = "0.7.0"
+base64 = "0.21.7"
+byteorder = "1.5.0"
+bytes = "1.5"
+either = "1.9"
+fnv = "1.0.7"
+futures = "0.3.30"
+futures-ticker = "0.0.3"
+futures-timer = "3.0.2"
+getrandom = "0.2.12"
+hex_fmt = "0.3.0"
+instant = "0.1.12"
+libp2p = { version = "0.53", default-features = false, features = ["identify", "yamux", "noise", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa", "metrics", "quic", "upnp"]}
+quick-protobuf = "0.8"
+quick-protobuf-codec = "0.3"
+rand = "0.8"
+regex = "1.10.3"
+serde = { version = "1", optional = true, features = ["derive"] }
+sha2 = "0.10.8"
+smallvec = "1.13.1"
+tracing = "0.1.37"
+void = "1.0.2"
+
+prometheus-client = "0.22.0"
+
+[dev-dependencies]
+async-std = { version = "1.6.3", features = ["unstable"] }
+hex = "0.4.2"
+quickcheck = { workspace = true }
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+
+# Passing arguments to the docsrs builder in order to properly document cfg's.
+# More information: https://docs.rs/about/builds#cross-compiling
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]
+rustc-args = ["--cfg", "docsrs"]

beacon_node/lighthouse_network/src/gossipsub/backoff.rs → beacon_node/gossipsub/src/backoff.rs

@@ -19,7 +19,7 @@
 // DEALINGS IN THE SOFTWARE.
 
 //! Data structure for efficiently storing known back-off's when pruning peers.
-use crate::gossipsub::topic::TopicHash;
+use crate::topic::TopicHash;
 use instant::Instant;
 use libp2p::identity::PeerId;
 use std::collections::{

beacon_node/lighthouse_network/src/gossipsub/behaviour.rs → beacon_node/gossipsub/src/behaviour.rs

@@ -57,16 +57,16 @@ use super::time_cache::DuplicateCache;
 use super::topic::{Hasher, Topic, TopicHash};
 use super::transform::{DataTransform, IdentityTransform};
 use super::types::{
-    ControlAction, Message, MessageAcceptance, MessageId, PeerInfo, RawMessage, Subscription,
-    SubscriptionAction,
+    ControlAction, FailedMessages, Message, MessageAcceptance, MessageId, PeerInfo, RawMessage,
+    Subscription, SubscriptionAction,
 };
 use super::types::{Graft, IHave, IWant, PeerConnections, PeerKind, Prune};
 use super::{backoff::BackoffStorage, types::RpcSender};
 use super::{
     config::{Config, ValidationMode},
     types::RpcOut,
 };
-use super::{FailedMessages, PublishError, SubscriptionError, TopicScoreParams, ValidationError};
+use super::{PublishError, SubscriptionError, TopicScoreParams, ValidationError};
 use instant::SystemTime;
 use quick_protobuf::{MessageWrite, Writer};
 use std::{cmp::Ordering::Equal, fmt::Debug};

beacon_node/lighthouse_network/src/gossipsub/behaviour/tests.rs → beacon_node/gossipsub/src/behaviour/tests.rs

@@ -21,11 +21,11 @@
 // Collection of tests for the gossipsub network behaviour
 
 use super::*;
-use crate::gossipsub::subscription_filter::WhitelistSubscriptionFilter;
-use crate::gossipsub::transform::{DataTransform, IdentityTransform};
-use crate::gossipsub::types::{RpcOut, RpcReceiver};
-use crate::gossipsub::ValidationError;
-use crate::gossipsub::{
+use crate::subscription_filter::WhitelistSubscriptionFilter;
+use crate::transform::{DataTransform, IdentityTransform};
+use crate::types::{RpcOut, RpcReceiver};
+use crate::ValidationError;
+use crate::{
     config::Config, config::ConfigBuilder, types::Rpc, IdentTopic as Topic, TopicScoreParams,
 };
 use async_std::net::Ipv4Addr;

beacon_node/lighthouse_network/src/gossipsub/config.rs → beacon_node/gossipsub/src/config.rs

@@ -36,7 +36,7 @@ pub enum ValidationMode {
     /// be present as well as the sequence number. All messages must have valid signatures.
     ///
     /// NOTE: This setting will reject messages from nodes using
-    /// [`crate::gossipsub::behaviour::MessageAuthenticity::Anonymous`] and all messages that do not have
+    /// [`crate::behaviour::MessageAuthenticity::Anonymous`] and all messages that do not have
     /// signatures.
     Strict,
     /// This setting permits messages that have no author, sequence number or signature. If any of
@@ -195,7 +195,7 @@ impl Config {
 
     /// When set to `true`, prevents automatic forwarding of all received messages. This setting
     /// allows a user to validate the messages before propagating them to their peers. If set to
-    /// true, the user must manually call [`crate::gossipsub::Behaviour::report_message_validation_result()`]
+    /// true, the user must manually call [`crate::Behaviour::report_message_validation_result()`]
     /// on the behaviour to forward message once validated (default is `false`).
     /// The default is `false`.
     pub fn validate_messages(&self) -> bool {
@@ -611,7 +611,7 @@ impl ConfigBuilder {
 
     /// When set, prevents automatic forwarding of all received messages. This setting
     /// allows a user to validate the messages before propagating them to their peers. If set,
-    /// the user must manually call [`crate::gossipsub::Behaviour::report_message_validation_result()`] on the
+    /// the user must manually call [`crate::Behaviour::report_message_validation_result()`] on the
     /// behaviour to forward a message once validated.
     pub fn validate_messages(&mut self) -> &mut Self {
         self.config.validate_messages = true;
@@ -902,9 +902,9 @@ impl std::fmt::Debug for Config {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::gossipsub::topic::IdentityHash;
-    use crate::gossipsub::types::PeerKind;
-    use crate::gossipsub::Topic;
+    use crate::topic::IdentityHash;
+    use crate::types::PeerKind;
+    use crate::Topic;
     use libp2p::core::UpgradeInfo;
     use libp2p::swarm::StreamProtocol;
     use std::collections::hash_map::DefaultHasher;

beacon_node/gossipsub/src/lib.rs

@@ -0,0 +1,134 @@
+// Copyright 2020 Sigma Prime Pty Ltd.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the "Software"),
+// to deal in the Software without restriction, including without limitation
+// the rights to use, copy, modify, merge, publish, distribute, sublicense,
+// and/or sell copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+//! Implementation of the [Gossipsub](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/README.md) protocol.
+//!
+//! Gossipsub is a P2P pubsub (publish/subscription) routing layer designed to extend upon
+//! floodsub and meshsub routing protocols.
+//!
+//! # Overview
+//!
+//! *Note: The gossipsub protocol specifications
+//! (<https://github.com/libp2p/specs/tree/master/pubsub/gossipsub>) provide an outline for the
+//! routing protocol. They should be consulted for further detail.*
+//!
+//! Gossipsub  is a blend of meshsub for data and randomsub for mesh metadata. It provides bounded
+//! degree and amplification factor with the meshsub construction and augments it using gossip
+//! propagation of metadata with the randomsub technique.
+//!
+//! The router maintains an overlay mesh network of peers on which to efficiently send messages and
+//! metadata.  Peers use control messages to broadcast and request known messages and
+//! subscribe/unsubscribe from topics in the mesh network.
+//!
+//! # Important Discrepancies
+//!
+//! This section outlines the current implementation's potential discrepancies from that of other
+//! implementations, due to undefined elements in the current specification.
+//!
+//! - **Topics** -  In gossipsub, topics configurable by the `hash_topics` configuration parameter.
+//! Topics are of type [`TopicHash`]. The current go implementation uses raw utf-8 strings, and this
+//! is default configuration in rust-libp2p. Topics can be hashed (SHA256 hashed then base64
+//! encoded) by setting the `hash_topics` configuration parameter to true.
+//!
+//! - **Sequence Numbers** - A message on the gossipsub network is identified by the source
+//! [`PeerId`](libp2p_identity::PeerId) and a nonce (sequence number) of the message. The sequence numbers in
+//! this implementation are sent as raw bytes across the wire. They are 64-bit big-endian unsigned
+//! integers. When messages are signed, they are monotonically increasing integers starting from a
+//! random value and wrapping around u64::MAX. When messages are unsigned, they are chosen at random.
+//! NOTE: These numbers are sequential in the current go implementation.
+//!
+//! # Peer Discovery
+//!
+//! Gossipsub does not provide peer discovery by itself. Peer discovery is the process by which
+//! peers in a p2p network exchange information about each other among other reasons to become resistant
+//! against the failure or replacement of the
+//! [boot nodes](https://docs.libp2p.io/reference/glossary/#boot-node) of the network.
+//!
+//! Peer
+//! discovery can e.g. be implemented with the help of the [Kademlia](https://github.com/libp2p/specs/blob/master/kad-dht/README.md) protocol
+//! in combination with the [Identify](https://github.com/libp2p/specs/tree/master/identify) protocol. See the
+//! Kademlia implementation documentation for more information.
+//!
+//! # Using Gossipsub
+//!
+//! ## Gossipsub Config
+//!
+//! The [`Config`] struct specifies various network performance/tuning configuration
+//! parameters. Specifically it specifies:
+//!
+//! [`Config`]: struct.Config.html
+//!
+//! This struct implements the [`Default`] trait and can be initialised via
+//! [`Config::default()`].
+//!
+//!
+//! ## Behaviour
+//!
+//! The [`Behaviour`] struct implements the [`libp2p_swarm::NetworkBehaviour`] trait allowing it to
+//! act as the routing behaviour in a [`libp2p_swarm::Swarm`]. This struct requires an instance of
+//! [`PeerId`](libp2p_identity::PeerId) and [`Config`].
+//!
+//! [`Behaviour`]: struct.Behaviour.html
+
+//! ## Example
+//!
+//! For an example on how to use gossipsub, see the [chat-example](https://github.com/libp2p/rust-libp2p/tree/master/examples/chat).
+
+#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
+
+mod backoff;
+mod behaviour;
+mod config;
+mod error;
+mod gossip_promises;
+mod handler;
+mod mcache;
+mod metrics;
+mod peer_score;
+mod protocol;
+mod rpc_proto;
+mod subscription_filter;
+mod time_cache;
+mod topic;
+mod transform;
+mod types;
+
+pub use self::behaviour::{Behaviour, Event, MessageAuthenticity};
+pub use self::config::{Config, ConfigBuilder, ValidationMode, Version};
+pub use self::error::{ConfigBuilderError, PublishError, SubscriptionError, ValidationError};
+pub use self::metrics::Config as MetricsConfig;
+pub use self::peer_score::{
+    score_parameter_decay, score_parameter_decay_with_base, PeerScoreParams, PeerScoreThresholds,
+    TopicScoreParams,
+};
+pub use self::subscription_filter::{
+    AllowAllSubscriptionFilter, CallbackSubscriptionFilter, CombinedSubscriptionFilters,
+    MaxCountSubscriptionFilter, RegexSubscriptionFilter, TopicSubscriptionFilter,
+    WhitelistSubscriptionFilter,
+};
+pub use self::topic::{Hasher, Topic, TopicHash};
+pub use self::transform::{DataTransform, IdentityTransform};
+pub use self::types::{FailedMessages, Message, MessageAcceptance, MessageId, RawMessage};
+
+#[deprecated(note = "Will be removed from the public API.")]
+pub type Rpc = self::types::Rpc;
+
+pub type IdentTopic = Topic<self::topic::IdentityHash>;
+pub type Sha256Topic = Topic<self::topic::Sha256Hash>;

beacon_node/lighthouse_network/src/gossipsub/mcache.rs → beacon_node/gossipsub/src/mcache.rs

@@ -221,7 +221,7 @@ impl MessageCache {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::gossipsub::types::RawMessage;
+    use crate::types::RawMessage;
     use crate::{IdentTopic as Topic, TopicHash};
     use libp2p::identity::PeerId;
 

beacon_node/lighthouse_network/src/gossipsub/peer_score/params.rs → beacon_node/gossipsub/src/peer_score/params.rs

@@ -18,7 +18,7 @@
 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 // DEALINGS IN THE SOFTWARE.
 
-use crate::gossipsub::TopicHash;
+use crate::TopicHash;
 use std::collections::{HashMap, HashSet};
 use std::net::IpAddr;
 use std::time::Duration;

beacon_node/lighthouse_network/src/gossipsub/peer_score/tests.rs → beacon_node/gossipsub/src/peer_score/tests.rs

@@ -21,8 +21,8 @@
 /// A collection of unit tests mostly ported from the go implementation.
 use super::*;
 
-use crate::gossipsub::types::RawMessage;
-use crate::gossipsub::{IdentTopic as Topic, Message};
+use crate::types::RawMessage;
+use crate::{IdentTopic as Topic, Message};
 
 // estimates a value within variance
 fn within_variance(value: f64, expected: f64, variance: f64) -> bool {