Redesign CSF framework (RIPD-1361):

- Separate `Scheduler` from `BasicNetwork`.
- Add an event/collector framework for monitoring invariants and calculating statistics.
- Allow distinct network and trust connections between Peers.
- Add a simple routing strategy to support broadcasting arbitrary messages.
- Add a common directed graph (`Digraph`) class for representing network and trust topologies.
- Add a `PeerGroup` class for simpler specification of the trust and network topologies.
- Add a `LedgerOracle` class to ensure distinct ledger histories and simplify branch checking.
- Add a `Submitter` to send transactions in at fixed or random intervals to fixed or random peers.

Co-authored-by: Joseph McGee

Builds/VisualStudio2015/RippleD.vcxproj

@@ -4489,14 +4489,26 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\src\test\consensus\ByzantineFailureSim_test.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="..\..\src\test\consensus\Consensus_test.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\src\test\consensus\DistributedValidatorsSim_test.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="..\..\src\test\consensus\LedgerTiming_test.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\src\test\consensus\ScaleFreeSim_test.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="..\..\src\test\consensus\Validations_test.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
@@ -4545,19 +4557,61 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
     </ClCompile>
-    <ClCompile Include="..\..\src\test\csf\impl\UNL.cpp">
+    <ClInclude Include="..\..\src\test\csf\CollectorRef.h">
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\collectors.h">
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\Digraph.h">
+    </ClInclude>
+    <ClCompile Include="..\..\src\test\csf\Digraph_test.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+    </ClCompile>
+    <ClInclude Include="..\..\src\test\csf\events.h">
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\Histogram.h">
+    </ClInclude>
+    <ClCompile Include="..\..\src\test\csf\Histogram_test.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+    </ClCompile>
+    <ClCompile Include="..\..\src\test\csf\impl\ledgers.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
     </ClCompile>
-    <ClInclude Include="..\..\src\test\csf\Ledger.h">
+    <ClCompile Include="..\..\src\test\csf\impl\Sim.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+    </ClCompile>
+    <ClInclude Include="..\..\src\test\csf\ledgers.h">
     </ClInclude>
     <ClInclude Include="..\..\src\test\csf\Peer.h">
     </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\PeerGroup.h">
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\Proposal.h">
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\random.h">
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\Scheduler.h">
+    </ClInclude>
+    <ClCompile Include="..\..\src\test\csf\Scheduler_test.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+    </ClCompile>
     <ClInclude Include="..\..\src\test\csf\Sim.h">
     </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\SimTime.h">
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\submitters.h">
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\timers.h">
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\TrustGraph.h">
+    </ClInclude>
     <ClInclude Include="..\..\src\test\csf\Tx.h">
     </ClInclude>
-    <ClInclude Include="..\..\src\test\csf\UNL.h">
+    <ClInclude Include="..\..\src\test\csf\Validation.h">
     </ClInclude>
     <ClCompile Include="..\..\src\test\json\json_value_test.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>

Builds/VisualStudio2015/RippleD.vcxproj.filters

@@ -5250,12 +5250,21 @@
     <ClCompile Include="..\..\src\test\conditions\PreimageSha256_test.cpp">
       <Filter>test\conditions</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\test\consensus\ByzantineFailureSim_test.cpp">
+      <Filter>test\consensus</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\test\consensus\Consensus_test.cpp">
       <Filter>test\consensus</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\test\consensus\DistributedValidatorsSim_test.cpp">
+      <Filter>test\consensus</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\test\consensus\LedgerTiming_test.cpp">
       <Filter>test\consensus</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\test\consensus\ScaleFreeSim_test.cpp">
+      <Filter>test\consensus</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\test\consensus\Validations_test.cpp">
       <Filter>test\consensus</Filter>
     </ClCompile>
@@ -5295,22 +5304,73 @@
     <ClCompile Include="..\..\src\test\csf\BasicNetwork_test.cpp">
       <Filter>test\csf</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\src\test\csf\impl\UNL.cpp">
+    <ClInclude Include="..\..\src\test\csf\CollectorRef.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\collectors.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\Digraph.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
+    <ClCompile Include="..\..\src\test\csf\Digraph_test.cpp">
+      <Filter>test\csf</Filter>
+    </ClCompile>
+    <ClInclude Include="..\..\src\test\csf\events.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\Histogram.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
+    <ClCompile Include="..\..\src\test\csf\Histogram_test.cpp">
+      <Filter>test\csf</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\test\csf\impl\ledgers.cpp">
       <Filter>test\csf\impl</Filter>
     </ClCompile>
-    <ClInclude Include="..\..\src\test\csf\Ledger.h">
+    <ClCompile Include="..\..\src\test\csf\impl\Sim.cpp">
+      <Filter>test\csf\impl</Filter>
+    </ClCompile>
+    <ClInclude Include="..\..\src\test\csf\ledgers.h">
       <Filter>test\csf</Filter>
     </ClInclude>
     <ClInclude Include="..\..\src\test\csf\Peer.h">
       <Filter>test\csf</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\PeerGroup.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\Proposal.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\random.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\Scheduler.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
+    <ClCompile Include="..\..\src\test\csf\Scheduler_test.cpp">
+      <Filter>test\csf</Filter>
+    </ClCompile>
     <ClInclude Include="..\..\src\test\csf\Sim.h">
       <Filter>test\csf</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\SimTime.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\submitters.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\timers.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\test\csf\TrustGraph.h">
+      <Filter>test\csf</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\src\test\csf\Tx.h">
       <Filter>test\csf</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\src\test\csf\UNL.h">
+    <ClInclude Include="..\..\src\test\csf\Validation.h">
       <Filter>test\csf</Filter>
     </ClInclude>
     <ClCompile Include="..\..\src\test\json\json_value_test.cpp">

docs/consensus.qbk

@@ -458,10 +458,12 @@ struct Ledger
 {
   using ID = ...;
 
+  using Seq = //std::uint32_t?...;
+
   ID const & id() const;
 
   // Sequence number that is 1 more than the parent ledger's seq()
-  std::size_t seq() const;
+  Seq seq() const;
 
   // Whether the ledger's close time was a non-trivial consensus result
   bool closeAgree() const;
@@ -633,14 +635,14 @@ struct Adaptor
     // Propose the position to peers.
     void propose(ConsensusProposal<...> const & pos);
 
-    // Relay a received peer proposal on to other peer's.
-    void relay(PeerPosition_t const & pos);
+    // Share a received peer proposal with other peers.
+    void share(PeerPosition_t const & pos);
 
-    // Relay a disputed transaction to peers
-    void relay(TxSet::Tx const & tx);
+    // Share a disputed transaction with peers
+    void share(TxSet::Tx const & tx);
 
-    // Realy given transaction set with peers
-    void relay(TxSet const &s);
+    // Share given transaction set with peers
+    void share(TxSet const &s);
 
     //... implementation specific
 };
@@ -649,9 +651,9 @@ struct Adaptor
 The implementing class hides many details of the peer communication
 model from the generic code.
 
-* The =relay= member functions are responsible for sharing the given type with a
+* The =share= member functions are responsible for sharing the given type with a
   node's peers, but are agnostic to the mechanism. Ideally, messages are delivered
-  faster than =LEDGER_GRANULARITY=.
+  faster than =LEDGER_GRANULARITY=. 
 * The generic code does not specify how transactions are submitted by clients,
   propagated through the network or stored in the open ledger. Indeed, the open
   ledger is only conceptual from the perspective of the generic code---the

src/ripple/app/consensus/RCLConsensus.cpp

@@ -126,7 +126,7 @@ RCLConsensus::Adaptor::acquireLedger(LedgerHash const& ledger)
 
 
 void
-RCLConsensus::Adaptor::relay(RCLCxPeerPos const& peerPos)
+RCLConsensus::Adaptor::share(RCLCxPeerPos const& peerPos)
 {
     protocol::TMProposeSet prop;
 
@@ -150,7 +150,7 @@ RCLConsensus::Adaptor::relay(RCLCxPeerPos const& peerPos)
 }
 
 void
-RCLConsensus::Adaptor::relay(RCLCxTx const& tx)
+RCLConsensus::Adaptor::share(RCLCxTx const& tx)
 {
     // If we didn't relay this transaction recently, relay it to all peers
     if (app_.getHashRouter().shouldRelay(tx.id()))
@@ -204,7 +204,7 @@ RCLConsensus::Adaptor::propose(RCLCxPeerPos::Proposal const& proposal)
 }
 
 void
-RCLConsensus::Adaptor::relay(RCLTxSet const& set)
+RCLConsensus::Adaptor::share(RCLTxSet const& set)
 {
     inboundTransactions_.giveSet(set.id(), set.map_, false);
 }
@@ -254,19 +254,7 @@ RCLConsensus::Adaptor::getPrevLedger(
         app_.getValidations().currentTrustedDistribution(
             ledgerID, parentID, ledgerMaster_.getValidLedgerIndex());
 
-    uint256 netLgr = ledgerID;
-    int netLgrCount = 0;
-    for (auto const & it : ledgerCounts)
-    {
-        // Switch to ledger supported by more peers
-        // Or stick with ours on a tie
-        if ((it.second > netLgrCount) ||
-            ((it.second == netLgrCount) && (it.first == ledgerID)))
-        {
-            netLgr = it.first;
-            netLgrCount = it.second;
-        }
-    }
+    uint256 netLgr = getPreferredLedger(ledgerID, ledgerCounts);
 
     if (netLgr != ledgerID)
     {

src/ripple/app/consensus/RCLConsensus.h

@@ -158,21 +158,21 @@ class RCLConsensus
         boost::optional<RCLCxLedger>
         acquireLedger(LedgerHash const& ledger);
 
-        /** Relay the given proposal to all peers
+        /** Share the given proposal with all peers
 
-            @param peerPos The peer position to relay.
+            @param peerPos The peer position to share.
          */
         void
-        relay(RCLCxPeerPos const& peerPos);
+        share(RCLCxPeerPos const& peerPos);
 
-        /** Relay disputed transacction to peers.
+        /** Share disputed transaction to peers.
 
-            Only relay if the provided transaction hasn't been shared recently.
+            Only share if the provided transaction hasn't been shared recently.
 
-            @param tx The disputed transaction to relay.
+            @param tx The disputed transaction to share.
         */
         void
-        relay(RCLCxTx const& tx);
+        share(RCLCxTx const& tx);
 
         /** Acquire the transaction set associated with a proposal.
 
@@ -215,12 +215,12 @@ class RCLConsensus
         void
         propose(RCLCxPeerPos::Proposal const& proposal);
 
-        /** Relay the given tx set to peers.
+        /** Share the given tx set to peers.
 
             @param set The TxSet to share.
         */
         void
-        relay(RCLTxSet const& set);
+        share(RCLTxSet const& set);
 
         /** Get the ID of the previous ledger/last closed ledger(LCL) on the
            network

src/ripple/app/consensus/RCLCxLedger.h

@@ -37,6 +37,8 @@ class RCLCxLedger
 public:
     //! Unique identifier of a ledger
     using ID = LedgerHash;
+    //! Sequence number of a ledger
+    using Seq = LedgerIndex;
 
     /** Default constructor
 
@@ -55,28 +57,28 @@ class RCLCxLedger
     }
 
     //! Sequence number of the ledger.
-    auto const&
+    Seq const&
     seq() const
     {
         return ledger_->info().seq;
     }
 
     //! Unique identifier (hash) of this ledger.
-    auto const&
+    ID const&
     id() const
     {
         return ledger_->info().hash;
     }
 
     //! Unique identifier (hash) of this ledger's parent.
-    auto const&
+    ID const&
     parentID() const
     {
         return ledger_->info().parentHash;
     }
 
     //! Resolution used when calculating this ledger's close time.
-    auto
+    NetClock::duration
     closeTimeResolution() const
     {
         return ledger_->info().closeTimeResolution;
@@ -90,14 +92,14 @@ class RCLCxLedger
     }
 
     //! The close time of this ledger
-    auto
+    NetClock::time_point
     closeTime() const
     {
         return ledger_->info().closeTime;
     }
 
     //! The close time of this ledger's parent.
-    auto
+    NetClock::time_point
     parentCloseTime() const
     {
         return ledger_->info().parentCloseTime;