Implement Shards

XRPLF · Jan 17, 2018 · 718d217 · 718d217
1 parent aeda243
commit 718d217
Show file tree

Hide file tree

Showing 74 changed files with 3,985 additions and 1,404 deletions.
diff --git a/Builds/VisualStudio2015/RippleD.vcxproj b/Builds/VisualStudio2015/RippleD.vcxproj
@@ -2275,6 +2275,8 @@
     </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\DatabaseRotating.h">
     </ClInclude>
+    <ClInclude Include="..\..\src\ripple\nodestore\DatabaseShard.h">
+    </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\DummyScheduler.h">
     </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\Factory.h">
@@ -2289,7 +2291,19 @@
     </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\impl\codec.h">
     </ClInclude>
-    <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseImp.h">
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\Database.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseNodeImp.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+    <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseNodeImp.h">
     </ClInclude>
     <ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseRotatingImp.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
@@ -2299,6 +2313,14 @@
     </ClCompile>
     <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseRotatingImp.h">
     </ClInclude>
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseShardImp.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+    <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseShardImp.h">
+    </ClInclude>
     <ClCompile Include="..\..\src\ripple\nodestore\impl\DecodedBlob.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
@@ -2335,6 +2357,14 @@
       <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\Shard.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+    <ClInclude Include="..\..\src\ripple\nodestore\impl\Shard.h">
+    </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\impl\Tuning.h">
     </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\impl\varint.h">

diff --git a/Builds/VisualStudio2015/RippleD.vcxproj.filters b/Builds/VisualStudio2015/RippleD.vcxproj.filters
@@ -2931,6 +2931,9 @@
     <ClInclude Include="..\..\src\ripple\nodestore\DatabaseRotating.h">
       <Filter>ripple\nodestore</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\ripple\nodestore\DatabaseShard.h">
+      <Filter>ripple\nodestore</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\DummyScheduler.h">
       <Filter>ripple\nodestore</Filter>
     </ClInclude>
@@ -2946,7 +2949,13 @@
     <ClInclude Include="..\..\src\ripple\nodestore\impl\codec.h">
       <Filter>ripple\nodestore\impl</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseImp.h">
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\Database.cpp">
+      <Filter>ripple\nodestore\impl</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseNodeImp.cpp">
+      <Filter>ripple\nodestore\impl</Filter>
+    </ClCompile>
+    <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseNodeImp.h">
       <Filter>ripple\nodestore\impl</Filter>
     </ClInclude>
     <ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseRotatingImp.cpp">
@@ -2955,6 +2964,12 @@
     <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseRotatingImp.h">
       <Filter>ripple\nodestore\impl</Filter>
     </ClInclude>
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseShardImp.cpp">
+      <Filter>ripple\nodestore\impl</Filter>
+    </ClCompile>
+    <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseShardImp.h">
+      <Filter>ripple\nodestore\impl</Filter>
+    </ClInclude>
     <ClCompile Include="..\..\src\ripple\nodestore\impl\DecodedBlob.cpp">
       <Filter>ripple\nodestore\impl</Filter>
     </ClCompile>
@@ -2979,6 +2994,12 @@
     <ClCompile Include="..\..\src\ripple\nodestore\impl\NodeObject.cpp">
       <Filter>ripple\nodestore\impl</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\Shard.cpp">
+      <Filter>ripple\nodestore\impl</Filter>
+    </ClCompile>
+    <ClInclude Include="..\..\src\ripple\nodestore\impl\Shard.h">
+      <Filter>ripple\nodestore\impl</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\impl\Tuning.h">
       <Filter>ripple\nodestore\impl</Filter>
     </ClInclude>

diff --git a/doc/rippled-example.cfg b/doc/rippled-example.cfg
@@ -834,6 +834,33 @@
 #   [import_db]     Settings for performing a one-time import (optional)
 #   [database_path]   Path to the book-keeping databases.
 #
+#   [shard_db]      Settings for the Shard Database (optional)
+#
+#   Format (without spaces):
+#       One or more lines of case-insensitive key / value pairs:
+#       <key> '=' <value>
+#       ...
+#
+#   Example:
+#       type=nudb
+#       path=db/nudb
+#
+#   The "type" field must be present and controls the choice of backend:
+#
+#   type = NuDB
+#
+#   type = RocksDB
+#
+#       The RocksDB backend also provides these optional parameters:
+#
+#       compression         0 for none, 1 for Snappy compression
+#
+#   Required keys:
+#       path                Location to store the database (all types)
+#
+#       max_size_gb         Maximum disk space the database will utilize (in gigabytes)
+#
+#
 #   There are 4 bookkeeping SQLite database that the server creates and
 #   maintains. If you omit this configuration setting, it will default to
 #   creating a directory called "db" located in the same place as your

diff --git a/src/ripple/app/consensus/RCLConsensus.cpp b/src/ripple/app/consensus/RCLConsensus.cpp
@@ -36,6 +36,7 @@
 #include <ripple/basics/make_lock.h>
 #include <ripple/beast/core/LexicalCast.h>
 #include <ripple/consensus/LedgerTiming.h>
+#include <ripple/nodestore/DatabaseShard.h>
 #include <ripple/overlay/Overlay.h>
 #include <ripple/overlay/predicates.h>
 #include <ripple/protocol/Feature.h>
@@ -106,7 +107,7 @@ RCLConsensus::Adaptor::acquireLedger(LedgerHash const& ledger)
             app_.getJobQueue().addJob(
                 jtADVANCE, "getConsensusLedger", [app, hash](Job&) {
                     app->getInboundLedgers().acquire(
-                        hash, 0, InboundLedger::fcCONSENSUS);
+                        hash, 0, InboundLedger::Reason::CONSENSUS);
                 });
         }
         return boost::none;
@@ -625,9 +626,16 @@ RCLConsensus::Adaptor::notify(
     }
     s.set_firstseq(uMin);
     s.set_lastseq(uMax);
-    app_.overlay().foreach (
-        send_always(std::make_shared<Message>(s, protocol::mtSTATUS_CHANGE)));
-    JLOG(j_.trace()) << "send status change to peer";
+    if (auto shardStore = app_.getShardStore())
+    {
+        auto shards = shardStore->getCompleteShards();
+        if (! shards.empty())
+            s.set_shardseqs(shards);
+    }
+    app_.overlay ().foreach (send_always (
+        std::make_shared <Message> (
+            s, protocol::mtSTATUS_CHANGE)));
+    JLOG (j_.trace()) << "send status change to peer";
 }
 
 /** Apply a set of transactions to a ledger.

diff --git a/src/ripple/app/ledger/AccountStateSF.cpp b/src/ripple/app/ledger/AccountStateSF.cpp
@@ -22,19 +22,13 @@
 
 namespace ripple {
 
-AccountStateSF::AccountStateSF(Family& f, AbstractFetchPackContainer& fp)
-    : f_(f)
-    , fp_(fp)
+void
+AccountStateSF::gotNode(bool, SHAMapHash const& nodeHash,
+    std::uint32_t ledgerSeq, Blob&& nodeData,
+    SHAMapTreeNode::TNType) const
 {
-}
-
-void AccountStateSF::gotNode (bool fromFilter,
-                              SHAMapHash const& nodeHash,
-                              Blob&& nodeData,
-                              SHAMapTreeNode::TNType) const
-{
-    f_.db().store(hotACCOUNT_NODE, std::move(nodeData),
-        nodeHash.as_uint256());
+    db_.store(hotACCOUNT_NODE, std::move(nodeData),
+        nodeHash.as_uint256(), ledgerSeq);
 }
 
 boost::optional<Blob>

diff --git a/src/ripple/app/ledger/AccountStateSF.h b/src/ripple/app/ledger/AccountStateSF.h
@@ -21,31 +21,32 @@
 #define RIPPLE_APP_LEDGER_ACCOUNTSTATESF_H_INCLUDED
 
 #include <ripple/app/ledger/AbstractFetchPackContainer.h>
+#include <ripple/nodestore/Database.h>
 #include <ripple/shamap/SHAMapSyncFilter.h>
-#include <ripple/shamap/Family.h>
 
 namespace ripple {
 
 // This class is only needed on add functions
 // sync filter for account state nodes during ledger sync
-class AccountStateSF
-    : public SHAMapSyncFilter
+class AccountStateSF : public SHAMapSyncFilter
 {
-private:
-    Family& f_;
-    AbstractFetchPackContainer& fp_;
-
 public:
-    AccountStateSF(Family&, AbstractFetchPackContainer&);
+    AccountStateSF(NodeStore::Database& db, AbstractFetchPackContainer& fp)
+        : db_(db)
+        , fp_(fp)
+    {}
 
-    // Note that the nodeData is overwritten by this call
-    void gotNode (bool fromFilter,
-                  SHAMapHash const& nodeHash,
-                  Blob&& nodeData,
-                  SHAMapTreeNode::TNType) const override;
+    void
+    gotNode(bool fromFilter, SHAMapHash const& nodeHash,
+        std::uint32_t ledgerSeq, Blob&& nodeData,
+            SHAMapTreeNode::TNType type) const override;
 
     boost::optional<Blob>
     getNode(SHAMapHash const& nodeHash) const override;
+
+private:
+    NodeStore::Database& db_;
+    AbstractFetchPackContainer& fp_;
 };
 
 } // ripple

diff --git a/src/ripple/app/ledger/ConsensusTransSetSF.cpp b/src/ripple/app/ledger/ConsensusTransSetSF.cpp
@@ -38,9 +38,9 @@ ConsensusTransSetSF::ConsensusTransSetSF (Application& app, NodeCache& nodeCache
 {
 }
 
-void ConsensusTransSetSF::gotNode (
-    bool fromFilter, SHAMapHash const& nodeHash,
-    Blob&& nodeData, SHAMapTreeNode::TNType type) const
+void
+ConsensusTransSetSF::gotNode(bool fromFilter, SHAMapHash const& nodeHash,
+    std::uint32_t, Blob&& nodeData, SHAMapTreeNode::TNType type) const
 {
     if (fromFilter)
         return;

diff --git a/src/ripple/app/ledger/ConsensusTransSetSF.h b/src/ripple/app/ledger/ConsensusTransSetSF.h
@@ -39,10 +39,10 @@ class ConsensusTransSetSF : public SHAMapSyncFilter
     ConsensusTransSetSF (Application& app, NodeCache& nodeCache);
 
     // Note that the nodeData is overwritten by this call
-    void gotNode (bool fromFilter,
-                  SHAMapHash const& nodeHash,
-                  Blob&& nodeData,
-                  SHAMapTreeNode::TNType) const override;
+    void
+    gotNode(bool fromFilter, SHAMapHash const& nodeHash,
+        std::uint32_t ledgerSeq, Blob&& nodeData,
+            SHAMapTreeNode::TNType type) const override;
 
     boost::optional<Blob>
     getNode (SHAMapHash const& nodeHash) const override;

diff --git a/src/ripple/app/ledger/InboundLedger.h b/src/ripple/app/ledger/InboundLedger.h
@@ -44,18 +44,16 @@ class InboundLedger
         std::shared_ptr<protocol::TMLedgerData>>;
 
     // These are the reasons we might acquire a ledger
-    enum fcReason
+    enum class Reason
     {
-        fcHISTORY,      // Acquiring past ledger
-        fcGENERIC,      // Generic other reasons
-        fcVALIDATION,   // Validations suggest this ledger is important
-        fcCURRENT,      // This might be the current ledger
-        fcCONSENSUS,    // We believe the consensus round requires this ledger
+        HISTORY,  // Acquiring past ledger
+        SHARD,    // Acquiring for shard
+        GENERIC,  // Generic other reasons
+        CONSENSUS // We believe the consensus round requires this ledger
     };
 
-public:
-    InboundLedger(Application& app,
-        uint256 const& hash, std::uint32_t seq, fcReason reason, clock_type&);
+    InboundLedger(Application& app, uint256 const& hash,
+        std::uint32_t seq, Reason reason, clock_type&);
 
     ~InboundLedger ();
 
@@ -70,15 +68,24 @@ class InboundLedger
     {
         return mLedger;
     }
+
     std::uint32_t getSeq () const
     {
         return mSeq;
     }
 
+    Reason
+    getReason() const
+    {
+        return mReason;
+    }
+
     bool checkLocal ();
     void init (ScopedLockType& collectionLock);
 
-    bool gotData (std::weak_ptr<Peer>, std::shared_ptr<protocol::TMLedgerData>);
+    bool
+    gotData(std::weak_ptr<Peer>,
+        std::shared_ptr<protocol::TMLedgerData> const&);
 
     using neededHash_t =
         std::pair <protocol::TMGetObjectByHash::ObjectType, uint256>;
@@ -88,6 +95,10 @@ class InboundLedger
 
     void runData ();
 
+    static
+    LedgerInfo
+    deserializeHeader(Slice data, bool hasPrefix);
+
 private:
     enum class TriggerReason
     {
@@ -105,7 +116,7 @@ class InboundLedger
     std::vector<neededHash_t> getNeededHashes ();
 
     void addPeers ();
-    bool tryLocal ();
+    void tryDB (Family& f);
 
     void done ();
 
@@ -115,7 +126,7 @@ class InboundLedger
     {
         // For historical nodes, do not trigger too soon
         // since a fetch pack is probably coming
-        if (mReason != fcHISTORY)
+        if (mReason != Reason::HISTORY)
             trigger (peer, TriggerReason::added);
     }
 
@@ -146,24 +157,18 @@ class InboundLedger
     neededStateHashes (
         int max, SHAMapSyncFilter* filter) const;
 
-    LedgerInfo
-    deserializeHeader (
-        Slice data,
-        bool hasPrefix);
-
-private:
     std::shared_ptr<Ledger> mLedger;
-    bool               mHaveHeader;
-    bool               mHaveState;
-    bool               mHaveTransactions;
-    bool               mSignaled;
-    bool               mByHash;
-    std::uint32_t      mSeq;
-    fcReason           mReason;
+    bool mHaveHeader;
+    bool mHaveState;
+    bool mHaveTransactions;
+    bool mSignaled;
+    bool mByHash;
+    std::uint32_t mSeq;
+    Reason const mReason;
 
     std::set <uint256> mRecentNodes;
 
-    SHAMapAddNode      mStats;
+    SHAMapAddNode mStats;
 
     // Data we have received from peers
     std::mutex mReceivedDataLock;