Skip to content

Commit

Permalink
Implement Shards
Browse files Browse the repository at this point in the history
  • Loading branch information
@miguelportilla
miguelportilla committed Dec 19, 2017
1 parent c132697 commit 8603f39
Show file tree
Hide file tree
Showing 76 changed files with 3,984 additions and 1,393 deletions.
32 changes: 31 additions & 1 deletion Builds/VisualStudio2015/RippleD.vcxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2257,6 +2257,8 @@
</ClInclude>
<ClInclude Include="..\..\src\ripple\nodestore\DatabaseRotating.h">
</ClInclude>
<ClInclude Include="..\..\src\ripple\nodestore\DatabaseShard.h">
</ClInclude>
<ClInclude Include="..\..\src\ripple\nodestore\DummyScheduler.h">
</ClInclude>
<ClInclude Include="..\..\src\ripple\nodestore\Factory.h">
Expand All @@ -2271,7 +2273,19 @@
</ClInclude>
<ClInclude Include="..\..\src\ripple\nodestore\impl\codec.h">
</ClInclude>
<ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseImp.h">
<ClCompile Include="..\..\src\ripple\nodestore\impl\Database.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
<ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseNodeImp.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
<ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseNodeImp.h">
</ClInclude>
<ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseRotatingImp.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
Expand All @@ -2281,6 +2295,14 @@
</ClCompile>
<ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseRotatingImp.h">
</ClInclude>
<ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseShardImp.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
<ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseShardImp.h">
</ClInclude>
<ClCompile Include="..\..\src\ripple\nodestore\impl\DecodedBlob.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
Expand Down Expand Up @@ -2317,6 +2339,14 @@
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
<ClCompile Include="..\..\src\ripple\nodestore\impl\Shard.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
<ClInclude Include="..\..\src\ripple\nodestore\impl\Shard.h">
</ClInclude>
<ClInclude Include="..\..\src\ripple\nodestore\impl\Tuning.h">
</ClInclude>
<ClInclude Include="..\..\src\ripple\nodestore\impl\varint.h">
Expand Down
23 changes: 22 additions & 1 deletion Builds/VisualStudio2015/RippleD.vcxproj.filters
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2913,6 +2913,9 @@
<ClInclude Include="..\..\src\ripple\nodestore\DatabaseRotating.h">
<Filter>ripple\nodestore</Filter>
</ClInclude>
<ClInclude Include="..\..\src\ripple\nodestore\DatabaseShard.h">
<Filter>ripple\nodestore</Filter>
</ClInclude>
<ClInclude Include="..\..\src\ripple\nodestore\DummyScheduler.h">
<Filter>ripple\nodestore</Filter>
</ClInclude>
Expand All @@ -2928,7 +2931,13 @@
<ClInclude Include="..\..\src\ripple\nodestore\impl\codec.h">
<Filter>ripple\nodestore\impl</Filter>
</ClInclude>
<ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseImp.h">
<ClCompile Include="..\..\src\ripple\nodestore\impl\Database.cpp">
<Filter>ripple\nodestore\impl</Filter>
</ClCompile>
<ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseNodeImp.cpp">
<Filter>ripple\nodestore\impl</Filter>
</ClCompile>
<ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseNodeImp.h">
<Filter>ripple\nodestore\impl</Filter>
</ClInclude>
<ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseRotatingImp.cpp">
Expand All @@ -2937,6 +2946,12 @@
<ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseRotatingImp.h">
<Filter>ripple\nodestore\impl</Filter>
</ClInclude>
<ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseShardImp.cpp">
<Filter>ripple\nodestore\impl</Filter>
</ClCompile>
<ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseShardImp.h">
<Filter>ripple\nodestore\impl</Filter>
</ClInclude>
<ClCompile Include="..\..\src\ripple\nodestore\impl\DecodedBlob.cpp">
<Filter>ripple\nodestore\impl</Filter>
</ClCompile>
Expand All @@ -2961,6 +2976,12 @@
<ClCompile Include="..\..\src\ripple\nodestore\impl\NodeObject.cpp">
<Filter>ripple\nodestore\impl</Filter>
</ClCompile>
<ClCompile Include="..\..\src\ripple\nodestore\impl\Shard.cpp">
<Filter>ripple\nodestore\impl</Filter>
</ClCompile>
<ClInclude Include="..\..\src\ripple\nodestore\impl\Shard.h">
<Filter>ripple\nodestore\impl</Filter>
</ClInclude>
<ClInclude Include="..\..\src\ripple\nodestore\impl\Tuning.h">
<Filter>ripple\nodestore\impl</Filter>
</ClInclude>
Expand Down
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ use_boost(
filesystem
program_options
regex
serialization
system
thread)

Expand Down
3 changes: 2 additions & 1 deletion SConstruct
View file
Original file line number Diff line number Diff line change
Expand Up @@ -613,7 +613,8 @@ def config_env(toolchain, variant, env):
'boost_program_options',
'boost_regex',
'boost_system',
'boost_thread'
'boost_thread',
'boost_serialization'
]
env.Append(LIBS=['dl'])

Expand Down
27 changes: 27 additions & 0 deletions doc/rippled-example.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -836,6 +836,33 @@
# [import_db] Settings for performing a one-time import (optional)
# [database_path] Path to the book-keeping databases.
#
# [shard_db] Settings for the Shard Database (optional)
#
# Format (without spaces):
# One or more lines of case-insensitive key / value pairs:
# <key> '=' <value>
# ...
#
# Example:
# type=nudb
# path=db/nudb
#
# The "type" field must be present and controls the choice of backend:
#
# type = NuDB
#
# type = RocksDB
#
# The RocksDB backend also provides these optional parameters:
#
# compression 0 for none, 1 for Snappy compression
#
# Required keys:
# path Location to store the database (all types)
#
# max_size_gb Maximum disk space the database will utilize (in gigabytes)
#
#
# There are 4 bookkeeping SQLite database that the server creates and
# maintains. If you omit this configuration setting, it will default to
# creating a directory called "db" located in the same place as your
Expand Down
16 changes: 12 additions & 4 deletions src/ripple/app/consensus/RCLConsensus.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
#include <ripple/basics/make_lock.h>
#include <ripple/beast/core/LexicalCast.h>
#include <ripple/consensus/LedgerTiming.h>
#include <ripple/nodestore/DatabaseShard.h>
#include <ripple/overlay/Overlay.h>
#include <ripple/overlay/predicates.h>
#include <ripple/protocol/Feature.h>
Expand Down Expand Up @@ -106,7 +107,7 @@ RCLConsensus::Adaptor::acquireLedger(LedgerHash const& ledger)
app_.getJobQueue().addJob(
jtADVANCE, "getConsensusLedger", [app, hash](Job&) {
app->getInboundLedgers().acquire(
hash, 0, InboundLedger::fcCONSENSUS);
hash, 0, InboundLedger::Reason::CONSENSUS);
});
}
return boost::none;
Expand Down Expand Up @@ -625,9 +626,16 @@ RCLConsensus::Adaptor::notify(
}
s.set_firstseq(uMin);
s.set_lastseq(uMax);
app_.overlay().foreach (
send_always(std::make_shared<Message>(s, protocol::mtSTATUS_CHANGE)));
JLOG(j_.trace()) << "send status change to peer";
if (auto shardStore = app_.getShardStore())
{
auto shards = shardStore->getCompleteShards();
if (! shards.empty())
s.set_shardseqs(shards);
}
app_.overlay ().foreach (send_always (
std::make_shared <Message> (
s, protocol::mtSTATUS_CHANGE)));
JLOG (j_.trace()) << "send status change to peer";
}

/** Apply a set of transactions to a ledger.
Expand Down
18 changes: 6 additions & 12 deletions src/ripple/app/ledger/AccountStateSF.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,13 @@

namespace ripple {

AccountStateSF::AccountStateSF(Family& f, AbstractFetchPackContainer& fp)
: f_(f)
, fp_(fp)
void
AccountStateSF::gotNode(bool, SHAMapHash const& nodeHash,
std::uint32_t ledgerSeq, Blob&& nodeData,
SHAMapTreeNode::TNType) const
{
}

void AccountStateSF::gotNode (bool fromFilter,
SHAMapHash const& nodeHash,
Blob&& nodeData,
SHAMapTreeNode::TNType) const
{
f_.db().store(hotACCOUNT_NODE, std::move(nodeData),
nodeHash.as_uint256());
db_.store(hotACCOUNT_NODE, std::move(nodeData),
nodeHash.as_uint256(), ledgerSeq);
}

boost::optional<Blob>
Expand Down
27 changes: 14 additions & 13 deletions src/ripple/app/ledger/AccountStateSF.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,31 +21,32 @@
#define RIPPLE_APP_LEDGER_ACCOUNTSTATESF_H_INCLUDED

#include <ripple/app/ledger/AbstractFetchPackContainer.h>
#include <ripple/nodestore/Database.h>
#include <ripple/shamap/SHAMapSyncFilter.h>
#include <ripple/shamap/Family.h>

namespace ripple {

// This class is only needed on add functions
// sync filter for account state nodes during ledger sync
class AccountStateSF
: public SHAMapSyncFilter
class AccountStateSF : public SHAMapSyncFilter
{
private:
Family& f_;
AbstractFetchPackContainer& fp_;

public:
AccountStateSF(Family&, AbstractFetchPackContainer&);
AccountStateSF(NodeStore::Database& db, AbstractFetchPackContainer& fp)
: db_(db)
, fp_(fp)
{}

// Note that the nodeData is overwritten by this call
void gotNode (bool fromFilter,
SHAMapHash const& nodeHash,
Blob&& nodeData,
SHAMapTreeNode::TNType) const override;
void
gotNode(bool fromFilter, SHAMapHash const& nodeHash,
std::uint32_t ledgerSeq, Blob&& nodeData,
SHAMapTreeNode::TNType type) const override;

boost::optional<Blob>
getNode(SHAMapHash const& nodeHash) const override;

private:
NodeStore::Database& db_;
AbstractFetchPackContainer& fp_;
};

} // ripple
Expand Down
6 changes: 3 additions & 3 deletions src/ripple/app/ledger/ConsensusTransSetSF.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ ConsensusTransSetSF::ConsensusTransSetSF (Application& app, NodeCache& nodeCache
{
}

void ConsensusTransSetSF::gotNode (
bool fromFilter, SHAMapHash const& nodeHash,
Blob&& nodeData, SHAMapTreeNode::TNType type) const
void
ConsensusTransSetSF::gotNode(bool fromFilter, SHAMapHash const& nodeHash,
std::uint32_t, Blob&& nodeData, SHAMapTreeNode::TNType type) const
{
if (fromFilter)
return;
Expand Down
8 changes: 4 additions & 4 deletions src/ripple/app/ledger/ConsensusTransSetSF.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ class ConsensusTransSetSF : public SHAMapSyncFilter
ConsensusTransSetSF (Application& app, NodeCache& nodeCache);

// Note that the nodeData is overwritten by this call
void gotNode (bool fromFilter,
SHAMapHash const& nodeHash,
Blob&& nodeData,
SHAMapTreeNode::TNType) const override;
void
gotNode(bool fromFilter, SHAMapHash const& nodeHash,
std::uint32_t ledgerSeq, Blob&& nodeData,
SHAMapTreeNode::TNType type) const override;

boost::optional<Blob>
getNode (SHAMapHash const& nodeHash) const override;
Expand Down
Loading

0 comments on commit 8603f39

Please sign in to comment.