Skip to content

Commit

Permalink
Advance ripple.app.rdb
Browse files Browse the repository at this point in the history
  • Loading branch information
@undertome @manojsdoshi
undertome authored and manojsdoshi committed May 10, 2022
1 parent 767dd4f commit dac080f
Show file tree
Hide file tree
Showing 70 changed files with 3,651 additions and 3,865 deletions.
19 changes: 12 additions & 7 deletions Builds/CMake/RippledCore.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,13 +402,18 @@ target_sources (rippled PRIVATE
src/ripple/app/paths/impl/DirectStep.cpp
src/ripple/app/paths/impl/PaySteps.cpp
src/ripple/app/paths/impl/XRPEndpointStep.cpp
src/ripple/app/rdb/backend/RelationalDBInterfacePostgres.cpp
src/ripple/app/rdb/backend/RelationalDBInterfaceSqlite.cpp
src/ripple/app/rdb/impl/RelationalDBInterface.cpp
src/ripple/app/rdb/impl/RelationalDBInterface_global.cpp
src/ripple/app/rdb/impl/RelationalDBInterface_nodes.cpp
src/ripple/app/rdb/impl/RelationalDBInterface_postgres.cpp
src/ripple/app/rdb/impl/RelationalDBInterface_shards.cpp
src/ripple/app/rdb/backend/detail/impl/Node.cpp
src/ripple/app/rdb/backend/detail/impl/Shard.cpp
src/ripple/app/rdb/backend/impl/PostgresDatabase.cpp
src/ripple/app/rdb/backend/impl/SQLiteDatabase.cpp
src/ripple/app/rdb/impl/Download.cpp
src/ripple/app/rdb/impl/PeerFinder.cpp
src/ripple/app/rdb/impl/RelationalDatabase.cpp
src/ripple/app/rdb/impl/ShardArchive.cpp
src/ripple/app/rdb/impl/State.cpp
src/ripple/app/rdb/impl/UnitaryShard.cpp
src/ripple/app/rdb/impl/Vacuum.cpp
src/ripple/app/rdb/impl/Wallet.cpp
src/ripple/app/tx/impl/ApplyContext.cpp
src/ripple/app/tx/impl/BookTip.cpp
src/ripple/app/tx/impl/CancelCheck.cpp
Expand Down
2 changes: 1 addition & 1 deletion Builds/levelization/results/loops.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ Loop: ripple.app ripple.overlay
ripple.overlay ~= ripple.app

Loop: ripple.app ripple.peerfinder
ripple.peerfinder ~= ripple.app
ripple.app > ripple.peerfinder

Loop: ripple.app ripple.rpc
ripple.rpc > ripple.app
Expand Down
27 changes: 16 additions & 11 deletions src/ripple/app/ledger/Ledger.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
#include <ripple/app/misc/HashRouter.h>
#include <ripple/app/misc/LoadFeeTrack.h>
#include <ripple/app/misc/NetworkOPs.h>
#include <ripple/app/rdb/backend/RelationalDBInterfacePostgres.h>
#include <ripple/app/rdb/backend/RelationalDBInterfaceSqlite.h>
#include <ripple/app/rdb/backend/PostgresDatabase.h>
#include <ripple/app/rdb/backend/SQLiteDatabase.h>
#include <ripple/basics/Log.h>
#include <ripple/basics/StringUtilities.h>
#include <ripple/basics/contract.h>
Expand Down Expand Up @@ -930,9 +930,11 @@ saveValidatedLedger(
return true;
}

auto res = dynamic_cast<RelationalDBInterfaceSqlite*>(
&app.getRelationalDBInterface())
->saveValidatedLedger(ledger, current);
auto const db = dynamic_cast<SQLiteDatabase*>(&app.getRelationalDatabase());
if (!db)
Throw<std::runtime_error>("Failed to get relational database");

auto const res = db->saveValidatedLedger(ledger, current);

// Clients can now trust the database for
// information about this ledger sequence.
Expand Down Expand Up @@ -1053,7 +1055,7 @@ std::tuple<std::shared_ptr<Ledger>, std::uint32_t, uint256>
getLatestLedger(Application& app)
{
const std::optional<LedgerInfo> info =
app.getRelationalDBInterface().getNewestLedgerInfo();
app.getRelationalDatabase().getNewestLedgerInfo();
if (!info)
return {std::shared_ptr<Ledger>(), {}, {}};
return {loadLedgerHelper(*info, app, true), info->seq, info->hash};
Expand All @@ -1063,7 +1065,7 @@ std::shared_ptr<Ledger>
loadByIndex(std::uint32_t ledgerIndex, Application& app, bool acquire)
{
if (std::optional<LedgerInfo> info =
app.getRelationalDBInterface().getLedgerInfoByIndex(ledgerIndex))
app.getRelationalDatabase().getLedgerInfoByIndex(ledgerIndex))
{
std::shared_ptr<Ledger> ledger = loadLedgerHelper(*info, app, acquire);
finishLoadByIndexOrHash(ledger, app.config(), app.journal("Ledger"));
Expand All @@ -1076,7 +1078,7 @@ std::shared_ptr<Ledger>
loadByHash(uint256 const& ledgerHash, Application& app, bool acquire)
{
if (std::optional<LedgerInfo> info =
app.getRelationalDBInterface().getLedgerInfoByHash(ledgerHash))
app.getRelationalDatabase().getLedgerInfoByHash(ledgerHash))
{
std::shared_ptr<Ledger> ledger = loadLedgerHelper(*info, app, acquire);
finishLoadByIndexOrHash(ledger, app.config(), app.journal("Ledger"));
Expand Down Expand Up @@ -1165,9 +1167,12 @@ flatFetchTransactions(ReadView const& ledger, Application& app)
return {};
}

auto nodestoreHashes = dynamic_cast<RelationalDBInterfacePostgres*>(
&app.getRelationalDBInterface())
->getTxHashes(ledger.info().seq);
auto const db =
dynamic_cast<PostgresDatabase*>(&app.getRelationalDatabase());
if (!db)
Throw<std::runtime_error>("Failed to get relational database");

auto nodestoreHashes = db->getTxHashes(ledger.info().seq);

return flatFetchTransactions(app, nodestoreHashes);
}
Expand Down
27 changes: 12 additions & 15 deletions src/ripple/app/ledger/impl/LedgerMaster.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,7 @@
#include <ripple/app/misc/TxQ.h>
#include <ripple/app/misc/ValidatorList.h>
#include <ripple/app/paths/PathRequests.h>
#include <ripple/app/rdb/RelationalDBInterface_postgres.h>
#include <ripple/app/rdb/backend/RelationalDBInterfacePostgres.h>
#include <ripple/app/rdb/backend/PostgresDatabase.h>
#include <ripple/app/tx/apply.h>
#include <ripple/basics/Log.h>
#include <ripple/basics/MathUtilities.h>
Expand Down Expand Up @@ -278,10 +277,10 @@ LedgerMaster::getValidatedLedgerAge()

#ifdef RIPPLED_REPORTING
if (app_.config().reporting())
return static_cast<RelationalDBInterfacePostgres*>(
&app_.getRelationalDBInterface())
return static_cast<PostgresDatabase*>(&app_.getRelationalDatabase())
->getValidatedLedgerAge();
#endif

std::chrono::seconds valClose{mValidLedgerSign.load()};
if (valClose == 0s)
{
Expand Down Expand Up @@ -309,8 +308,7 @@ LedgerMaster::isCaughtUp(std::string& reason)

#ifdef RIPPLED_REPORTING
if (app_.config().reporting())
return static_cast<RelationalDBInterfacePostgres*>(
&app_.getRelationalDBInterface())
return static_cast<PostgresDatabase*>(&app_.getRelationalDatabase())
->isCaughtUp(reason);
#endif

Expand Down Expand Up @@ -743,7 +741,7 @@ LedgerMaster::tryFill(std::shared_ptr<Ledger const> ledger)
mCompleteLedgers.insert(range(minHas, maxHas));
}
maxHas = minHas;
ledgerHashes = app_.getRelationalDBInterface().getHashesByIndex(
ledgerHashes = app_.getRelationalDatabase().getHashesByIndex(
(seq < 500) ? 0 : (seq - 499), seq);
it = ledgerHashes.find(seq);

Expand Down Expand Up @@ -927,8 +925,8 @@ LedgerMaster::setFullLedger(
{
// Check the SQL database's entry for the sequence before this
// ledger, if it's not this ledger's parent, invalidate it
uint256 prevHash = app_.getRelationalDBInterface().getHashByIndex(
ledger->info().seq - 1);
uint256 prevHash =
app_.getRelationalDatabase().getHashByIndex(ledger->info().seq - 1);
if (prevHash.isNonZero() && prevHash != ledger->info().parentHash)
clearLedger(ledger->info().seq - 1);
}
Expand Down Expand Up @@ -1664,7 +1662,7 @@ LedgerMaster::getValidatedLedger()
#ifdef RIPPLED_REPORTING
if (app_.config().reporting())
{
auto seq = app_.getRelationalDBInterface().getMaxLedgerSeq();
auto seq = app_.getRelationalDatabase().getMaxLedgerSeq();
if (!seq)
return {};
return getLedgerBySeq(*seq);
Expand Down Expand Up @@ -1700,8 +1698,7 @@ LedgerMaster::getCompleteLedgers()
{
#ifdef RIPPLED_REPORTING
if (app_.config().reporting())
return static_cast<RelationalDBInterfacePostgres*>(
&app_.getRelationalDBInterface())
return static_cast<PostgresDatabase*>(&app_.getRelationalDatabase())
->getCompleteLedgers();
#endif
std::lock_guard sl(mCompleteLock);
Expand Down Expand Up @@ -1746,7 +1743,7 @@ LedgerMaster::getHashBySeq(std::uint32_t index)
if (hash.isNonZero())
return hash;

return app_.getRelationalDBInterface().getHashByIndex(index);
return app_.getRelationalDatabase().getHashByIndex(index);
}

std::optional<LedgerHash>
Expand Down Expand Up @@ -1967,7 +1964,7 @@ LedgerMaster::fetchForHistory(
fillInProgress = mFillInProgress;
}
if (fillInProgress == 0 &&
app_.getRelationalDBInterface().getHashByIndex(seq - 1) ==
app_.getRelationalDatabase().getHashByIndex(seq - 1) ==
ledger->info().parentHash)
{
{
Expand Down Expand Up @@ -2363,7 +2360,7 @@ LedgerMaster::getFetchPackCacheSize() const
std::optional<LedgerIndex>
LedgerMaster::minSqlSeq()
{
return app_.getRelationalDBInterface().getMinLedgerSeq();
return app_.getRelationalDatabase().getMinLedgerSeq();
}

} // namespace ripple
32 changes: 15 additions & 17 deletions src/ripple/app/main/Application.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@
#include <ripple/app/misc/ValidatorKeys.h>
#include <ripple/app/misc/ValidatorSite.h>
#include <ripple/app/paths/PathRequests.h>
#include <ripple/app/rdb/RelationalDBInterface_global.h>
#include <ripple/app/rdb/backend/RelationalDBInterfacePostgres.h>
#include <ripple/app/rdb/Wallet.h>
#include <ripple/app/rdb/backend/PostgresDatabase.h>
#include <ripple/app/reporting/ReportingETL.h>
#include <ripple/app/tx/apply.h>
#include <ripple/basics/ByteUtilities.h>
Expand Down Expand Up @@ -219,7 +219,7 @@ class ApplicationImp : public Application, public BasicApp
boost::asio::steady_timer sweepTimer_;
boost::asio::steady_timer entropyTimer_;

std::unique_ptr<RelationalDBInterface> mRelationalDBInterface;
std::unique_ptr<RelationalDatabase> mRelationalDatabase;
std::unique_ptr<DatabaseCon> mWalletDB;
std::unique_ptr<Overlay> overlay_;

Expand Down Expand Up @@ -877,11 +877,11 @@ class ApplicationImp : public Application, public BasicApp
return *txQ_;
}

RelationalDBInterface&
getRelationalDBInterface() override
RelationalDatabase&
getRelationalDatabase() override
{
assert(mRelationalDBInterface.get() != nullptr);
return *mRelationalDBInterface;
assert(mRelationalDatabase.get() != nullptr);
return *mRelationalDatabase;
}

DatabaseCon&
Expand All @@ -907,14 +907,14 @@ class ApplicationImp : public Application, public BasicApp
//--------------------------------------------------------------------------

bool
initRDBMS()
initRelationalDatabase()
{
assert(mWalletDB.get() == nullptr);

try
{
mRelationalDBInterface =
RelationalDBInterface::init(*this, *config_, *m_jobQueue);
mRelationalDatabase =
RelationalDatabase::init(*this, *config_, *m_jobQueue);

// wallet database
auto setup = setup_DatabaseCon(*config_, m_journal);
Expand Down Expand Up @@ -1041,7 +1041,7 @@ class ApplicationImp : public Application, public BasicApp
doSweep()
{
if (!config_->standalone() &&
!getRelationalDBInterface().transactionDbHasSpace(*config_))
!getRelationalDatabase().transactionDbHasSpace(*config_))
{
signalStop();
}
Expand All @@ -1066,8 +1066,7 @@ class ApplicationImp : public Application, public BasicApp
cachedSLEs_.sweep();

#ifdef RIPPLED_REPORTING
if (auto pg = dynamic_cast<RelationalDBInterfacePostgres*>(
&*mRelationalDBInterface))
if (auto pg = dynamic_cast<PostgresDatabase*>(&*mRelationalDatabase))
pg->sweep();
#endif

Expand Down Expand Up @@ -1162,7 +1161,7 @@ ApplicationImp::setup()
if (!config_->standalone())
timeKeeper_->run(config_->SNTP_SERVERS);

if (!initRDBMS() || !initNodeStore())
if (!initRelationalDatabase() || !initNodeStore())
return false;

if (shardStore_)
Expand Down Expand Up @@ -1619,8 +1618,7 @@ ApplicationImp::run()
ledgerCleaner_->stop();
if (reportingETL_)
reportingETL_->stop();
if (auto pg = dynamic_cast<RelationalDBInterfacePostgres*>(
&*mRelationalDBInterface))
if (auto pg = dynamic_cast<PostgresDatabase*>(&*mRelationalDatabase))
pg->stop();
m_nodeStore->stop();
perfLog_->stop();
Expand Down Expand Up @@ -2137,7 +2135,7 @@ ApplicationImp::nodeToShards()
void
ApplicationImp::setMaxDisallowedLedger()
{
auto seq = getRelationalDBInterface().getMaxLedgerSeq();
auto seq = getRelationalDatabase().getMaxLedgerSeq();
if (seq)
maxDisallowedLedger_ = *seq;

Expand Down
6 changes: 3 additions & 3 deletions src/ripple/app/main/Application.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ class ValidatorList;
class ValidatorSite;
class Cluster;

class RelationalDBInterface;
class RelationalDatabase;
class DatabaseCon;
class SHAMapStore;

Expand Down Expand Up @@ -251,8 +251,8 @@ class Application : public beast::PropertyStream::Source
openLedger() = 0;
virtual OpenLedger const&
openLedger() const = 0;
virtual RelationalDBInterface&
getRelationalDBInterface() = 0;
virtual RelationalDatabase&
getRelationalDatabase() = 0;

virtual std::chrono::milliseconds
getIOLatency() = 0;
Expand Down
36 changes: 33 additions & 3 deletions src/ripple/app/main/DBInit.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,22 @@ inline constexpr std::array<char const*, 5> LgrDBInit{
// Transaction database holds transactions and public keys
inline constexpr auto TxDBName{"transaction.db"};

inline constexpr std::array TxDBPragma
// In C++17 omitting the explicit template parameters caused
// a crash
inline constexpr std::array<char const*, 4> TxDBPragma
{
"PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;",
"PRAGMA max_page_count=2147483646;",

#if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP)
"PRAGMA mmap_size=17179869184;"
#else

// Provide an explicit `no-op` SQL statement
// in order to keep the size of the array
// constant regardless of the preprocessor
// condition evaluation
"PRAGMA sqlite_noop_statement;"
#endif
};

Expand Down Expand Up @@ -117,12 +127,22 @@ inline constexpr std::array<char const*, 8> TxDBInit{
// The Ledger Meta database maps ledger hashes to shard indexes
inline constexpr auto LgrMetaDBName{"ledger_meta.db"};

inline constexpr std::array LgrMetaDBPragma
// In C++17 omitting the explicit template parameters caused
// a crash
inline constexpr std::array<char const*, 4> LgrMetaDBPragma
{
"PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;",
"PRAGMA max_page_count=2147483646;",

#if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP)
"PRAGMA mmap_size=17179869184;"
#else

// Provide an explicit `no-op` SQL statement
// in order to keep the size of the array
// constant regardless of the preprocessor
// condition evaluation
"PRAGMA sqlite_noop_statement;"
#endif
};

Expand All @@ -141,12 +161,22 @@ inline constexpr std::array<char const*, 3> LgrMetaDBInit{
// Transaction Meta database maps transaction IDs to shard indexes
inline constexpr auto TxMetaDBName{"transaction_meta.db"};

inline constexpr std::array TxMetaDBPragma
// In C++17 omitting the explicit template parameters caused
// a crash
inline constexpr std::array<char const*, 4> TxMetaDBPragma
{
"PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;",
"PRAGMA max_page_count=2147483646;",

#if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP)
"PRAGMA mmap_size=17179869184;"
#else

// Provide an explicit `no-op` SQL statement
// in order to keep the size of the array
// constant regardless of the preprocessor
// condition evaluation
"PRAGMA sqlite_noop_statement;"
#endif
};

Expand Down
Loading

1 comment on commit dac080f

@intelliot
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#3965

Please sign in to comment.