diff --git a/Builds/CMake/deps/Nudb.cmake b/Builds/CMake/deps/Nudb.cmake index 750b940bfd5..8bdb0c06f89 100644 --- a/Builds/CMake/deps/Nudb.cmake +++ b/Builds/CMake/deps/Nudb.cmake @@ -12,7 +12,7 @@ if (is_root_project) # NuDB not needed in the case of xrpl_core inclusion build FetchContent_Declare( nudb_src GIT_REPOSITORY https://github.com/CPPAlliance/NuDB.git - GIT_TAG 2.0.3 + GIT_TAG 2.0.5 ) FetchContent_GetProperties(nudb_src) if(NOT nudb_src_POPULATED) @@ -23,7 +23,7 @@ if (is_root_project) # NuDB not needed in the case of xrpl_core inclusion build ExternalProject_Add (nudb_src PREFIX ${nih_cache_path} GIT_REPOSITORY https://github.com/CPPAlliance/NuDB.git - GIT_TAG 2.0.3 + GIT_TAG 2.0.5 CONFIGURE_COMMAND "" BUILD_COMMAND "" TEST_COMMAND "" diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index b9acd63e748..de1ae696baf 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -940,6 +940,8 @@ class ApplicationImp : public Application, public RootStoppable, public BasicApp std::unique_ptr source = NodeStore::Manager::instance().make_Database( "NodeStore.import", + megabytes( + config_->getValueFor(SizedItem::burstSize, boost::none)), dummyScheduler, 0, dummyRoot, diff --git a/src/ripple/app/misc/SHAMapStoreImp.cpp b/src/ripple/app/misc/SHAMapStoreImp.cpp index 642593be59d..ef8ce21f365 100644 --- a/src/ripple/app/misc/SHAMapStoreImp.cpp +++ b/src/ripple/app/misc/SHAMapStoreImp.cpp @@ -259,6 +259,8 @@ SHAMapStoreImp::makeNodeStore(std::string const& name, std::int32_t readThreads) { db = NodeStore::Manager::instance().make_Database( name, + megabytes( + app_.config().getValueFor(SizedItem::burstSize, boost::none)), scheduler_, readThreads, app_.getJobQueue(), @@ -562,7 +564,10 @@ SHAMapStoreImp::makeBackendRotating(std::string path) section.set("path", newPath.string()); auto backend{NodeStore::Manager::instance().make_Backend( - section, scheduler_, app_.logs().journal(nodeStoreName_))}; + section, + megabytes(app_.config().getValueFor(SizedItem::burstSize, boost::none)), + scheduler_, + app_.logs().journal(nodeStoreName_))}; backend->open(); return backend; } diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index 2915f1415e8..f66bf3b8bda 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -57,7 +57,8 @@ enum class SizedItem : std::size_t { hashNodeDBCache, txnDBCache, lgrDBCache, - openFinalLimit + openFinalLimit, + burstSize }; // This entire derived class is deprecated. diff --git a/src/ripple/core/impl/Config.cpp b/src/ripple/core/impl/Config.cpp index b42947b5376..fe0da432ace 100644 --- a/src/ripple/core/impl/Config.cpp +++ b/src/ripple/core/impl/Config.cpp @@ -40,25 +40,27 @@ namespace ripple { -inline constexpr std::array>, 12> - sizedItems{ - {// FIXME: We should document each of these items, explaining exactly - // what - // they control and whether there exists an explicit config - // option that can be used to override the default. - {SizedItem::sweepInterval, {{10, 30, 60, 90, 120}}}, - {SizedItem::treeCacheSize, - {{128000, 256000, 512000, 768000, 2048000}}}, - {SizedItem::treeCacheAge, {{30, 60, 90, 120, 900}}}, - {SizedItem::ledgerSize, {{32, 128, 256, 384, 768}}}, - {SizedItem::ledgerAge, {{30, 90, 180, 240, 900}}}, - {SizedItem::ledgerFetch, {{2, 3, 4, 5, 8}}}, - {SizedItem::nodeCacheSize, {{16384, 32768, 131072, 262144, 524288}}}, - {SizedItem::nodeCacheAge, {{60, 90, 120, 900, 1800}}}, - {SizedItem::hashNodeDBCache, {{4, 12, 24, 64, 128}}}, - {SizedItem::txnDBCache, {{4, 12, 24, 64, 128}}}, - {SizedItem::lgrDBCache, {{4, 8, 16, 32, 128}}}, - {SizedItem::openFinalLimit, {{8, 16, 32, 64, 128}}}}}; +// The configurable node sizes are "tiny", "small", "medium", "large", "huge" +inline constexpr std::array>, 13> + sizedItems{{ + // FIXME: We should document each of these items, explaining exactly + // what + // they control and whether there exists an explicit config + // option that can be used to override the default. + {SizedItem::sweepInterval, {{10, 30, 60, 90, 120}}}, + {SizedItem::treeCacheSize, {{128000, 256000, 512000, 768000, 2048000}}}, + {SizedItem::treeCacheAge, {{30, 60, 90, 120, 900}}}, + {SizedItem::ledgerSize, {{32, 128, 256, 384, 768}}}, + {SizedItem::ledgerAge, {{30, 90, 180, 240, 900}}}, + {SizedItem::ledgerFetch, {{2, 3, 4, 5, 8}}}, + {SizedItem::nodeCacheSize, {{16384, 32768, 131072, 262144, 524288}}}, + {SizedItem::nodeCacheAge, {{60, 90, 120, 900, 1800}}}, + {SizedItem::hashNodeDBCache, {{4, 12, 24, 64, 128}}}, + {SizedItem::txnDBCache, {{4, 12, 24, 64, 128}}}, + {SizedItem::lgrDBCache, {{4, 8, 16, 32, 128}}}, + {SizedItem::openFinalLimit, {{8, 16, 32, 64, 128}}}, + {SizedItem::burstSize, {{4, 8, 16, 32, 48}}}, + }}; // Ensure that the order of entries in the table corresponds to the // order of entries in the enum: diff --git a/src/ripple/nodestore/Factory.h b/src/ripple/nodestore/Factory.h index eabad965f44..c34113eac1b 100644 --- a/src/ripple/nodestore/Factory.h +++ b/src/ripple/nodestore/Factory.h @@ -42,6 +42,7 @@ class Factory @param keyBytes The fixed number of bytes per key. @param parameters A set of key/value configuration pairs. + @param burstSize Backend burst size in bytes. @param scheduler The scheduler to use for running tasks. @return A pointer to the Backend object. */ @@ -49,6 +50,7 @@ class Factory createInstance( size_t keyBytes, Section const& parameters, + std::size_t burstSize, Scheduler& scheduler, beast::Journal journal) = 0; @@ -56,6 +58,7 @@ class Factory @param keyBytes The fixed number of bytes per key. @param parameters A set of key/value configuration pairs. + @param burstSize Backend burst size in bytes. @param scheduler The scheduler to use for running tasks. @param context The context used by database. @return A pointer to the Backend object. @@ -64,6 +67,7 @@ class Factory createInstance( size_t keyBytes, Section const& parameters, + std::size_t burstSize, Scheduler& scheduler, nudb::context& context, beast::Journal journal) diff --git a/src/ripple/nodestore/Manager.h b/src/ripple/nodestore/Manager.h index 60350e4d48c..1b51b9edd4a 100644 --- a/src/ripple/nodestore/Manager.h +++ b/src/ripple/nodestore/Manager.h @@ -60,6 +60,7 @@ class Manager virtual std::unique_ptr make_Backend( Section const& parameters, + std::size_t burstSize, Scheduler& scheduler, beast::Journal journal) = 0; @@ -81,6 +82,7 @@ class Manager thrown. @param name A diagnostic label for the database. + @param burstSize Backend burst size in bytes. @param scheduler The scheduler to use for performing asynchronous tasks. @param readThreads The number of async read threads to create @param backendParameters The parameter string for the persistent @@ -93,6 +95,7 @@ class Manager virtual std::unique_ptr make_Database( std::string const& name, + std::size_t burstSize, Scheduler& scheduler, int readThreads, Stoppable& parent, @@ -106,6 +109,7 @@ class Manager std::unique_ptr make_Backend( Section const& config, + std::size_t burstSize, Scheduler& scheduler, beast::Journal journal); diff --git a/src/ripple/nodestore/backend/MemoryFactory.cpp b/src/ripple/nodestore/backend/MemoryFactory.cpp index e3b9937b006..411f44805fa 100644 --- a/src/ripple/nodestore/backend/MemoryFactory.cpp +++ b/src/ripple/nodestore/backend/MemoryFactory.cpp @@ -55,6 +55,7 @@ class MemoryFactory : public Factory createInstance( size_t keyBytes, Section const& keyValues, + std::size_t burstSize, Scheduler& scheduler, beast::Journal journal) override; @@ -227,6 +228,7 @@ std::unique_ptr MemoryFactory::createInstance( size_t keyBytes, Section const& keyValues, + std::size_t, Scheduler& scheduler, beast::Journal journal) { diff --git a/src/ripple/nodestore/backend/NuDBFactory.cpp b/src/ripple/nodestore/backend/NuDBFactory.cpp index 09563410921..7a375e3e361 100644 --- a/src/ripple/nodestore/backend/NuDBFactory.cpp +++ b/src/ripple/nodestore/backend/NuDBFactory.cpp @@ -42,6 +42,7 @@ class NuDBBackend : public Backend beast::Journal const j_; size_t const keyBytes_; + std::size_t const burstSize_; std::string const name_; nudb::store db_; std::atomic deletePath_; @@ -50,10 +51,12 @@ class NuDBBackend : public Backend NuDBBackend( size_t keyBytes, Section const& keyValues, + std::size_t burstSize, Scheduler& scheduler, beast::Journal journal) : j_(journal) , keyBytes_(keyBytes) + , burstSize_(burstSize) , name_(get(keyValues, "path")) , deletePath_(false) , scheduler_(scheduler) @@ -66,11 +69,13 @@ class NuDBBackend : public Backend NuDBBackend( size_t keyBytes, Section const& keyValues, + std::size_t burstSize, Scheduler& scheduler, nudb::context& context, beast::Journal journal) : j_(journal) , keyBytes_(keyBytes) + , burstSize_(burstSize) , name_(get(keyValues, "path")) , db_(context) , deletePath_(false) @@ -130,6 +135,7 @@ class NuDBBackend : public Backend Throw(ec); if (db_.appnum() != currentType) Throw("nodestore: unknown appnum"); + db_.set_burst(burstSize_); } bool @@ -333,23 +339,25 @@ class NuDBFactory : public Factory createInstance( size_t keyBytes, Section const& keyValues, + std::size_t burstSize, Scheduler& scheduler, beast::Journal journal) override { return std::make_unique( - keyBytes, keyValues, scheduler, journal); + keyBytes, keyValues, burstSize, scheduler, journal); } std::unique_ptr createInstance( size_t keyBytes, Section const& keyValues, + std::size_t burstSize, Scheduler& scheduler, nudb::context& context, beast::Journal journal) override { return std::make_unique( - keyBytes, keyValues, scheduler, context, journal); + keyBytes, keyValues, burstSize, scheduler, context, journal); } }; diff --git a/src/ripple/nodestore/backend/NullFactory.cpp b/src/ripple/nodestore/backend/NullFactory.cpp index eddce54a7d3..6cd1503f760 100644 --- a/src/ripple/nodestore/backend/NullFactory.cpp +++ b/src/ripple/nodestore/backend/NullFactory.cpp @@ -136,7 +136,12 @@ class NullFactory : public Factory } std::unique_ptr - createInstance(size_t, Section const&, Scheduler&, beast::Journal) override + createInstance( + size_t, + Section const&, + std::size_t, + Scheduler&, + beast::Journal) override { return std::make_unique(); } diff --git a/src/ripple/nodestore/backend/RocksDBFactory.cpp b/src/ripple/nodestore/backend/RocksDBFactory.cpp index b5f5627a9b3..4fc36b7e1b7 100644 --- a/src/ripple/nodestore/backend/RocksDBFactory.cpp +++ b/src/ripple/nodestore/backend/RocksDBFactory.cpp @@ -450,6 +450,7 @@ class RocksDBFactory : public Factory createInstance( size_t keyBytes, Section const& keyValues, + std::size_t, Scheduler& scheduler, beast::Journal journal) override { diff --git a/src/ripple/nodestore/impl/ManagerImp.cpp b/src/ripple/nodestore/impl/ManagerImp.cpp index 30c40fedcf4..ff5bc69938b 100644 --- a/src/ripple/nodestore/impl/ManagerImp.cpp +++ b/src/ripple/nodestore/impl/ManagerImp.cpp @@ -43,6 +43,7 @@ ManagerImp::missing_backend() std::unique_ptr ManagerImp::make_Backend( Section const& parameters, + std::size_t burstSize, Scheduler& scheduler, beast::Journal journal) { @@ -55,19 +56,20 @@ ManagerImp::make_Backend( missing_backend(); return factory->createInstance( - NodeObject::keyBytes, parameters, scheduler, journal); + NodeObject::keyBytes, parameters, burstSize, scheduler, journal); } std::unique_ptr ManagerImp::make_Database( std::string const& name, + std::size_t burstSize, Scheduler& scheduler, int readThreads, Stoppable& parent, Section const& config, beast::Journal journal) { - auto backend{make_Backend(config, scheduler, journal)}; + auto backend{make_Backend(config, burstSize, scheduler, journal)}; backend->open(); return std::make_unique( name, @@ -124,10 +126,12 @@ Manager::instance() std::unique_ptr make_Backend( Section const& config, + std::size_t burstSize, Scheduler& scheduler, beast::Journal journal) { - return Manager::instance().make_Backend(config, scheduler, journal); + return Manager::instance().make_Backend( + config, burstSize, scheduler, journal); } } // namespace NodeStore diff --git a/src/ripple/nodestore/impl/ManagerImp.h b/src/ripple/nodestore/impl/ManagerImp.h index 585e2720a20..5e2234529dd 100644 --- a/src/ripple/nodestore/impl/ManagerImp.h +++ b/src/ripple/nodestore/impl/ManagerImp.h @@ -54,12 +54,14 @@ class ManagerImp : public Manager std::unique_ptr make_Backend( Section const& parameters, + std::size_t burstSize, Scheduler& scheduler, beast::Journal journal) override; std::unique_ptr make_Database( std::string const& name, + std::size_t burstSize, Scheduler& scheduler, int readThreads, Stoppable& parent, diff --git a/src/ripple/nodestore/impl/Shard.cpp b/src/ripple/nodestore/impl/Shard.cpp index ae7e2c5f06f..a7f49bebe39 100644 --- a/src/ripple/nodestore/impl/Shard.cpp +++ b/src/ripple/nodestore/impl/Shard.cpp @@ -115,7 +115,12 @@ Shard::init(Scheduler& scheduler, nudb::context& context) return false; } backend_ = factory->createInstance( - NodeObject::keyBytes, section, scheduler, context, j_); + NodeObject::keyBytes, + section, + megabytes(app_.config().getValueFor(SizedItem::burstSize, boost::none)), + scheduler, + context, + j_); return open(lock); } diff --git a/src/test/nodestore/Backend_test.cpp b/src/test/nodestore/Backend_test.cpp index 23afaad2af3..674f60c07fb 100644 --- a/src/test/nodestore/Backend_test.cpp +++ b/src/test/nodestore/Backend_test.cpp @@ -58,8 +58,8 @@ class Backend_test : public TestBase { // Open the backend - std::unique_ptr backend = - Manager::instance().make_Backend(params, scheduler, journal); + std::unique_ptr backend = Manager::instance().make_Backend( + params, megabytes(4), scheduler, journal); backend->open(); // Write the batch @@ -83,8 +83,8 @@ class Backend_test : public TestBase { // Re-open the backend - std::unique_ptr backend = - Manager::instance().make_Backend(params, scheduler, journal); + std::unique_ptr backend = Manager::instance().make_Backend( + params, megabytes(4), scheduler, journal); backend->open(); // Read it back in diff --git a/src/test/nodestore/Database_test.cpp b/src/test/nodestore/Database_test.cpp index 826f5ccf5bf..2793f23d95b 100644 --- a/src/test/nodestore/Database_test.cpp +++ b/src/test/nodestore/Database_test.cpp @@ -462,7 +462,13 @@ class Database_test : public TestBase // Write to source db { std::unique_ptr src = Manager::instance().make_Database( - "test", scheduler, 2, parent, srcParams, journal_); + "test", + megabytes(4), + scheduler, + 2, + parent, + srcParams, + journal_); storeBatch(*src, batch); } @@ -471,7 +477,13 @@ class Database_test : public TestBase { // Re-open the db std::unique_ptr src = Manager::instance().make_Database( - "test", scheduler, 2, parent, srcParams, journal_); + "test", + megabytes(4), + scheduler, + 2, + parent, + srcParams, + journal_); // Set up the destination database beast::temp_dir dest_db; @@ -480,7 +492,13 @@ class Database_test : public TestBase destParams.set("path", dest_db.path()); std::unique_ptr dest = Manager::instance().make_Database( - "test", scheduler, 2, parent, destParams, journal_); + "test", + megabytes(4), + scheduler, + 2, + parent, + destParams, + journal_); testcase( "import into '" + destBackendType + "' from '" + @@ -528,7 +546,13 @@ class Database_test : public TestBase { // Open the database std::unique_ptr db = Manager::instance().make_Database( - "test", scheduler, 2, parent, nodeParams, journal_); + "test", + megabytes(4), + scheduler, + 2, + parent, + nodeParams, + journal_); // Write the batch storeBatch(*db, batch); @@ -553,7 +577,13 @@ class Database_test : public TestBase { // Re-open the database without the ephemeral DB std::unique_ptr db = Manager::instance().make_Database( - "test", scheduler, 2, parent, nodeParams, journal_); + "test", + megabytes(4), + scheduler, + 2, + parent, + nodeParams, + journal_); // Read it back in Batch copy; @@ -572,7 +602,13 @@ class Database_test : public TestBase // Verify default earliest ledger sequence std::unique_ptr db = Manager::instance().make_Database( - "test", scheduler, 2, parent, nodeParams, journal_); + "test", + megabytes(4), + scheduler, + 2, + parent, + nodeParams, + journal_); BEAST_EXPECT( db->earliestLedgerSeq() == XRP_LEDGER_EARLIEST_SEQ); } @@ -583,7 +619,13 @@ class Database_test : public TestBase nodeParams.set("earliest_seq", "0"); std::unique_ptr db = Manager::instance().make_Database( - "test", scheduler, 2, parent, nodeParams, journal_); + "test", + megabytes(4), + scheduler, + 2, + parent, + nodeParams, + journal_); } catch (std::runtime_error const& e) { @@ -596,7 +638,13 @@ class Database_test : public TestBase nodeParams.set("earliest_seq", "1"); std::unique_ptr db = Manager::instance().make_Database( - "test", scheduler, 2, parent, nodeParams, journal_); + "test", + megabytes(4), + scheduler, + 2, + parent, + nodeParams, + journal_); // Verify database uses the earliest ledger sequence setting BEAST_EXPECT(db->earliestLedgerSeq() == 1); @@ -610,7 +658,13 @@ class Database_test : public TestBase "earliest_seq", std::to_string(XRP_LEDGER_EARLIEST_SEQ)); std::unique_ptr db2 = Manager::instance().make_Database( - "test", scheduler, 2, parent, nodeParams, journal_); + "test", + megabytes(4), + scheduler, + 2, + parent, + nodeParams, + journal_); } catch (std::runtime_error const& e) { diff --git a/src/test/nodestore/Timing_test.cpp b/src/test/nodestore/Timing_test.cpp index 59b9aa47f0e..c3dfd79b6fc 100644 --- a/src/test/nodestore/Timing_test.cpp +++ b/src/test/nodestore/Timing_test.cpp @@ -261,7 +261,7 @@ class Timing_test : public beast::unit_test::suite beast::Journal journal) { DummyScheduler scheduler; - auto backend = make_Backend(config, scheduler, journal); + auto backend = make_Backend(config, megabytes(4), scheduler, journal); BEAST_EXPECT(backend != nullptr); backend->open(); @@ -318,7 +318,7 @@ class Timing_test : public beast::unit_test::suite beast::Journal journal) { DummyScheduler scheduler; - auto backend = make_Backend(config, scheduler, journal); + auto backend = make_Backend(config, megabytes(4), scheduler, journal); BEAST_EXPECT(backend != nullptr); backend->open(); @@ -389,7 +389,7 @@ class Timing_test : public beast::unit_test::suite beast::Journal journal) { DummyScheduler scheduler; - auto backend = make_Backend(config, scheduler, journal); + auto backend = make_Backend(config, megabytes(4), scheduler, journal); BEAST_EXPECT(backend != nullptr); backend->open(); @@ -462,7 +462,7 @@ class Timing_test : public beast::unit_test::suite beast::Journal journal) { DummyScheduler scheduler; - auto backend = make_Backend(config, scheduler, journal); + auto backend = make_Backend(config, megabytes(4), scheduler, journal); BEAST_EXPECT(backend != nullptr); backend->open(); @@ -551,7 +551,7 @@ class Timing_test : public beast::unit_test::suite do_work(Section const& config, Params const& params, beast::Journal journal) { DummyScheduler scheduler; - auto backend = make_Backend(config, scheduler, journal); + auto backend = make_Backend(config, megabytes(4), scheduler, journal); BEAST_EXPECT(backend != nullptr); backend->setDeletePath(); backend->open(); diff --git a/src/test/shamap/common.h b/src/test/shamap/common.h index 760f8ca4e82..7a13bb81405 100644 --- a/src/test/shamap/common.h +++ b/src/test/shamap/common.h @@ -61,7 +61,7 @@ class TestNodeFamily : public Family testSection.set("type", "memory"); testSection.set("Path", "SHAMap_test"); db_ = NodeStore::Manager::instance().make_Database( - "test", scheduler_, 1, parent_, testSection, j); + "test", megabytes(4), scheduler_, 1, parent_, testSection, j); } NodeStore::Database&