diff --git a/cfg/rippled-example.cfg b/cfg/rippled-example.cfg index 32b30f3ef04..43982b9a951 100644 --- a/cfg/rippled-example.cfg +++ b/cfg/rippled-example.cfg @@ -1052,7 +1052,7 @@ # [crawl] # # List of options to control what data is reported through the /crawl endpoint -# See https://developers.ripple.com/peer-protocol.html#peer-crawler +# See https://xrpl.org/peer-crawler.html # # # @@ -1093,6 +1093,16 @@ # counts = 0 # unl = 1 # +# [vl] +# +# Options to control what data is reported through the /vl endpoint +# See [...] +# +# enable = +# +# Enable or disable access to /vl requests. Default is '1' which +# enables access. +# #------------------------------------------------------------------------------- # # 9. Example Settings diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index 3e8bb2a52d0..fcb0961ade0 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -365,7 +365,7 @@ class ApplicationImp std::unique_ptr serverHandler_; std::unique_ptr m_amendmentTable; std::unique_ptr mFeeTrack; - std::unique_ptr mHashRouter; + std::unique_ptr hashRouter_; RCLValidations mValidations; std::unique_ptr m_loadManager; std::unique_ptr txQ_; @@ -377,7 +377,7 @@ class ApplicationImp std::unique_ptr mTxnDB; std::unique_ptr mLedgerDB; std::unique_ptr mWalletDB; - std::unique_ptr m_overlay; + std::unique_ptr overlay_; std::vector > websocketServers_; boost::asio::signal_set m_signals; @@ -520,7 +520,8 @@ class ApplicationImp logs_->journal("ManifestCache"))) , validators_ (std::make_unique ( - *validatorManifests_, *publisherManifests_, *timeKeeper_, + *validatorManifests_, *publisherManifests_, + *timeKeeper_, config_->legacy("database_path"), logs_->journal("ValidatorList"), config_->VALIDATION_QUORUM)) , validatorSites_ (std::make_unique (*this)) @@ -531,7 +532,7 @@ class ApplicationImp , mFeeTrack (std::make_unique(logs_->journal("LoadManager"))) - , mHashRouter (std::make_unique( + , hashRouter_ (std::make_unique( stopwatch(), HashRouter::getDefaultHoldTime (), HashRouter::getDefaultRecoverLimit ())) @@ -760,7 +761,7 @@ class ApplicationImp HashRouter& getHashRouter () override { - return *mHashRouter; + return *hashRouter_; } RCLValidations& getValidations () override @@ -828,7 +829,8 @@ class ApplicationImp Overlay& overlay () override { - return *m_overlay; + assert(overlay_); + return *overlay_; } TxQ& getTxQ() override @@ -1480,10 +1482,10 @@ bool ApplicationImp::setup() // move the instantiation inside a conditional: // // if (!config_.standalone()) - m_overlay = make_Overlay (*this, setup_Overlay(*config_), *m_jobQueue, + overlay_ = make_Overlay (*this, setup_Overlay(*config_), *m_jobQueue, *serverHandler_, *m_resourceManager, *m_resolver, get_io_service(), *config_, m_collectorManager->collector ()); - add (*m_overlay); // add to PropertyStream + add (*overlay_); // add to PropertyStream if (!config_->standalone()) { @@ -1675,7 +1677,7 @@ int ApplicationImp::fdRequired() const int needed = 128; // 1.5 times the configured peer limit for peer connections: - needed += static_cast(0.5 + (1.5 * m_overlay->limit())); + needed += static_cast(0.5 + (1.5 * overlay_->limit())); // the number of fds needed by the backend (internally // doubled if online delete is enabled). @@ -2131,7 +2133,7 @@ ApplicationImp::journal (std::string const& name) bool ApplicationImp::nodeToShards() { - assert(m_overlay); + assert(overlay_); assert(!config_->standalone()); if (config_->section(ConfigSection::shardDatabase()).empty()) @@ -2152,7 +2154,7 @@ bool ApplicationImp::nodeToShards() bool ApplicationImp::validateShards() { - assert(m_overlay); + assert(overlay_); assert(!config_->standalone()); if (config_->section(ConfigSection::shardDatabase()).empty()) diff --git a/src/ripple/app/misc/HashRouter.h b/src/ripple/app/misc/HashRouter.h index f0b92d460f6..aa44e1f5c89 100644 --- a/src/ripple/app/misc/HashRouter.h +++ b/src/ripple/app/misc/HashRouter.h @@ -197,8 +197,10 @@ class HashRouter boost::optional> shouldRelay(uint256 const& key); /** Determines whether the hashed item should be recovered + from the open ledger into the next open ledger or the transaction + queue. - @return `bool` indicates whether the item should be relayed + @return `bool` indicates whether the item should be recovered */ bool shouldRecover(uint256 const& key); diff --git a/src/ripple/app/misc/ValidatorList.h b/src/ripple/app/misc/ValidatorList.h index ad24c3e608c..4512fa0999e 100644 --- a/src/ripple/app/misc/ValidatorList.h +++ b/src/ripple/app/misc/ValidatorList.h @@ -35,6 +35,10 @@ namespace ripple { +// predeclaration +class Overlay; +class HashRouter; + enum class ListDisposition { /// List is valid @@ -123,11 +127,17 @@ class ValidatorList std::size_t sequence; TimeKeeper::time_point expiration; std::string siteUri; + std::string rawManifest; + std::string rawBlob; + std::string rawSignature; + std::uint32_t rawVersion; + uint256 hash; }; ManifestCache& validatorManifests_; ManifestCache& publisherManifests_; TimeKeeper& timeKeeper_; + boost::filesystem::path const dataPath_; beast::Journal const j_; std::shared_timed_mutex mutable mutex_; @@ -147,16 +157,45 @@ class ValidatorList // Currently supported version of publisher list format static constexpr std::uint32_t requiredListVersion = 1; + static const std::string filePrefix_; public: ValidatorList ( ManifestCache& validatorManifests, ManifestCache& publisherManifests, TimeKeeper& timeKeeper, + std::string const& databasePath, beast::Journal j, boost::optional minimumQuorum = boost::none); ~ValidatorList () = default; + /** Describes the result of processing a Validator List (UNL), + including some of the information from the list which can + be used by the caller to know which list publisher is + involved. + */ + struct PublisherListStats + { + explicit PublisherListStats(ListDisposition d) + : disposition(d) + { + } + + PublisherListStats(ListDisposition d, PublicKey key, + bool avail, std::size_t seq) + : disposition(d) + , publisherKey(key) + , available(avail) + , sequence(seq) + { + } + + ListDisposition disposition; + boost::optional publisherKey; + bool available = false; + boost::optional sequence; + }; + /** Load configured trusted keys. @param localSigningKey This node's validation public key @@ -180,6 +219,44 @@ class ValidatorList std::vector const& configKeys, std::vector const& publisherKeys); + /** Apply published list of public keys, then broadcast it to all + peers that have not seen it or sent it. + + @param manifest base64-encoded publisher key manifest + + @param blob base64-encoded json containing published validator list + + @param signature Signature of the decoded blob + + @param version Version of published list format + + @param siteUri Uri of the site from which the list was validated + + @param hash Hash of the data parameters + + @param overlay Overlay object which will handle sending the message + + @param hashRouter HashRouter object which will determine which + peers not to send to + + @return `ListDisposition::accepted`, plus some of the publisher + information, if list was successfully applied + + @par Thread Safety + + May be called concurrently + */ + PublisherListStats + applyListAndBroadcast ( + std::string const& manifest, + std::string const& blob, + std::string const& signature, + std::uint32_t version, + std::string siteUri, + uint256 const& hash, + Overlay& overlay, + HashRouter& hashRouter); + /** Apply published list of public keys @param manifest base64-encoded publisher key manifest @@ -190,19 +267,38 @@ class ValidatorList @param version Version of published list format - @return `ListDisposition::accepted` if list was successfully applied + @param siteUri Uri of the site from which the list was validated + + @param hash Optional hash of the data parameters. + Defaults to uninitialized + + @return `ListDisposition::accepted`, plus some of the publisher + information, if list was successfully applied @par Thread Safety May be called concurrently */ - ListDisposition + PublisherListStats applyList ( std::string const& manifest, std::string const& blob, std::string const& signature, std::uint32_t version, - std::string siteUri); + std::string siteUri, + boost::optional const& hash = {}); + + /* Attempt to read previously stored list files. Expected to only be + called when loading from URL fails. + + @return A list of valid file:// URLs, if any. + + @par Thread Safety + + May be called concurrently + */ + std::vector + loadLists(); /** Update trusted nodes @@ -333,6 +429,47 @@ class ValidatorList for_each_listed ( std::function func) const; + /** Invokes the callback once for every available publisher list's raw + data members + + @note Undefined behavior results when calling ValidatorList members + from within the callback + + The arguments passed into the lambda are: + + @li The raw manifest string + + @li The raw "blob" string containing the values for the validator list + + @li The signature string used to sign the blob + + @li The version number + + @li The `PublicKey` of the blob signer (matches the value from + [validator_list_keys]) + + @li The sequence number of the "blob" + + @li The precomputed hash of the original / raw elements + + @par Thread Safety + + May be called concurrently + */ + void + for_each_available ( + std::function func) const; + + /** Returns the current valid list for the given publisher key, + if available, as a Json object. + */ + boost::optional + getAvailable(boost::beast::string_view const& pubKey); + /** Return the number of configured validator list sites. */ std::size_t count() const; @@ -371,6 +508,17 @@ class ValidatorList private: + /** Get the filename used for caching UNLs + */ + boost::filesystem::path + GetCacheFileName(PublicKey const& pubKey); + + /** Write a JSON UNL to a cache file + */ + void + CacheValidatorFile(PublicKey const& pubKey, + PublisherList const& publisher); + /** Check response for trusted valid published list @return `ListDisposition::accepted` if list can be applied diff --git a/src/ripple/app/misc/ValidatorSite.h b/src/ripple/app/misc/ValidatorSite.h index 655a93ca713..490c26ffb08 100644 --- a/src/ripple/app/misc/ValidatorSite.h +++ b/src/ripple/app/misc/ValidatorSite.h @@ -244,6 +244,11 @@ class ValidatorSite detail::response_type& res, std::size_t siteIdx, std::lock_guard& lock); + + /// If no sites are provided, or a site fails to load, + /// get a list of local cache files from the ValidatorList. + bool + missingSite(); }; } // ripple diff --git a/src/ripple/app/misc/impl/ValidatorList.cpp b/src/ripple/app/misc/impl/ValidatorList.cpp index 2b6f204526a..19b1ceeaf4e 100644 --- a/src/ripple/app/misc/impl/ValidatorList.cpp +++ b/src/ripple/app/misc/impl/ValidatorList.cpp @@ -18,11 +18,15 @@ //============================================================================== #include +#include #include +#include #include #include #include +#include #include +#include #include #include @@ -54,15 +58,19 @@ to_string(ListDisposition disposition) return "unknown"; } +const std::string ValidatorList::filePrefix_ = "cache."; + ValidatorList::ValidatorList ( ManifestCache& validatorManifests, ManifestCache& publisherManifests, TimeKeeper& timeKeeper, + std::string const& databasePath, beast::Journal j, boost::optional minimumQuorum) : validatorManifests_ (validatorManifests) , publisherManifests_ (publisherManifests) , timeKeeper_ (timeKeeper) + , dataPath_(databasePath) , j_ (j) , quorum_ (minimumQuorum.value_or(1)) // Genesis ledger quorum , minimumQuorum_ (minimumQuorum) @@ -192,16 +200,122 @@ ValidatorList::load ( return true; } -ListDisposition +boost::filesystem::path +ValidatorList::GetCacheFileName(PublicKey const& pubKey) +{ + return dataPath_ / (filePrefix_ + strHex(pubKey)); +} + +void +ValidatorList::CacheValidatorFile(PublicKey const& pubKey, + PublisherList const& publisher) +{ + if (dataPath_.empty()) + return; + + boost::filesystem::path const filename = + GetCacheFileName(pubKey); + + boost::system::error_code ec; + + Json::Value value(Json::objectValue); + + value["manifest"] = publisher.rawManifest; + value["blob"] = publisher.rawBlob; + value["signature"] = publisher.rawSignature; + value["version"] = publisher.rawVersion; + + writeFileContents(ec, filename, value.toStyledString()); + + if (ec) + { + // Log and ignore any file I/O exceptions + JLOG(j_.error()) << + "Problem writing " << + filename << + " " << + ec.value() << + ": " << + ec.message(); + } +} + +ValidatorList::PublisherListStats +ValidatorList::applyListAndBroadcast( + std::string const& manifest, + std::string const& blob, + std::string const& signature, + std::uint32_t version, + std::string siteUri, + uint256 const& hash, + Overlay& overlay, + HashRouter& hashRouter) +{ + auto const result = applyList(manifest, blob, signature, + version, std::move(siteUri), hash); + auto const disposition = result.disposition; + + bool broadcast = disposition == ListDisposition::accepted || + disposition == ListDisposition::same_sequence; + + if (broadcast) + { + assert(result.available && result.publisherKey && result.sequence); + auto const toSkip = hashRouter.shouldRelay(hash); + + if (toSkip) + { + protocol::TMValidatorList msg; + msg.set_manifest(manifest); + msg.set_blob(blob); + msg.set_signature(signature); + msg.set_version(version); + + auto const& publisherKey = *result.publisherKey; + auto const sequence = *result.sequence; + + // Can't use overlay.foreach here because we need to modify + // the peer, and foreach provides a const& + auto message = + std::make_shared(msg, protocol::mtVALIDATORLIST); + for (auto& peer : overlay.getActivePeers()) + { + if (toSkip->count(peer->id()) == 0 && + peer->supportsFeature( + ProtocolFeature::ValidatorListPropagation) && + peer->publisherListSequence(publisherKey) < sequence) + { + peer->send(message); + + JLOG(j_.debug()) + << "Sent validator list for " << strHex(publisherKey) + << " with sequence " << sequence << " to " + << peer->getRemoteAddress().to_string() << " (" + << peer->id() << ")"; + // Don't send it next time. + hashRouter.addSuppressionPeer(hash, peer->id()); + peer->setPublisherListSequence(publisherKey, sequence); + } + } + } + } + + return result; +} + +ValidatorList::PublisherListStats ValidatorList::applyList ( std::string const& manifest, std::string const& blob, std::string const& signature, std::uint32_t version, - std::string siteUri) + std::string siteUri, + boost::optional const& hash) { + using namespace std::string_literals; + if (version != requiredListVersion) - return ListDisposition::unsupported_version; + return PublisherListStats{ ListDisposition::unsupported_version }; std::unique_lock lock{mutex_}; @@ -209,16 +323,37 @@ ValidatorList::applyList ( PublicKey pubKey; auto const result = verify (list, pubKey, manifest, blob, signature); if (result != ListDisposition::accepted) - return result; + { + if (result == ListDisposition::same_sequence && + publisherLists_.count(pubKey)) + { + // We've seen this valid list already, so return + // what we know about it. + auto const& publisher = publisherLists_[pubKey]; + return PublisherListStats{ result, pubKey, + publisher.available, publisher.sequence }; + } + return PublisherListStats{ result }; + } // Update publisher's list Json::Value const& newList = list["validators"]; - publisherLists_[pubKey].available = true; - publisherLists_[pubKey].sequence = list["sequence"].asUInt (); - publisherLists_[pubKey].expiration = TimeKeeper::time_point{ + auto& publisher = publisherLists_[pubKey]; + publisher.available = true; + publisher.sequence = list["sequence"].asUInt (); + publisher.expiration = TimeKeeper::time_point{ TimeKeeper::duration{list["expiration"].asUInt()}}; - publisherLists_[pubKey].siteUri = std::move(siteUri); - std::vector& publisherList = publisherLists_[pubKey].list; + publisher.siteUri = std::move(siteUri); + publisher.rawManifest = manifest; + publisher.rawBlob = blob; + publisher.rawSignature = signature; + publisher.rawVersion = version; + if(hash) + publisher.hash = *hash; + std::vector& publisherList = publisher.list; + + PublisherListStats const applyResult{ result, pubKey, + publisher.available, publisher.sequence }; std::vector oldList = publisherList; publisherList.clear (); @@ -311,7 +446,65 @@ ValidatorList::applyList ( } } - return ListDisposition::accepted; + // Cache the validator list in a file + CacheValidatorFile(pubKey, publisher); + + return applyResult; +} + +std::vector +ValidatorList::loadLists() +{ + using namespace std::string_literals; + using namespace boost::filesystem; + using namespace boost::system::errc; + + std::unique_lock lock{mutex_}; + + std::vector sites; + sites.reserve(publisherLists_.size()); + for (auto const& [pubKey, publisher] : publisherLists_) + { + boost::system::error_code ec; + + if (publisher.available) + continue; + + boost::filesystem::path const filename = + GetCacheFileName(pubKey); + + auto const fullPath{ canonical(filename, ec) }; + if (ec) + continue; + + auto size = file_size(fullPath, ec); + if (!ec && !size) + { + // Treat an empty file as a missing file, because + // nobody else is going to write it. + ec = make_error_code(no_such_file_or_directory); + } + if (ec) + continue; + + std::string const prefix = [&fullPath]() { +#if _MSC_VER // MSVC: Windows paths need a leading / added + { + return fullPath.root_path() == "/"s ? + "file://" : "file:///"; + } +#else + { + (void)fullPath; + return "file://"; + } +#endif + }(); + sites.emplace_back(prefix + fullPath.string()); + } + + // Then let the ValidatorSites do the rest of the work. + return sites; } ListDisposition @@ -594,6 +787,57 @@ ValidatorList::for_each_listed ( func (v.first, trusted(v.first)); } +void +ValidatorList::for_each_available ( + std::function func) const +{ + std::shared_lock read_lock{mutex_}; + + for (auto const& [key, pl] : publisherLists_) + { + if (!pl.available) + continue; + func(pl.rawManifest, pl.rawBlob, pl.rawSignature, pl.rawVersion, + key, pl.sequence, pl.hash); + } +} + +boost::optional +ValidatorList::getAvailable(boost::beast::string_view const& pubKey) +{ + std::shared_lock read_lock{mutex_}; + + auto const keyBlob = strViewUnHex (pubKey); + + if (! keyBlob || ! publicKeyType(makeSlice(*keyBlob))) + { + JLOG (j_.info()) << + "Invalid requested validator list publisher key: " << pubKey; + return {}; + } + + auto id = PublicKey(makeSlice(*keyBlob)); + + auto iter = publisherLists_.find(id); + + if (iter == publisherLists_.end() + || !iter->second.available) + return {}; + + Json::Value value(Json::objectValue); + + value["manifest"] = iter->second.rawManifest; + value["blob"] = iter->second.rawBlob; + value["signature"] = iter->second.rawSignature; + value["version"] = iter->second.rawVersion; + + return value; +} + std::size_t ValidatorList::calculateQuorum ( std::size_t trusted, std::size_t seen) diff --git a/src/ripple/app/misc/impl/ValidatorSite.cpp b/src/ripple/app/misc/impl/ValidatorSite.cpp index 4dc00f0a2bc..286184d1ebf 100644 --- a/src/ripple/app/misc/impl/ValidatorSite.cpp +++ b/src/ripple/app/misc/impl/ValidatorSite.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -116,10 +117,23 @@ ValidatorSite::~ValidatorSite() } } +bool +ValidatorSite::missingSite() +{ + auto const sites = app_.validators().loadLists(); + return sites.empty() || load(sites); +} + bool ValidatorSite::load ( std::vector const& siteURIs) { + // If no sites are provided, act as if a site failed to load. + if (siteURIs.empty()) + { + return missingSite(); + } + JLOG (j_.debug()) << "Loading configured validator list sites"; @@ -374,54 +388,59 @@ ValidatorSite::parseJsonResponse ( throw std::runtime_error{"missing fields"}; } - auto const disp = app_.validators().applyList ( - body["manifest"].asString (), - body["blob"].asString (), - body["signature"].asString(), - body["version"].asUInt(), - sites_[siteIdx].activeResource->uri); + auto const manifest = body["manifest"].asString (); + auto const blob = body["blob"].asString (); + auto const signature = body["signature"].asString(); + auto const version = body["version"].asUInt(); + auto const& uri = sites_[siteIdx].activeResource->uri; + auto const hash = sha512Half(manifest, blob, signature, version); + auto const applyResult = app_.validators().applyListAndBroadcast ( + manifest, + blob, + signature, + version, + uri, + hash, + app_.overlay(), + app_.getHashRouter()); + auto const disp = applyResult.disposition; sites_[siteIdx].lastRefreshStatus.emplace( Site::Status{clock_type::now(), disp, ""}); - if (ListDisposition::accepted == disp) + switch (disp) { + case ListDisposition::accepted: JLOG (j_.debug()) << "Applied new validator list from " << - sites_[siteIdx].activeResource->uri; - } - else if (ListDisposition::same_sequence == disp) - { + uri; + break; + case ListDisposition::same_sequence: JLOG (j_.debug()) << "Validator list with current sequence from " << - sites_[siteIdx].activeResource->uri; - } - else if (ListDisposition::stale == disp) - { + uri; + break; + case ListDisposition::stale: JLOG (j_.warn()) << "Stale validator list from " << - sites_[siteIdx].activeResource->uri; - } - else if (ListDisposition::untrusted == disp) - { + uri; + break; + case ListDisposition::untrusted: JLOG (j_.warn()) << "Untrusted validator list from " << - sites_[siteIdx].activeResource->uri; - } - else if (ListDisposition::invalid == disp) - { + uri; + break; + case ListDisposition::invalid: JLOG (j_.warn()) << "Invalid validator list from " << - sites_[siteIdx].activeResource->uri; - } - else if (ListDisposition::unsupported_version == disp) - { + uri; + break; + case ListDisposition::unsupported_version: JLOG (j_.warn()) << "Unsupported version validator list from " << - sites_[siteIdx].activeResource->uri; - } - else - { + uri; + break; + default: BOOST_ASSERT(false); } @@ -509,6 +528,10 @@ ValidatorSite::onSiteFetch( if (retry) sites_[siteIdx].nextRefresh = clock_type::now() + error_retry_interval; + + // See if there's a copy saved locally from last time we + // saw the list. + missingSite(); }; if (ec) { @@ -592,7 +615,7 @@ ValidatorSite::onTextFetch( sites_[siteIdx].activeResource->uri << " " << ec.value() << - ":" << + ": " << ec.message(); throw std::runtime_error{"fetch error"}; } diff --git a/src/ripple/basics/FileUtilities.h b/src/ripple/basics/FileUtilities.h index d38ffa396d6..de8c9f9306b 100644 --- a/src/ripple/basics/FileUtilities.h +++ b/src/ripple/basics/FileUtilities.h @@ -27,13 +27,14 @@ namespace ripple { -// TODO: Should this one function have its own file, or can it -// be absorbed somewhere else? - std::string getFileContents(boost::system::error_code& ec, boost::filesystem::path const& sourcePath, boost::optional maxSize = boost::none); +void writeFileContents(boost::system::error_code& ec, + boost::filesystem::path const& destPath, + std::string const& contents); + } #endif diff --git a/src/ripple/basics/StringUtilities.h b/src/ripple/basics/StringUtilities.h index 7fa6ea34d18..7f28b2b6179 100644 --- a/src/ripple/basics/StringUtilities.h +++ b/src/ripple/basics/StringUtilities.h @@ -25,6 +25,7 @@ #include #include +#include #include #include @@ -63,7 +64,60 @@ inline static std::string sqlEscape (Blob const& vecSrc) uint64_t uintFromHex (std::string const& strSrc); -boost::optional strUnHex (std::string const& strSrc); +template +boost::optional +strUnHex(std::size_t strSize, Iterator begin, Iterator end) +{ + Blob out; + + out.reserve((strSize + 1) / 2); + + auto iter = begin; + + if (strSize & 1) + { + int c = charUnHex(*iter); + + if (c < 0) + return {}; + + out.push_back(c); + ++iter; + } + + while (iter != end) + { + int cHigh = charUnHex(*iter); + ++iter; + + if (cHigh < 0) + return {}; + + int cLow = charUnHex(*iter); + ++iter; + + if (cLow < 0) + return {}; + + out.push_back(static_cast((cHigh << 4) | cLow)); + } + + return {std::move(out)}; +} + +inline +boost::optional +strUnHex (std::string const& strSrc) +{ + return strUnHex(strSrc.size(), strSrc.cbegin(), strSrc.cend()); +} + +inline +boost::optional +strViewUnHex (boost::string_view const& strSrc) +{ + return strUnHex(strSrc.size(), strSrc.cbegin(), strSrc.cend()); +} struct parsedURL { diff --git a/src/ripple/basics/impl/FileUtilities.cpp b/src/ripple/basics/impl/FileUtilities.cpp index d24b0d7b2fc..edcd611ca04 100644 --- a/src/ripple/basics/impl/FileUtilities.cpp +++ b/src/ripple/basics/impl/FileUtilities.cpp @@ -60,4 +60,28 @@ std::string getFileContents(boost::system::error_code& ec, return result; } +void writeFileContents(boost::system::error_code& ec, + boost::filesystem::path const& destPath, + std::string const& contents) +{ + using namespace boost::filesystem; + using namespace boost::system::errc; + + ofstream fileStream(destPath, std::ios::out | std::ios::trunc); + + if (!fileStream) + { + ec = make_error_code(static_cast(errno)); + return; + } + + fileStream << contents; + + if (fileStream.bad ()) + { + ec = make_error_code(static_cast(errno)); + return; + } +} + } diff --git a/src/ripple/basics/impl/StringUtilities.cpp b/src/ripple/basics/impl/StringUtilities.cpp index e62bb31a4cd..05d3e25709e 100644 --- a/src/ripple/basics/impl/StringUtilities.cpp +++ b/src/ripple/basics/impl/StringUtilities.cpp @@ -30,45 +30,6 @@ namespace ripple { -boost::optional strUnHex (std::string const& strSrc) -{ - Blob out; - - out.reserve ((strSrc.size () + 1) / 2); - - auto iter = strSrc.cbegin (); - - if (strSrc.size () & 1) - { - int c = charUnHex (*iter); - - if (c < 0) - return {}; - - out.push_back(c); - ++iter; - } - - while (iter != strSrc.cend ()) - { - int cHigh = charUnHex (*iter); - ++iter; - - if (cHigh < 0) - return {}; - - int cLow = charUnHex (*iter); - ++iter; - - if (cLow < 0) - return {}; - - out.push_back (static_cast((cHigh << 4) | cLow)); - } - - return {std::move(out)}; -} - uint64_t uintFromHex (std::string const& strSrc) { uint64_t uValue (0); diff --git a/src/ripple/overlay/Overlay.h b/src/ripple/overlay/Overlay.h index 087e0bf5641..16830923249 100644 --- a/src/ripple/overlay/Overlay.h +++ b/src/ripple/overlay/Overlay.h @@ -80,6 +80,7 @@ class Overlay int ipLimit = 0; std::uint32_t crawlOptions = 0; boost::optional networkID; + bool vlEnabled = true; }; using PeerSequence = std::vector >; diff --git a/src/ripple/overlay/Peer.h b/src/ripple/overlay/Peer.h index 583cb04f33a..113d4802edc 100644 --- a/src/ripple/overlay/Peer.h +++ b/src/ripple/overlay/Peer.h @@ -35,6 +35,10 @@ class Charge; // Maximum hops to attempt when crawling shards. cs = crawl shards static constexpr std::uint32_t csHopLimit = 3; +enum class ProtocolFeature { + ValidatorListPropagation, +}; + /** Represents a peer connection in the overlay. */ class Peer { @@ -95,6 +99,17 @@ class Peer virtual Json::Value json() = 0; + virtual bool + supportsFeature(ProtocolFeature f) const = 0; + + virtual + boost::optional + publisherListSequence(PublicKey const&) const = 0; + + virtual + void + setPublisherListSequence(PublicKey const&, std::size_t const) = 0; + // // Ledger // diff --git a/src/ripple/overlay/impl/OverlayImpl.cpp b/src/ripple/overlay/impl/OverlayImpl.cpp index 602c692f6ca..db01cee0cfa 100644 --- a/src/ripple/overlay/impl/OverlayImpl.cpp +++ b/src/ripple/overlay/impl/OverlayImpl.cpp @@ -1016,7 +1016,7 @@ OverlayImpl::json () } bool -OverlayImpl::processRequest (http_request_type const& req, +OverlayImpl::processCrawl (http_request_type const& req, Handoff& handoff) { if (req.target() != "/crawl" || setup_.crawlOptions == CrawlOptions::Disabled) @@ -1052,6 +1052,62 @@ OverlayImpl::processRequest (http_request_type const& req, return true; } +bool +OverlayImpl::processValidatorList (http_request_type const& req, + Handoff& handoff) +{ + // If the target is in the form "/vl/", + // return the most recent validator list for that key. + if (!req.target().starts_with("/vl/") || + !setup_.vlEnabled) + return false; + + auto key = req.target(); + if (key.starts_with("/vl/")) + key.remove_prefix(strlen("/vl/")); + else + key.remove_prefix(strlen("/unl/")); + if(key.empty()) + return false; + + // find the list + auto vl = app_.validators().getAvailable(key); + + boost::beast::http::response msg; + msg.version(req.version()); + msg.insert("Server", BuildInfo::getFullVersionString()); + msg.insert("Content-Type", "application/json"); + msg.insert("Connection", "close"); + + if (!vl) + { + // 404 not found + msg.result(boost::beast::http::status::not_found); + msg.insert("Content-Length", "0"); + + msg.body() = Json::nullValue; + } + else + { + msg.result(boost::beast::http::status::ok); + + msg.body() = *vl; + } + + msg.prepare_payload(); + handoff.response = std::make_shared(msg); + return true; +} + +bool +OverlayImpl::processRequest (http_request_type const& req, + Handoff& handoff) +{ + // Take advantage of || short-circuiting + return processCrawl(req, handoff) || + processValidatorList(req, handoff); +} + Overlay::PeerSequence OverlayImpl::getActivePeers() { @@ -1331,6 +1387,11 @@ setup_Overlay (BasicConfig const& config) } } } + { + auto const& section = config.section("vl"); + + set(setup.vlEnabled, "enabled", section); + } try { diff --git a/src/ripple/overlay/impl/OverlayImpl.h b/src/ripple/overlay/impl/OverlayImpl.h index b4e5838478d..f04c7129bd9 100644 --- a/src/ripple/overlay/impl/OverlayImpl.h +++ b/src/ripple/overlay/impl/OverlayImpl.h @@ -385,6 +385,30 @@ class OverlayImpl : public Overlay http_request_type const& request, address_type remote_address, std::string msg); + /** Handles crawl requests. Crawl returns information about the + node and its peers so crawlers can map the network. + + @return true if the request was handled. + */ + bool + processCrawl (http_request_type const& req, + Handoff& handoff); + + /** Handles validator list requests. + Using a /vl/ URL, will retrieve the + latest valdiator list (or UNL) that this node has for that + public key, if the node trusts that public key. + + @return true if the request was handled. + */ + bool + processValidatorList (http_request_type const& req, + Handoff& handoff); + + /** Handles non-peer protocol requests. + + @return true if the request was handled. + */ bool processRequest (http_request_type const& req, Handoff& handoff); diff --git a/src/ripple/overlay/impl/PeerImp.cpp b/src/ripple/overlay/impl/PeerImp.cpp index 3c35b004a4f..122e3dd197c 100644 --- a/src/ripple/overlay/impl/PeerImp.cpp +++ b/src/ripple/overlay/impl/PeerImp.cpp @@ -405,6 +405,17 @@ PeerImp::json() return ret; } +bool +PeerImp::supportsFeature(ProtocolFeature f) const +{ + switch (f) + { + case ProtocolFeature::ValidatorListPropagation: + return protocol_ >= make_protocol(2, 1); + } + return false; +} + //------------------------------------------------------------------------------ bool @@ -803,6 +814,36 @@ PeerImp::doProtocolStart() { onReadMessage(error_code(), 0); + // Send all the validator lists that have been loaded + if (supportsFeature(ProtocolFeature::ValidatorListPropagation)) + { + app_.validators().for_each_available( + [&](std::string const& manifest, + std::string const& blob, std::string const& signature, + std::uint32_t version, + PublicKey const& pubKey, std::size_t sequence, + uint256 const& hash) + { + protocol::TMValidatorList vl; + + vl.set_manifest(manifest); + vl.set_blob(blob); + vl.set_signature(signature); + vl.set_version(version); + + JLOG(p_journal_.debug()) << "Sending validator list for " << + strHex(pubKey) << " with sequence " << + sequence << " to " << + remote_address_.to_string() << " (" << id_ << ")"; + auto m = std::make_shared(vl, protocol::mtVALIDATORLIST); + send(m); + // Don't send it next time. + app_.getHashRouter().addSuppressionPeer(hash, id_); + setPublisherListSequence(pubKey, sequence); + } + ); + } + protocol::TMManifests tm; app_.validatorManifests ().for_each_manifest ( @@ -1965,6 +2006,137 @@ PeerImp::onMessage (std::shared_ptr const& m) } } +void +PeerImp::onMessage (std::shared_ptr const& m) +{ + try + { + if (!supportsFeature(ProtocolFeature::ValidatorListPropagation)) + { + JLOG(p_journal_.debug()) + << "ValidatorList: received validator list from peer using " + << "protocol version " << to_string(protocol_) + << " which shouldn't support this feature."; + fee_ = Resource::feeUnwantedData; + return; + } + auto const& manifest = m->manifest(); + auto const& blob = m->blob(); + auto const& signature = m->signature(); + auto const version = m->version(); + auto const hash = sha512Half(manifest, blob, signature, version); + + JLOG(p_journal_.debug()) << "Received validator list from " << + remote_address_.to_string() << " (" << id_ << ")"; + + if (! app_.getHashRouter ().addSuppressionPeer(hash, id_)) + { + JLOG(p_journal_.debug()) << + "ValidatorList: received duplicate validator list"; + // Charging this fee here won't hurt the peer in the normal + // course of operation (ie. refresh every 5 minutes), but + // will add up if the peer is misbehaving. + fee_ = Resource::feeUnwantedData; + return; + } + + auto const applyResult = app_.validators().applyListAndBroadcast ( + manifest, + blob, + signature, + version, + remote_address_.to_string(), + hash, + app_.overlay(), + app_.getHashRouter()); + auto const disp = applyResult.disposition; + + JLOG(p_journal_.debug()) << "Processed validator list from " << + (applyResult.publisherKey ? strHex(*applyResult.publisherKey) : + "unknown or invalid publisher") << " from " << + remote_address_.to_string() << " (" << id_ << ") with result " << + to_string(disp); + + switch (disp) + { + case ListDisposition::accepted: + JLOG (p_journal_.debug()) << + "Applied new validator list from peer " << remote_address_; + { + std::lock_guard sl(recentLock_); + + assert(applyResult.sequence && applyResult.publisherKey); + auto const& pubKey = *applyResult.publisherKey; +#ifndef NDEBUG + if (auto const iter = publisherListSequences_.find(pubKey); + iter != publisherListSequences_.end()) + { + assert(iter->second < *applyResult.sequence); + } +#endif + publisherListSequences_[pubKey] = *applyResult.sequence; + } + break; + case ListDisposition::same_sequence: + JLOG (p_journal_.warn()) << + "Validator list with current sequence from peer " << + remote_address_; + // Charging this fee here won't hurt the peer in the normal + // course of operation (ie. refresh every 5 minutes), but + // will add up if the peer is misbehaving. + fee_ = Resource::feeUnwantedData; +#ifndef NDEBUG + { + std::lock_guard sl(recentLock_); + assert(applyResult.sequence && applyResult.publisherKey); + assert(publisherListSequences_[*applyResult.publisherKey] + == *applyResult.sequence); + } +#endif // !NDEBUG + + break; + case ListDisposition::stale: + JLOG (p_journal_.warn()) << + "Stale validator list from peer " << remote_address_; + // There are very few good reasons for a peer to send an + // old list, particularly more than once. + fee_ = Resource::feeBadData; + break; + case ListDisposition::untrusted: + JLOG (p_journal_.warn()) << + "Untrusted validator list from peer " << remote_address_; + // Charging this fee here won't hurt the peer in the normal + // course of operation (ie. refresh every 5 minutes), but + // will add up if the peer is misbehaving. + fee_ = Resource::feeUnwantedData; + break; + case ListDisposition::invalid: + JLOG (p_journal_.warn()) << + "Invalid validator list from peer " << remote_address_; + // This shouldn't ever happen with a well-behaved peer + fee_ = Resource::feeInvalidSignature; + break; + case ListDisposition::unsupported_version: + JLOG (p_journal_.warn()) << + "Unsupported version validator list from peer " << + remote_address_; + // During a version transition, this may be legitimate. + // If it happens frequently, that's probably bad. + fee_ = Resource::feeBadData; + break; + default: + assert(false); + } + } + catch (std::exception const& e) + { + JLOG(p_journal_.warn()) << + "ValidatorList: Exception, " << e.what() << + " from peer " << remote_address_; + fee_ = Resource::feeBadData; + } +} + void PeerImp::onMessage (std::shared_ptr const& m) { diff --git a/src/ripple/overlay/impl/PeerImp.h b/src/ripple/overlay/impl/PeerImp.h index 69897f58203..559acb26bc0 100644 --- a/src/ripple/overlay/impl/PeerImp.h +++ b/src/ripple/overlay/impl/PeerImp.h @@ -194,6 +194,9 @@ class PeerImp int large_sendq_ = 0; int no_ping_ = 0; std::unique_ptr load_event_; + // The highest sequence of each PublisherList that has + // been sent to or received from this peer. + hash_map publisherListSequences_; std::mutex mutable shardInfoMutex_; hash_map shardInfo_; @@ -342,6 +345,29 @@ class PeerImp Json::Value json() override; + bool + supportsFeature(ProtocolFeature f) const override; + + boost::optional + publisherListSequence(PublicKey const& pubKey) const override + { + std::lock_guard sl(recentLock_); + + auto iter = publisherListSequences_.find(pubKey); + if (iter != publisherListSequences_.end()) + return iter->second; + return {}; + } + + void + setPublisherListSequence(PublicKey const& pubKey, std::size_t const seq) + override + { + std::lock_guard sl(recentLock_); + + publisherListSequences_[pubKey] = seq; + } + // // Ledger // @@ -488,6 +514,7 @@ class PeerImp void onMessage (std::shared_ptr const& m); void onMessage (std::shared_ptr const& m); void onMessage (std::shared_ptr const& m); + void onMessage (std::shared_ptr const& m); void onMessage (std::shared_ptr const& m); void onMessage (std::shared_ptr const& m); diff --git a/src/ripple/overlay/impl/ProtocolMessage.h b/src/ripple/overlay/impl/ProtocolMessage.h index 6dd047f8cf5..8bc8ef77bb9 100644 --- a/src/ripple/overlay/impl/ProtocolMessage.h +++ b/src/ripple/overlay/impl/ProtocolMessage.h @@ -56,6 +56,7 @@ protocolMessageName (int type) case protocol::mtPROPOSE_LEDGER: return "propose"; case protocol::mtSTATUS_CHANGE: return "status"; case protocol::mtHAVE_SET: return "have_set"; + case protocol::mtVALIDATORLIST: return "validator_list"; case protocol::mtVALIDATION: return "validation"; case protocol::mtGET_OBJECTS: return "get_objects"; default: @@ -230,6 +231,9 @@ invokeProtocolMessage (Buffers const& buffers, Handler& handler) case protocol::mtVALIDATION: success = detail::invoke(*header, buffers, handler); break; + case protocol::mtVALIDATORLIST: + success = detail::invoke (*header, buffers, handler); + break; case protocol::mtGET_OBJECTS: success = detail::invoke(*header, buffers, handler); break; diff --git a/src/ripple/overlay/impl/ProtocolVersion.cpp b/src/ripple/overlay/impl/ProtocolVersion.cpp index bebcf4f1e70..156c7ed8e9c 100644 --- a/src/ripple/overlay/impl/ProtocolVersion.cpp +++ b/src/ripple/overlay/impl/ProtocolVersion.cpp @@ -36,7 +36,8 @@ constexpr ProtocolVersion const supportedProtocolList[] { { 1, 2 }, - { 2, 0 } + { 2, 0 }, + { 2, 1 } }; // This ugly construct ensures that supportedProtocolList is sorted in strictly @@ -130,10 +131,8 @@ parseProtocolVersions(boost::beast::string_view const& value) } boost::optional -negotiateProtocolVersion(boost::beast::string_view const& versions) +negotiateProtocolVersion(std::vector const& versions) { - auto const them = parseProtocolVersions(versions); - boost::optional result; // The protocol version we want to negotiate is the largest item in the @@ -148,13 +147,21 @@ negotiateProtocolVersion(boost::beast::string_view const& versions) }; std::set_intersection( - std::begin(them), std::end(them), + std::begin(versions), std::end(versions), std::begin(supportedProtocolList), std::end(supportedProtocolList), boost::make_function_output_iterator(pickVersion)); return result; } +boost::optional +negotiateProtocolVersion(boost::beast::string_view const& versions) +{ + auto const them = parseProtocolVersions(versions); + + return negotiateProtocolVersion(them); +} + std::string const& supportedProtocolVersions() { diff --git a/src/ripple/overlay/impl/ProtocolVersion.h b/src/ripple/overlay/impl/ProtocolVersion.h index 6e26d2c278b..66f19a8bad7 100644 --- a/src/ripple/overlay/impl/ProtocolVersion.h +++ b/src/ripple/overlay/impl/ProtocolVersion.h @@ -37,6 +37,7 @@ namespace ripple { using ProtocolVersion = std::pair; inline +constexpr ProtocolVersion make_protocol(std::uint16_t major, std::uint16_t minor) { @@ -62,6 +63,10 @@ to_string(ProtocolVersion const& p); std::vector parseProtocolVersions(boost::beast::string_view const& s); +/** Given a list of supported protocol versions, choose the one we prefer. */ +boost::optional +negotiateProtocolVersion(std::vector const& versions); + /** Given a list of supported protocol versions, choose the one we prefer. */ boost::optional negotiateProtocolVersion(boost::beast::string_view const& versions); diff --git a/src/ripple/overlay/impl/TrafficCount.cpp b/src/ripple/overlay/impl/TrafficCount.cpp index 8e0a69f062f..ca5952b68af 100644 --- a/src/ripple/overlay/impl/TrafficCount.cpp +++ b/src/ripple/overlay/impl/TrafficCount.cpp @@ -46,6 +46,9 @@ TrafficCount::category TrafficCount::categorize ( if (type == protocol::mtTRANSACTION) return TrafficCount::category::transaction; + if (type == protocol::mtVALIDATORLIST) + return TrafficCount::category::validatorlist; + if (type == protocol::mtVALIDATION) return TrafficCount::category::validation; diff --git a/src/ripple/overlay/impl/TrafficCount.h b/src/ripple/overlay/impl/TrafficCount.h index cf89b0327ad..602e8850612 100644 --- a/src/ripple/overlay/impl/TrafficCount.h +++ b/src/ripple/overlay/impl/TrafficCount.h @@ -75,6 +75,7 @@ class TrafficCount transaction, proposal, validation, + validatorlist, shards, // shard-related traffic // TMHaveSet message: @@ -189,6 +190,7 @@ class TrafficCount {"transactions"}, // category::transaction {"proposals"}, // category::proposal {"validations"}, // category::validation + {"validator_lists"}, // category::validatorlist {"shards"}, // category::shards {"set_get"}, // category::get_set {"set_share"}, // category::share_set diff --git a/src/ripple/proto/ripple.proto b/src/ripple/proto/ripple.proto index a82d628704a..471a25338ec 100644 --- a/src/ripple/proto/ripple.proto +++ b/src/ripple/proto/ripple.proto @@ -22,6 +22,7 @@ enum MessageType mtSHARD_INFO = 51; mtGET_PEER_SHARD_INFO = 52; mtPEER_SHARD_INFO = 53; + mtVALIDATORLIST = 54; } // token, iterations, target, challenge = issue demand for proof of work @@ -197,6 +198,14 @@ message TMHaveTransactionSet required bytes hash = 2; } +// Validator list +message TMValidatorList +{ + required bytes manifest = 1; + required bytes blob = 2; + required bytes signature = 3; + required uint32 version = 4; +} // Used to sign a final closed ledger after reprocessing message TMValidation diff --git a/src/test/app/Manifest_test.cpp b/src/test/app/Manifest_test.cpp index 6f8759b15e5..7d1e223ef54 100644 --- a/src/test/app/Manifest_test.cpp +++ b/src/test/app/Manifest_test.cpp @@ -254,8 +254,11 @@ class Manifest_test : public beast::unit_test::suite sort (getPopulatedManifests (m))); jtx::Env env (*this); + auto& app = env.app(); auto unl = std::make_unique ( - m, m, env.timeKeeper(), env.journal); + m, m, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); { // save should not store untrusted master keys to db diff --git a/src/test/app/ValidatorList_test.cpp b/src/test/app/ValidatorList_test.cpp index 21d2564d0e4..2f0661a5994 100644 --- a/src/test/app/ValidatorList_test.cpp +++ b/src/test/app/ValidatorList_test.cpp @@ -163,15 +163,20 @@ class ValidatorList_test : public beast::unit_test::suite ManifestCache manifests; jtx::Env env (*this); + auto& app = env.app(); { auto trustedKeys = std::make_unique ( - manifests, manifests, env.timeKeeper(), env.journal); + manifests, manifests, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); BEAST_EXPECT(trustedKeys->quorum () == 1); } { std::size_t minQuorum = 0; auto trustedKeys = std::make_unique ( - manifests, manifests, env.timeKeeper(), env.journal, minQuorum); + manifests, manifests, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal, minQuorum); BEAST_EXPECT(trustedKeys->quorum () == minQuorum); } } @@ -182,6 +187,7 @@ class ValidatorList_test : public beast::unit_test::suite testcase ("Config Load"); jtx::Env env (*this); + auto& app = env.app(); PublicKey emptyLocalKey; std::vector const emptyCfgKeys; std::vector const emptyCfgPublishers; @@ -230,7 +236,9 @@ class ValidatorList_test : public beast::unit_test::suite { ManifestCache manifests; auto trustedKeys = std::make_unique ( - manifests, manifests, env.timeKeeper(), env.journal); + manifests, manifests, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); // Correct (empty) configuration BEAST_EXPECT(trustedKeys->load ( @@ -252,7 +260,9 @@ class ValidatorList_test : public beast::unit_test::suite // load should add validator keys from config ManifestCache manifests; auto trustedKeys = std::make_unique ( - manifests, manifests, env.timeKeeper(), env.journal); + manifests, manifests, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); BEAST_EXPECT(trustedKeys->load ( emptyLocalKey, cfgKeys, emptyCfgPublishers)); @@ -290,7 +300,9 @@ class ValidatorList_test : public beast::unit_test::suite // local validator key on config list ManifestCache manifests; auto trustedKeys = std::make_unique ( - manifests, manifests, env.timeKeeper(), env.journal); + manifests, manifests, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); auto const localSigningPublic = parseBase58 ( TokenType::NodePublic, cfgKeys.front()); @@ -307,7 +319,9 @@ class ValidatorList_test : public beast::unit_test::suite // local validator key not on config list ManifestCache manifests; auto trustedKeys = std::make_unique ( - manifests, manifests, env.timeKeeper(), env.journal); + manifests, manifests, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); auto const localSigningPublic = randomNode(); BEAST_EXPECT(trustedKeys->load ( @@ -322,7 +336,9 @@ class ValidatorList_test : public beast::unit_test::suite // local validator key (with manifest) not on config list ManifestCache manifests; auto trustedKeys = std::make_unique ( - manifests, manifests, env.timeKeeper(), env.journal); + manifests, manifests, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); manifests.applyManifest (*deserializeManifest(cfgManifest)); @@ -338,7 +354,9 @@ class ValidatorList_test : public beast::unit_test::suite { ManifestCache manifests; auto trustedKeys = std::make_unique ( - manifests, manifests, env.timeKeeper(), env.journal); + manifests, manifests, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); // load should reject invalid validator list signing keys std::vector badPublishers( @@ -375,7 +393,9 @@ class ValidatorList_test : public beast::unit_test::suite ManifestCache valManifests; ManifestCache pubManifests; auto trustedKeys = std::make_unique ( - valManifests, pubManifests, env.timeKeeper(), env.journal); + valManifests, pubManifests, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); auto const pubRevokedSecret = randomSecretKey(); auto const pubRevokedPublic = @@ -414,8 +434,11 @@ class ValidatorList_test : public beast::unit_test::suite ManifestCache manifests; jtx::Env env (*this); + auto& app = env.app(); auto trustedKeys = std::make_unique ( - manifests, manifests, env.app().timeKeeper(), env.journal); + manifests, manifests, env.app().timeKeeper(), + app.config().legacy("database_path"), + env.journal); auto const publisherSecret = randomSecretKey(); auto const publisherPublic = @@ -453,7 +476,8 @@ class ValidatorList_test : public beast::unit_test::suite BEAST_EXPECT(ListDisposition::stale == trustedKeys->applyList ( - manifest1, expiredblob, expiredSig, version, siteUri)); + manifest1, expiredblob, expiredSig, + version, siteUri).disposition); // apply single list using namespace std::chrono_literals; @@ -463,8 +487,9 @@ class ValidatorList_test : public beast::unit_test::suite list1, sequence, expiration.time_since_epoch().count()); auto const sig1 = signList (blob1, pubSigningKeys1); - BEAST_EXPECT(ListDisposition::accepted == trustedKeys->applyList ( - manifest1, blob1, sig1, version, siteUri)); + BEAST_EXPECT(ListDisposition::accepted == + trustedKeys->applyList ( manifest1, blob1, + sig1, version, siteUri).disposition); for (auto const& val : list1) { @@ -479,13 +504,13 @@ class ValidatorList_test : public beast::unit_test::suite pubSigningKeys1.first, pubSigningKeys1.second, 1)); BEAST_EXPECT(ListDisposition::untrusted == trustedKeys->applyList ( - untrustedManifest, blob1, sig1, version, siteUri)); + untrustedManifest, blob1, sig1, version, siteUri).disposition); // do not use list with unhandled version auto const badVersion = 666; BEAST_EXPECT(ListDisposition::unsupported_version == trustedKeys->applyList ( - manifest1, blob1, sig1, badVersion, siteUri)); + manifest1, blob1, sig1, badVersion, siteUri).disposition); // apply list with highest sequence number auto const sequence2 = 2; @@ -495,7 +520,7 @@ class ValidatorList_test : public beast::unit_test::suite BEAST_EXPECT(ListDisposition::accepted == trustedKeys->applyList ( - manifest1, blob2, sig2, version, siteUri)); + manifest1, blob2, sig2, version, siteUri).disposition); for (auto const& val : list1) { @@ -512,11 +537,11 @@ class ValidatorList_test : public beast::unit_test::suite // do not re-apply lists with past or current sequence numbers BEAST_EXPECT(ListDisposition::stale == trustedKeys->applyList ( - manifest1, blob1, sig1, version, siteUri)); + manifest1, blob1, sig1, version, siteUri).disposition); BEAST_EXPECT(ListDisposition::same_sequence == trustedKeys->applyList ( - manifest1, blob2, sig2, version, siteUri)); + manifest1, blob2, sig2, version, siteUri).disposition); // apply list with new publisher key updated by manifest auto const pubSigningKeys2 = randomKeyPair(KeyType::secp256k1); @@ -531,7 +556,7 @@ class ValidatorList_test : public beast::unit_test::suite BEAST_EXPECT(ListDisposition::accepted == trustedKeys->applyList ( - manifest2, blob3, sig3, version, siteUri)); + manifest2, blob3, sig3, version, siteUri).disposition); auto const sequence4 = 4; auto const blob4 = makeList ( @@ -539,7 +564,7 @@ class ValidatorList_test : public beast::unit_test::suite auto const badSig = signList (blob4, pubSigningKeys1); BEAST_EXPECT(ListDisposition::invalid == trustedKeys->applyList ( - manifest1, blob4, badSig, version, siteUri)); + manifest1, blob4, badSig, version, siteUri).disposition); // do not apply list with revoked publisher key // applied list is removed due to revoked publisher key @@ -554,7 +579,7 @@ class ValidatorList_test : public beast::unit_test::suite BEAST_EXPECT(ListDisposition::untrusted == trustedKeys->applyList ( - maxManifest, blob5, sig5, version, siteUri)); + maxManifest, blob5, sig5, version, siteUri).disposition); BEAST_EXPECT(! trustedKeys->trustedPublisher(publisherPublic)); for (auto const& val : list1) @@ -574,8 +599,11 @@ class ValidatorList_test : public beast::unit_test::suite PublicKey emptyLocalKeyOuter; ManifestCache manifestsOuter; jtx::Env env (*this); + auto& app = env.app(); auto trustedKeysOuter = std::make_unique ( - manifestsOuter, manifestsOuter, env.timeKeeper(), env.journal); + manifestsOuter, manifestsOuter, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); std::vector cfgPublishersOuter; hash_set activeValidatorsOuter; @@ -722,7 +750,9 @@ class ValidatorList_test : public beast::unit_test::suite { // Make quorum unattainable if lists from any publishers are unavailable auto trustedKeys = std::make_unique ( - manifestsOuter, manifestsOuter, env.timeKeeper(), env.journal); + manifestsOuter, manifestsOuter, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); auto const publisherSecret = randomSecretKey(); auto const publisherPublic = derivePublicKey(KeyType::ed25519, publisherSecret); @@ -746,7 +776,9 @@ class ValidatorList_test : public beast::unit_test::suite std::size_t const minQuorum = 1; ManifestCache manifests; auto trustedKeys = std::make_unique ( - manifests, manifests, env.timeKeeper(), env.journal, minQuorum); + manifests, manifests, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal, minQuorum); std::size_t n = 10; std::vector cfgKeys; @@ -786,7 +818,9 @@ class ValidatorList_test : public beast::unit_test::suite { // Remove expired published list auto trustedKeys = std::make_unique ( - manifestsOuter, manifestsOuter, env.app().timeKeeper(), env.journal); + manifestsOuter, manifestsOuter, env.app().timeKeeper(), + app.config().legacy("database_path"), + env.journal); PublicKey emptyLocalKey; std::vector emptyCfgKeys; @@ -819,7 +853,7 @@ class ValidatorList_test : public beast::unit_test::suite BEAST_EXPECT(ListDisposition::accepted == trustedKeys->applyList ( - manifest, blob, sig, version, siteUri)); + manifest, blob, sig, version, siteUri).disposition); TrustChanges changes = trustedKeys->updateTrusted(activeValidators); @@ -853,7 +887,7 @@ class ValidatorList_test : public beast::unit_test::suite BEAST_EXPECT(ListDisposition::accepted == trustedKeys->applyList ( - manifest, blob2, sig2, version, siteUri)); + manifest, blob2, sig2, version, siteUri).disposition); changes = trustedKeys->updateTrusted (activeValidators); BEAST_EXPECT(changes.removed.empty()); @@ -872,7 +906,9 @@ class ValidatorList_test : public beast::unit_test::suite { // Test 1-9 configured validators auto trustedKeys = std::make_unique ( - manifestsOuter, manifestsOuter, env.timeKeeper(), env.journal); + manifestsOuter, manifestsOuter, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); std::vector cfgPublishers; hash_set activeValidators; @@ -903,7 +939,9 @@ class ValidatorList_test : public beast::unit_test::suite { // Test 2-9 configured validators as validator auto trustedKeys = std::make_unique ( - manifestsOuter, manifestsOuter, env.timeKeeper(), env.journal); + manifestsOuter, manifestsOuter, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); auto const localKey = randomNode(); std::vector cfgPublishers; @@ -943,7 +981,9 @@ class ValidatorList_test : public beast::unit_test::suite // Trusted set should include all validators from multiple lists ManifestCache manifests; auto trustedKeys = std::make_unique ( - manifests, manifests, env.timeKeeper(), env.journal); + manifests, manifests, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); hash_set activeValidators; std::vector valKeys; @@ -983,8 +1023,9 @@ class ValidatorList_test : public beast::unit_test::suite valKeys, sequence, expiration.time_since_epoch().count()); auto const sig = signList (blob, pubSigningKeys); - BEAST_EXPECT(ListDisposition::accepted == trustedKeys->applyList ( - manifest, blob, sig, version, siteUri)); + BEAST_EXPECT(ListDisposition::accepted == + trustedKeys->applyList (manifest, blob, sig, version, + siteUri).disposition); }; // Apply multiple published lists @@ -1016,6 +1057,7 @@ class ValidatorList_test : public beast::unit_test::suite std::string const siteUri = "testExpires.test"; jtx::Env env(*this); + auto& app = env.app(); auto toStr = [](PublicKey const& publicKey) { return toBase58(TokenType::NodePublic, publicKey); @@ -1025,7 +1067,9 @@ class ValidatorList_test : public beast::unit_test::suite { ManifestCache manifests; auto trustedKeys = std::make_unique( - manifests, manifests, env.timeKeeper(), env.journal); + manifests, manifests, env.timeKeeper(), + app.config().legacy("database_path"), + env.journal); // Empty list has no expiration BEAST_EXPECT(trustedKeys->expires() == boost::none); @@ -1044,7 +1088,9 @@ class ValidatorList_test : public beast::unit_test::suite { ManifestCache manifests; auto trustedKeys = std::make_unique( - manifests, manifests, env.app().timeKeeper(), env.journal); + manifests, manifests, env.app().timeKeeper(), + app.config().legacy("database_path"), + env.journal); std::vector validators = {randomValidator()}; hash_set activeValidators; @@ -1104,7 +1150,8 @@ class ValidatorList_test : public beast::unit_test::suite // Apply first list BEAST_EXPECT( ListDisposition::accepted == trustedKeys->applyList( - prep1.manifest, prep1.blob, prep1.sig, prep1.version, siteUri)); + prep1.manifest, prep1.blob, prep1.sig, + prep1.version, siteUri).disposition); // One list still hasn't published, so expiration is still unknown BEAST_EXPECT(trustedKeys->expires() == boost::none); @@ -1112,7 +1159,8 @@ class ValidatorList_test : public beast::unit_test::suite // Apply second list BEAST_EXPECT( ListDisposition::accepted == trustedKeys->applyList( - prep2.manifest, prep2.blob, prep2.sig, prep2.version, siteUri)); + prep2.manifest, prep2.blob, prep2.sig, + prep2.version, siteUri).disposition); // We now have loaded both lists, so expiration is known BEAST_EXPECT( diff --git a/src/test/basics/FileUtilities_test.cpp b/src/test/basics/FileUtilities_test.cpp index b0910192961..5e8c6abfa6c 100644 --- a/src/test/basics/FileUtilities_test.cpp +++ b/src/test/basics/FileUtilities_test.cpp @@ -36,11 +36,14 @@ class FileUtilities_test : public beast::unit_test::suite "This file is very short. That's all we need."; FileDirGuard file(*this, "test_file", "test.txt", - expectedContents); + "This is temporary text that should get overwritten"); error_code ec; auto const path = file.file(); + writeFileContents(ec, path, expectedContents); + BEAST_EXPECT(!ec); + { // Test with no max auto const good = getFileContents(ec, path);