Propagate validator lists (VLs or UNLs) over the peer network:

* Whenever a node downloads a new VL, send it to all peers that
  haven't already sent or received it. It also saves it to the
  database_dir as a Json text file named "cache." plus the public key of
  the list signer. Any files that exist for public keys provided in
  [validator_list_keys] will be loaded and processed if any download
  from [validator_list_sites] fails or no [validator_list_sites] are
  configured.
* Whenever a node receives a broadcast VL message, it treats it as if
  it had downloaded it on it's own, broadcasting to other peers as
  described above.
* Because nodes normally download the VL once every 5 minutes, a single
  node downloading a VL with an updated sequence number could
  potentially propagate across a large part of a well-connected network
  before any other nodes attempt to download, decreasing the amount of
  time that different parts of the network are using different VLs.
* Send all of our current valid VLs to new peers on connection.
  This is probably the "noisiest" part of this change, but will give
  poorly connected or poorly networked nodes the best chance of syncing
  quickly. Nodes which have no http(s) access configured or available
  can get a VL with no extra effort.
* Requests on the peer port to the /vl/<pubkey> endpoint will return
  that VL in the same JSON format as is used to download now, IF the
  node trusts and has a valid instance of that VL.
* Upgrade protocol version to 2.1. VLs will only be sent to 2.1 and
  higher nodes.
* Resolves #2953

cfg/rippled-example.cfg

@@ -1052,7 +1052,7 @@
 # [crawl]
 #
 #   List of options to control what data is reported through the /crawl endpoint
-#   See https://developers.ripple.com/peer-protocol.html#peer-crawler
+#   See https://xrpl.org/peer-crawler.html
 #
 #   <flag>
 #
@@ -1093,6 +1093,16 @@
 #   counts = 0
 #   unl = 1
 #
+# [vl]
+#
+#   Options to control what data is reported through the /vl endpoint
+#   See [...]
+#
+#   enable = <flag>
+#
+#       Enable or disable access to /vl requests. Default is '1' which
+#       enables access.
+#
 #-------------------------------------------------------------------------------
 #
 # 9. Example Settings

src/ripple/app/main/Application.cpp

@@ -365,7 +365,7 @@ class ApplicationImp
     std::unique_ptr <ServerHandler> serverHandler_;
     std::unique_ptr <AmendmentTable> m_amendmentTable;
     std::unique_ptr <LoadFeeTrack> mFeeTrack;
-    std::unique_ptr <HashRouter> mHashRouter;
+    std::unique_ptr <HashRouter> hashRouter_;
     RCLValidations mValidations;
     std::unique_ptr <LoadManager> m_loadManager;
     std::unique_ptr <TxQ> txQ_;
@@ -377,7 +377,7 @@ class ApplicationImp
     std::unique_ptr <DatabaseCon> mTxnDB;
     std::unique_ptr <DatabaseCon> mLedgerDB;
     std::unique_ptr <DatabaseCon> mWalletDB;
-    std::unique_ptr <Overlay> m_overlay;
+    std::unique_ptr <Overlay> overlay_;
     std::vector <std::unique_ptr<Stoppable>> websocketServers_;
 
     boost::asio::signal_set m_signals;
@@ -520,7 +520,8 @@ class ApplicationImp
             logs_->journal("ManifestCache")))
 
         , validators_ (std::make_unique<ValidatorList> (
-            *validatorManifests_, *publisherManifests_, *timeKeeper_,
+            *validatorManifests_, *publisherManifests_,
+            *timeKeeper_, config_->legacy("database_path"),
             logs_->journal("ValidatorList"), config_->VALIDATION_QUORUM))
 
         , validatorSites_ (std::make_unique<ValidatorSite> (*this))
@@ -531,7 +532,7 @@ class ApplicationImp
 
         , mFeeTrack (std::make_unique<LoadFeeTrack>(logs_->journal("LoadManager")))
 
-        , mHashRouter (std::make_unique<HashRouter>(
+        , hashRouter_ (std::make_unique<HashRouter>(
             stopwatch(), HashRouter::getDefaultHoldTime (),
             HashRouter::getDefaultRecoverLimit ()))
 
@@ -760,7 +761,7 @@ class ApplicationImp
 
     HashRouter& getHashRouter () override
     {
-        return *mHashRouter;
+        return *hashRouter_;
     }
 
     RCLValidations& getValidations () override
@@ -828,7 +829,8 @@ class ApplicationImp
 
     Overlay& overlay () override
     {
-        return *m_overlay;
+        assert(overlay_);
+        return *overlay_;
     }
 
     TxQ& getTxQ() override
@@ -1480,10 +1482,10 @@ bool ApplicationImp::setup()
     //             move the instantiation inside a conditional:
     //
     //             if (!config_.standalone())
-    m_overlay = make_Overlay (*this, setup_Overlay(*config_), *m_jobQueue,
+    overlay_ = make_Overlay (*this, setup_Overlay(*config_), *m_jobQueue,
         *serverHandler_, *m_resourceManager, *m_resolver, get_io_service(),
         *config_, m_collectorManager->collector ());
-    add (*m_overlay); // add to PropertyStream
+    add (*overlay_); // add to PropertyStream
 
     if (!config_->standalone())
     {
@@ -1675,7 +1677,7 @@ int ApplicationImp::fdRequired() const
     int needed = 128;
 
     // 1.5 times the configured peer limit for peer connections:
-    needed += static_cast<int>(0.5 + (1.5 * m_overlay->limit()));
+    needed += static_cast<int>(0.5 + (1.5 * overlay_->limit()));
 
     // the number of fds needed by the backend (internally
     // doubled if online delete is enabled).
@@ -2131,7 +2133,7 @@ ApplicationImp::journal (std::string const& name)
 
 bool ApplicationImp::nodeToShards()
 {
-    assert(m_overlay);
+    assert(overlay_);
     assert(!config_->standalone());
 
     if (config_->section(ConfigSection::shardDatabase()).empty())
@@ -2152,7 +2154,7 @@ bool ApplicationImp::nodeToShards()
 
 bool ApplicationImp::validateShards()
 {
-    assert(m_overlay);
+    assert(overlay_);
     assert(!config_->standalone());
 
     if (config_->section(ConfigSection::shardDatabase()).empty())

src/ripple/app/misc/HashRouter.h

@@ -197,8 +197,10 @@ class HashRouter
     boost::optional<std::set<PeerShortID>> shouldRelay(uint256 const& key);
 
     /** Determines whether the hashed item should be recovered
+        from the open ledger into the next open ledger or the transaction
+        queue.
 
-        @return `bool` indicates whether the item should be relayed
+        @return `bool` indicates whether the item should be recovered
     */
     bool shouldRecover(uint256 const& key);
 

src/ripple/app/misc/ValidatorList.h

@@ -35,6 +35,10 @@
 
 namespace ripple {
 
+// predeclaration
+class Overlay;
+class HashRouter;
+
 enum class ListDisposition
 {
     /// List is valid
@@ -123,11 +127,17 @@ class ValidatorList
         std::size_t sequence;
         TimeKeeper::time_point expiration;
         std::string siteUri;
+        std::string rawManifest;
+        std::string rawBlob;
+        std::string rawSignature;
+        std::uint32_t rawVersion;
+        uint256 hash;
     };
 
     ManifestCache& validatorManifests_;
     ManifestCache& publisherManifests_;
     TimeKeeper& timeKeeper_;
+    boost::filesystem::path const dataPath_;
     beast::Journal const j_;
     std::shared_timed_mutex mutable mutex_;
 
@@ -147,16 +157,45 @@ class ValidatorList
 
     // Currently supported version of publisher list format
     static constexpr std::uint32_t requiredListVersion = 1;
+    static const std::string filePrefix_;
 
 public:
     ValidatorList (
         ManifestCache& validatorManifests,
         ManifestCache& publisherManifests,
         TimeKeeper& timeKeeper,
+        std::string const& databasePath,
         beast::Journal j,
         boost::optional<std::size_t> minimumQuorum = boost::none);
     ~ValidatorList () = default;
 
+    /** Describes the result of processing a Validator List (UNL),
+    including some of the information from the list which can
+    be used by the caller to know which list publisher is
+    involved.
+    */
+    struct PublisherListStats
+    {
+        explicit PublisherListStats(ListDisposition d)
+            : disposition(d)
+        {
+        }
+
+        PublisherListStats(ListDisposition d, PublicKey key,
+            bool avail, std::size_t seq)
+            : disposition(d)
+            , publisherKey(key)
+            , available(avail)
+            , sequence(seq)
+        {
+        }
+
+        ListDisposition disposition;
+        boost::optional<PublicKey> publisherKey;
+        bool available = false;
+        boost::optional<std::size_t> sequence;
+    };
+
     /** Load configured trusted keys.
 
         @param localSigningKey This node's validation public key
@@ -180,6 +219,44 @@ class ValidatorList
         std::vector<std::string> const& configKeys,
         std::vector<std::string> const& publisherKeys);
 
+    /** Apply published list of public keys, then broadcast it to all
+        peers that have not seen it or sent it.
+
+        @param manifest base64-encoded publisher key manifest
+
+        @param blob base64-encoded json containing published validator list
+
+        @param signature Signature of the decoded blob
+
+        @param version Version of published list format
+
+        @param siteUri Uri of the site from which the list was validated
+
+        @param hash Hash of the data parameters
+
+        @param overlay Overlay object which will handle sending the message
+
+        @param hashRouter HashRouter object which will determine which
+            peers not to send to
+
+        @return `ListDisposition::accepted`, plus some of the publisher
+            information, if list was successfully applied
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    PublisherListStats
+    applyListAndBroadcast (
+        std::string const& manifest,
+        std::string const& blob,
+        std::string const& signature,
+        std::uint32_t version,
+        std::string siteUri,
+        uint256 const& hash,
+        Overlay& overlay,
+        HashRouter& hashRouter);
+
     /** Apply published list of public keys
 
         @param manifest base64-encoded publisher key manifest
@@ -190,19 +267,38 @@ class ValidatorList
 
         @param version Version of published list format
 
-        @return `ListDisposition::accepted` if list was successfully applied
+        @param siteUri Uri of the site from which the list was validated
+
+        @param hash Optional hash of the data parameters.
+            Defaults to uninitialized
+
+        @return `ListDisposition::accepted`, plus some of the publisher
+            information, if list was successfully applied
 
         @par Thread Safety
 
         May be called concurrently
     */
-    ListDisposition
+    PublisherListStats
     applyList (
         std::string const& manifest,
         std::string const& blob,
         std::string const& signature,
         std::uint32_t version,
-        std::string siteUri);
+        std::string siteUri,
+        boost::optional<uint256> const& hash = {});
+
+    /* Attempt to read previously stored list files. Expected to only be
+       called when loading from URL fails.
+
+       @return A list of valid file:// URLs, if any.
+
+       @par Thread Safety
+
+       May be called concurrently
+    */
+    std::vector<std::string>
+    loadLists();
 
     /** Update trusted nodes
 
@@ -333,6 +429,47 @@ class ValidatorList
     for_each_listed (
         std::function<void(PublicKey const&, bool)> func) const;
 
+    /** Invokes the callback once for every available publisher list's raw
+        data members
+
+        @note Undefined behavior results when calling ValidatorList members
+        from within the callback
+
+        The arguments passed into the lambda are:
+
+        @li The raw manifest string
+
+        @li The raw "blob" string containing the values for the validator list
+
+        @li The signature string used to sign the blob
+
+        @li The version number
+
+        @li The `PublicKey` of the blob signer (matches the value from
+            [validator_list_keys])
+
+        @li The sequence number of the "blob"
+
+        @li The precomputed hash of the original / raw elements
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    void
+    for_each_available (
+        std::function<void(std::string const& manifest,
+            std::string const& blob, std::string const& signature,
+            std::uint32_t version,
+            PublicKey const& pubKey, std::size_t sequence,
+            uint256 const& hash)> func) const;
+
+    /** Returns the current valid list for the given publisher key,
+        if available, as a Json object.
+    */
+    boost::optional<Json::Value>
+    getAvailable(boost::beast::string_view const& pubKey);
+
     /** Return the number of configured validator list sites. */
     std::size_t
     count() const;
@@ -371,6 +508,17 @@ class ValidatorList
 
 
 private:
+    /** Get the filename used for caching UNLs
+    */
+    boost::filesystem::path
+    GetCacheFileName(PublicKey const& pubKey);
+
+    /** Write a JSON UNL to a cache file
+    */
+    void
+    CacheValidatorFile(PublicKey const& pubKey,
+        PublisherList const& publisher);
+
     /** Check response for trusted valid published list
 
         @return `ListDisposition::accepted` if list can be applied

src/ripple/app/misc/ValidatorSite.h

@@ -244,6 +244,11 @@ class ValidatorSite
         detail::response_type& res,
         std::size_t siteIdx,
         std::lock_guard<std::mutex>& lock);
+
+    /// If no sites are provided, or a site fails to load,
+    /// get a list of local cache files from the ValidatorList.
+    bool
+    missingSite();
 };
 
 } // ripple