Improve protocol-level handshaking protocol:

This commit restructures the HTTP based protocol negotiation that `rippled`
executes and introduces support for negotiation of compression for peer
links which, if implemented, should result in significant bandwidth savings
for some server roles.

This commit also introduces the new `[network_id]` configuration option
that administrators can use to specify which network the server is part of
and intends to join. This makes it possible for servers from different
networks to drop the link early.

The changeset also improves the log messages generated when negotiation
of a peer link upgrade fails. In the past, no useful information would
be logged, making it more difficult for admins to troubleshoot errors.

This commit also fixes RIPD-237 and RIPD-451

Builds/CMake/RippledCore.cmake

@@ -176,7 +176,6 @@ install (
 install (
   FILES
     src/ripple/crypto/GenerateDeterministicKey.h
-    src/ripple/crypto/KeyType.h
     src/ripple/crypto/RFC1751.h
     src/ripple/crypto/csprng.h
   DESTINATION include/ripple/crypto)
@@ -214,6 +213,7 @@ install (
     src/ripple/protocol/Indexes.h
     src/ripple/protocol/InnerObjectFormats.h
     src/ripple/protocol/Issue.h
+    src/ripple/protocol/KeyType.h
     src/ripple/protocol/Keylet.h
     src/ripple/protocol/KnownFormats.h
     src/ripple/protocol/LedgerFormats.h
@@ -602,12 +602,13 @@ else ()
     #]===============================]
     src/ripple/overlay/impl/Cluster.cpp
     src/ripple/overlay/impl/ConnectAttempt.cpp
+    src/ripple/overlay/impl/Handshake.cpp
     src/ripple/overlay/impl/Message.cpp
     src/ripple/overlay/impl/OverlayImpl.cpp
     src/ripple/overlay/impl/PeerImp.cpp
     src/ripple/overlay/impl/PeerReservationTable.cpp
     src/ripple/overlay/impl/PeerSet.cpp
-    src/ripple/overlay/impl/TMHello.cpp
+    src/ripple/overlay/impl/ProtocolVersion.cpp
     src/ripple/overlay/impl/TrafficCount.cpp
     #[===============================[
        nounity, main sources:
@@ -920,7 +921,7 @@ else ()
        nounity, test sources:
          subdir: overlay
     #]===============================]
-    src/test/overlay/TMHello_test.cpp
+    src/test/overlay/ProtocolVersion_test.cpp
     src/test/overlay/cluster_test.cpp
     src/test/overlay/short_read_test.cpp
     #[===============================[
@@ -933,7 +934,6 @@ else ()
        nounity, test sources:
          subdir: protocol
     #]===============================]
-    src/test/protocol/BuildInfo_test.cpp
     src/test/protocol/IOUAmount_test.cpp
     src/test/protocol/InnerObjectFormats_test.cpp
     src/test/protocol/Issue_test.cpp

cfg/rippled-example.cfg

@@ -733,6 +733,24 @@
 #   node is a validator.
 #
 #
+#
+# [network_id]
+#
+#   Specify the network which this server is configured to connect to and
+#   track. If set, the server will not establish connections with servers
+#   that are explicitly configured to track another network.
+#
+#   Network identifiers are usually unsigned integers in the range 0 to
+#   4294967295 inclusive. The server also maps the following well-known
+#   names to the corresponding numerical identifier:
+#
+#   main    -> 0
+#   testnet -> 1
+#
+#   If this value is not specified the server is not explicitly configured
+#   to track a particular network.
+#
+#
 #-------------------------------------------------------------------------------
 #
 # 4. HTTPS Client

src/ripple/overlay/Message.h

@@ -24,6 +24,7 @@
 #include <boost/asio/buffer.hpp>
 #include <boost/asio/buffers_iterator.hpp>
 #include <algorithm>
+#include <array>
 #include <cstdint>
 #include <iterator>
 #include <memory>
@@ -47,26 +48,9 @@ namespace ripple {
 class Message : public std::enable_shared_from_this <Message>
 {
 public:
-    using pointer = std::shared_ptr<Message>;
-
-public:
-    /** Number of bytes in a message header.
-
-        A message will not be processed unless a full header is received and
-        no messages smaller than this will be serialized.
-    */
-    static std::size_t constexpr kHeaderBytes = 6;
-
-    /** The largest size that a message can be.
-
-        Sending a message whose size exceeds this may result in the connection
-        being dropped. A larger message size may be supported in the future or
-        negotiated as part of a protocol upgrade.
-     */
-    static std::size_t constexpr kMaxMessageSize = 64 * 1024 * 1024;
-
     Message (::google::protobuf::Message const& message, int type);
 
+public:
     /** Retrieve the packed message data. */
     std::vector <uint8_t> const&
     getBuffer () const
@@ -81,92 +65,8 @@ class Message : public std::enable_shared_from_this <Message>
         return mCategory;
     }
 
-    /** Determine bytewise equality. */
-    bool operator == (Message const& other) const;
-
-    /** Calculate the length of a packed message. */
-    /** @{ */
-    static unsigned getLength (std::vector <uint8_t> const& buf);
-
-    template <class FwdIter>
-    static
-    std::enable_if_t<std::is_same<typename
-        FwdIter::value_type, std::uint8_t>::value, std::size_t>
-    size (FwdIter first, FwdIter last)
-    {
-        if (std::distance(first, last) <
-                Message::kHeaderBytes)
-            return 0;
-        std::size_t n;
-        n  = std::size_t{*first++} << 24;
-        n += std::size_t{*first++} << 16;
-        n += std::size_t{*first++} <<  8;
-        n += std::size_t{*first};
-        return n;
-    }
-
-    template <class BufferSequence>
-    static
-    std::size_t
-    size (BufferSequence const& buffers)
-    {
-        return size(buffers_begin(buffers),
-            buffers_end(buffers));
-    }
-    /** @} */
-
-    /** Determine the type of a packed message. */
-    /** @{ */
-    static int getType (std::vector <uint8_t> const& buf);
-
-    template <class FwdIter>
-    static
-    std::enable_if_t<std::is_same<typename
-        FwdIter::value_type, std::uint8_t>::value, int>
-    type (FwdIter first, FwdIter last)
-    {
-        if (std::distance(first, last) <
-                Message::kHeaderBytes)
-            return 0;
-        return (int{*std::next(first, 4)} << 8) |
-            *std::next(first, 5);
-    }
-
-    template <class BufferSequence>
-    static
-    int
-    type (BufferSequence const& buffers)
-    {
-        return type(buffers_begin(buffers),
-            buffers_end(buffers));
-    }
-    /** @} */
-
 private:
-    template <class BufferSequence, class Value = std::uint8_t>
-    static
-    boost::asio::buffers_iterator<BufferSequence, Value>
-    buffers_begin (BufferSequence const& buffers)
-    {
-        return boost::asio::buffers_iterator<
-            BufferSequence, Value>::begin (buffers);
-    }
-
-    template <class BufferSequence, class Value = std::uint8_t>
-    static
-    boost::asio::buffers_iterator<BufferSequence, Value>
-    buffers_end (BufferSequence const& buffers)
-    {
-        return boost::asio::buffers_iterator<
-            BufferSequence, Value>::end (buffers);
-    }
-
-        // Encodes the size and type into a header at the beginning of buf
-    //
-    void encodeHeader (unsigned size, int type);
-
     std::vector <uint8_t> mBuffer;
-
     std::size_t mCategory;
 };
 

src/ripple/overlay/Overlay.h

@@ -32,6 +32,7 @@
 #include <type_traits>
 #include <boost/asio/buffer.hpp>
 #include <boost/asio/ip/tcp.hpp>
+#include <boost/optional.hpp>
 #include <functional>
 
 namespace boost { namespace asio { namespace ssl { class context; } } }
@@ -71,6 +72,7 @@ class Overlay
         beast::IP::Address public_ip;
         int ipLimit = 0;
         std::uint32_t crawlOptions = 0;
+        boost::optional<std::uint32_t> networkID;
     };
 
     using PeerSequence = std::vector <std::shared_ptr<Peer>>;

src/ripple/overlay/Peer.h

@@ -56,7 +56,7 @@ class Peer
 
     virtual
     void
-    send (Message::pointer const& m) = 0;
+    send (std::shared_ptr<Message> const& m) = 0;
 
     virtual
     beast::IP::Endpoint
@@ -105,7 +105,6 @@ class Peer
     virtual bool hasShard (std::uint32_t shardIndex) const = 0;
     virtual bool hasTxSet (uint256 const& hash) const = 0;
     virtual void cycleStatus () = 0;
-    virtual bool supportsVersion (int version) = 0;
     virtual bool hasRange (std::uint32_t uMin, std::uint32_t uMax) = 0;
 };