Merge branch 'ripple:develop' into develop

Builds/CMake/RippledCore.cmake

@@ -420,6 +420,11 @@ target_sources (rippled PRIVATE
   src/ripple/app/tx/impl/DepositPreauth.cpp
   src/ripple/app/tx/impl/Escrow.cpp
   src/ripple/app/tx/impl/InvariantCheck.cpp
+  src/ripple/app/tx/impl/NFTokenAcceptOffer.cpp
+  src/ripple/app/tx/impl/NFTokenBurn.cpp
+  src/ripple/app/tx/impl/NFTokenCancelOffer.cpp
+  src/ripple/app/tx/impl/NFTokenCreateOffer.cpp
+  src/ripple/app/tx/impl/NFTokenMint.cpp
   src/ripple/app/tx/impl/OfferStream.cpp
   src/ripple/app/tx/impl/PayChan.cpp
   src/ripple/app/tx/impl/Payment.cpp
@@ -432,6 +437,7 @@ target_sources (rippled PRIVATE
   src/ripple/app/tx/impl/Transactor.cpp
   src/ripple/app/tx/impl/apply.cpp
   src/ripple/app/tx/impl/applySteps.cpp
+  src/ripple/app/tx/impl/details/NFTokenUtils.cpp
   #[===============================[
      main sources:
        subdir: basics (partial)
@@ -593,6 +599,7 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/handlers/LogLevel.cpp
   src/ripple/rpc/handlers/LogRotate.cpp
   src/ripple/rpc/handlers/Manifest.cpp
+  src/ripple/rpc/handlers/NFTOffers.cpp
   src/ripple/rpc/handlers/NodeToShard.cpp
   src/ripple/rpc/handlers/NoRippleCheck.cpp
   src/ripple/rpc/handlers/OwnerInfo.cpp
@@ -687,6 +694,9 @@ if (tests)
     src/test/app/LoadFeeTrack_test.cpp
     src/test/app/Manifest_test.cpp
     src/test/app/MultiSign_test.cpp
+    src/test/app/NFToken_test.cpp
+    src/test/app/NFTokenBurn_test.cpp
+    src/test/app/NFTokenDir_test.cpp
     src/test/app/OfferStream_test.cpp
     src/test/app/Offer_test.cpp
     src/test/app/OversizeMeta_test.cpp
@@ -836,6 +846,7 @@ if (tests)
     src/test/jtx/impl/sig.cpp
     src/test/jtx/impl/tag.cpp
     src/test/jtx/impl/ticket.cpp
+    src/test/jtx/impl/token.cpp
     src/test/jtx/impl/trust.cpp
     src/test/jtx/impl/txflags.cpp
     src/test/jtx/impl/utility.cpp

Builds/CMake/deps/Nudb.cmake

@@ -18,14 +18,14 @@ if (is_root_project) # NuDB not needed in the case of xrpl_core inclusion build
     message (STATUS "Pausing to download NuDB...")
     FetchContent_Populate(nudb_src)
   endif()
-endif ()
 
-file(TO_CMAKE_PATH "${nudb_src_SOURCE_DIR}" nudb_src_SOURCE_DIR)
-# specify as system includes so as to avoid warnings
-target_include_directories (nudb SYSTEM INTERFACE ${nudb_src_SOURCE_DIR}/include)
-target_link_libraries (nudb
-  INTERFACE
-    Boost::thread
-    Boost::system)
-add_library (NIH::nudb ALIAS nudb)
-target_link_libraries (ripple_libs INTERFACE NIH::nudb)
+  file(TO_CMAKE_PATH "${nudb_src_SOURCE_DIR}" nudb_src_SOURCE_DIR)
+  # specify as system includes so as to avoid warnings
+  target_include_directories (nudb SYSTEM INTERFACE ${nudb_src_SOURCE_DIR}/include)
+  target_link_libraries (nudb
+    INTERFACE
+      Boost::thread
+      Boost::system)
+  add_library (NIH::nudb ALIAS nudb)
+  target_link_libraries (ripple_libs INTERFACE NIH::nudb)
+endif ()

Builds/containers/gitlab-ci/push_to_artifactory.sh

@@ -15,15 +15,17 @@ cd build/dpkg/packages
 CURLARGS="-sk -X${action} -urippled:${ARTIFACTORY_DEPLOY_KEY_RIPPLED}"
 RIPPLED_PKG=$(ls rippled_*.deb)
 RIPPLED_DEV_PKG=$(ls rippled-dev_*.deb)
+RIPPLED_REPORTING_PKG=$(ls rippled-reporting_*.deb)
 RIPPLED_DBG_PKG=$(ls rippled-dbgsym_*.deb)
+RIPPLED_REPORTING_DBG_PKG=$(ls rippled-reporting-dbgsym_*.deb)
 # TODO - where to upload src tgz?
 RIPPLED_SRC=$(ls rippled_*.orig.tar.gz)
 DEB_MATRIX=";deb.component=${COMPONENT};deb.architecture=amd64"
 for dist in stretch buster bullseye bionic focal jammy; do
     DEB_MATRIX="${DEB_MATRIX};deb.distribution=${dist}"
 done
 echo "{ \"debs\": {" > "${TOPDIR}/files.info"
-for deb in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ; do
+for deb in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ${RIPPLED_REPORTING_PKG} ${RIPPLED_REPORTING_DBG_PKG}; do
     # first item doesn't get a comma separator
     if [ $deb != $RIPPLED_PKG ] ; then
         echo "," >> "${TOPDIR}/files.info"
@@ -48,10 +50,11 @@ cd build/rpm/packages
 RIPPLED_PKG=$(ls rippled-[0-9]*.x86_64.rpm)
 RIPPLED_DEV_PKG=$(ls rippled-devel*.rpm)
 RIPPLED_DBG_PKG=$(ls rippled-debuginfo*.rpm)
+RIPPLED_REPORTING_PKG=$(ls rippled-reporting*.rpm)
 # TODO - where to upload src rpm ?
 RIPPLED_SRC=$(ls rippled-[0-9]*.src.rpm)
 echo "\"rpms\": {" >> "${TOPDIR}/files.info"
-for rpm in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ; do
+for rpm in ${RIPPLED_PKG} ${RIPPLED_DEV_PKG} ${RIPPLED_DBG_PKG} ${RIPPLED_REPORTING_PKG}; do
     # first item doesn't get a comma separator
     if [ $rpm != $RIPPLED_PKG ] ; then
         echo "," >> "${TOPDIR}/files.info"

Builds/containers/gitlab-ci/smoketest.sh

@@ -61,7 +61,11 @@ if [ "${pkgtype}" = "dpkg" ] ; then
 else
     yum -y update
     if [ "${install_from}" = "repo" ] ; then
-        yum -y install yum-utils coreutils util-linux
+        pkgs=("yum-utils coreutils util-linux")
+        if [ "$ID" = "rocky" ]; then
+            pkgs="${pkgs[@]/coreutils}"
+        fi
+        yum install -y $pkgs
         REPOFILE="/etc/yum.repos.d/artifactory.repo"
         echo "[Artifactory]" > ${REPOFILE}
         echo "name=Artifactory" >> ${REPOFILE}

RELEASENOTES.md

@@ -13,6 +13,68 @@ Have new ideas? Need help with setting up your node? Come visit us [here](https:
 
 # Releases
 
+## Version 1.9.0
+This is the 1.9.0 release of `rippled`, the reference implementation of the XRP Ledger protocol. This release brings several features and improvements.
+
+### New and Improved Features
+- **Introduce NFT support (XLS020):** This release introduces support for non-fungible tokens, currently available to the developer community for broader review and testing.  Developers can create applications that allow users to mint, transfer, and ultimately burn (if desired) NFTs on the XRP Ledger. You can try out the new NFT transactions using the [nft-devnet](https://xrpl.org/xrp-testnet-faucet.html). Note that some fields and error codes from earlier releases of the supporting code have been refactored for this release, shown in the Code Refactoring section, below. [70779f](https://github.com/ripple/rippled/commit/70779f6850b5f33cdbb9cf4129bc1c259af0013e)
+
+- **Simplify the Job Queue:** This is a refactor aimed at cleaning up and simplifying the existing job queue. Currently, all jobs are canceled at the same time and in the same way, so this commit removes the unnecessary per-job cancellation token. [#3656](https://github.com/ripple/rippled/pull/3656)
+
+- **Optimize trust line caching:** The existing trust line caching code was suboptimal in that it stored redundant information, pinned SLEs into memory, and required multiple memory allocations per cached object. This commit eliminates redundant data, reduces the size of cached objects and unpinning SLEs from memory, and uses value types to avoid the need for `std::shared_ptr`. As a result of these changes, the effective size of a cached object includes the overhead of the memory allocator, and the `std::shared_ptr` should be reduced by at least 64 bytes. This is significant, as there can easily be tens of millions of these objects. [4d5459](https://github.com/ripple/rippled/commit/4d5459d041da8f5a349c5f458d664e5865e1f1b5)
+
+- **Incremental improvements to pathfinding memory usage:** This commit aborts background pathfinding when closed or disconnected, exits the pathfinding job thread if there are no requests left, does not create the path find a job if there are no requests, and refactors to remove the circular dependency between InfoSub and PathRequest. [#4111](https://github.com/ripple/rippled/pull/4111)
+
+- **Improve deterministic transaction sorting in TxQ:** This commit ensures that transactions with the same fee level are sorted by TxID XORed with the parent ledger hash, the TxQ is re-sorted after every ledger, and attempts to future-proof the TxQ tie-breaking test. [#4077](https://github.com/ripple/rippled/pull/4077)
+
+- **Improve stop signaling for Application:** [34ca45](https://github.com/ripple/rippled/commit/34ca45713244d0defc39549dd43821784b2a5c1d)
+
+- **Eliminate SHAMapInnerNode lock contention:** The `SHAMapInnerNode` class had a global mutex to protect the array of node children. Profiling suggested that around 4% of all attempts to lock the global would block. This commit removes that global mutex, and replaces it with a new per-node 16-way spinlock (implemented so as not to affect the size of an inner node object), effectively eliminating the lock contention. [1b9387](https://github.com/ripple/rippled/commit/1b9387eddc1f52165d3243d2ace9be0c62495eea)
+
+- **Improve ledger-fetching logic:** When fetching ledgers, the existing code would isolate the peer that sent the most useful responses, and issue follow-up queries only to that peer. This commit increases the query aggressiveness, and changes the mechanism used to select which peers to issue follow-up queries to so as to more evenly spread the load among those peers that provided useful responses. [48803a](https://github.com/ripple/rippled/commit/48803a48afc3bede55d71618c2ee38fd9dbfd3b0)
+
+- **Simplify and improve order book tracking:** The order book tracking code would use `std::shared_ptr` to track the lifetime of objects. This commit changes the logic to eliminate the overhead of `std::shared_ptr` by using value types, resulting in significant memory savings. [b9903b](https://github.com/ripple/rippled/commit/b9903bbcc483a384decf8d2665f559d123baaba2)
+
+- **Negative cache support for node store:** This commit allows the cache to service requests for nodes that were previously looked up but not found, reducing the need to perform I/O in several common scenarios. [3eb8aa](https://github.com/ripple/rippled/commit/3eb8aa8b80bd818f04c99cee2cfc243192709667)
+
+- **Improve asynchronous database handlers:** This commit optimizes the way asynchronous node store operations are processed, both by reducing the number of times locks are held and by minimizing the number of memory allocations and data copying. [6faaa9](https://github.com/ripple/rippled/commit/6faaa91850d6b2eb9fbf16c1256bf7ef11ac4646)
+
+- **Cleanup AcceptedLedger and AcceptedLedgerTx:** This commit modernizes the `AcceptedLedger` and `AcceptedLedgerTx` classes, reduces their memory footprint, and reduces unnecessary dynamic memory allocations. [8f5868](https://github.com/ripple/rippled/commit/8f586870917818133924bf2e11acab5321c2b588)
+
+### Code Refactoring
+
+This release includes name changes in the NFToken API for SFields, RPC return labels, and error codes for clarity and consistency. To refactor your code, migrate the names of these items to the new names as listed below.
+
+#### `SField` name changes:
+* `TokenTaxon -> NFTokenTaxon`
+* `MintedTokens -> MintedNFTokens`
+* `BurnedTokens -> BurnedNFTokens`
+* `TokenID -> NFTokenID`
+* `TokenOffers -> NFTokenOffers`
+* `BrokerFee -> NFTokenBrokerFee`
+* `Minter -> NFTokenMinter`
+* `NonFungibleToken -> NFToken`
+* `NonFungibleTokens -> NFTokens`
+* `BuyOffer -> NFTokenBuyOffer`
+* `SellOffer -> NFTokenSellOffer`
+* `OfferNode -> NFTokenOfferNode`
+
+#### RPC return labels
+* `tokenid -> nft_id`
+* `index -> nft_offer_index`
+
+#### Error codes
+* `temBAD_TRANSFER_FEE -> temBAD_NFTOKEN_TRANSFER_FEE`
+* `tefTOKEN_IS_NOT_TRANSFERABLE -> tefNFTOKEN_IS_NOT_TRANSFERABLE`
+* `tecNO_SUITABLE_PAGE -> tecNO_SUITABLE_NFTOKEN_PAGE`
+* `tecBUY_SELL_MISMATCH -> tecNFTOKEN_BUY_SELL_MISMATCH`
+* `tecOFFER_TYPE_MISMATCH -> tecNFTOKEN_OFFER_TYPE_MISMATCH`
+* `tecCANT_ACCEPT_OWN_OFFER -> tecCANT_ACCEPT_OWN_NFTOKEN_OFFER`
+
+
+### Bug Fixes
+- **Fix deletion of orphan node store directories:** Orphaned node store directories should only be deleted if the proper node store directories are confirmed to exist. [06e87e](https://github.com/ripple/rippled/commit/06e87e0f6add5b880d647e14ab3d950decfcf416)
+
 ## Version 1.8.5
 This is the 1.8.5 release of `rippled`, the reference implementation of the XRP Ledger protocol. This release includes fixes and updates for stability and security, and improvements to build scripts. There are no user-facing API or protocol changes in this release.
 

src/ripple/app/misc/NetworkOPs.cpp

@@ -1178,6 +1178,7 @@ NetworkOPsImp::processTransaction(
     if ((newFlags & SF_BAD) != 0)
     {
         // cached bad
+        JLOG(m_journal.warn()) << transaction->getID() << ": cached bad!\n";
         transaction->setStatus(INVALID);
         transaction->setResult(temBAD_SIGNATURE);
         return;

src/ripple/app/tx/impl/CancelOffer.cpp

@@ -27,8 +27,7 @@ namespace ripple {
 NotTEC
 CancelOffer::preflight(PreflightContext const& ctx)
 {
-    auto const ret = preflight1(ctx);
-    if (!isTesSuccess(ret))
+    if (auto const ret = preflight1(ctx); !isTesSuccess(ret))
         return ret;
 
     auto const uTxFlags = ctx.tx.getFlags();

src/ripple/app/tx/impl/CashCheck.cpp

@@ -125,22 +125,13 @@ CashCheck::preclaim(PreclaimContext const& ctx)
             return tecDST_TAG_NEEDED;
         }
     }
+
+    if (hasExpired(ctx.view, sleCheck->at(~sfExpiration)))
     {
-        using duration = NetClock::duration;
-        using timepoint = NetClock::time_point;
-        auto const optExpiry = sleCheck->at(~sfExpiration);
-
-        // Expiration is defined in terms of the close time of the parent
-        // ledger, because we definitively know the time that it closed but
-        // we do not know the closing time of the ledger that is under
-        // construction.
-        if (optExpiry &&
-            (ctx.view.parentCloseTime() >= timepoint{duration{*optExpiry}}))
-        {
-            JLOG(ctx.j.warn()) << "Cashing a check that has already expired.";
-            return tecEXPIRED;
-        }
+        JLOG(ctx.j.warn()) << "Cashing a check that has already expired.";
+        return tecEXPIRED;
     }
+
     {
         // Preflight verified exactly one of Amount or DeliverMin is present.
         // Make sure the requested amount is reasonable.

src/ripple/app/tx/impl/Change.cpp

@@ -180,7 +180,7 @@ Change::applyAmendment()
         // This amendment now has a majority
         newMajorities.push_back(STObject(sfMajority));
         auto& entry = newMajorities.back();
-        entry.emplace_back(STHash256(sfAmendment, amendment));
+        entry.emplace_back(STUInt256(sfAmendment, amendment));
         entry.emplace_back(STUInt32(
             sfCloseTime, view().parentCloseTime().time_since_epoch().count()));
 

src/ripple/app/tx/impl/CreateCheck.cpp

@@ -146,21 +146,10 @@ CreateCheck::preclaim(PreclaimContext const& ctx)
             }
         }
     }
+    if (hasExpired(ctx.view, ctx.tx[~sfExpiration]))
     {
-        using duration = NetClock::duration;
-        using timepoint = NetClock::time_point;
-        auto const optExpiry = ctx.tx[~sfExpiration];
-
-        // Expiration is defined in terms of the close time of the parent
-        // ledger, because we definitively know the time that it closed but
-        // we do not know the closing time of the ledger that is under
-        // construction.
-        if (optExpiry &&
-            (ctx.view.parentCloseTime() >= timepoint{duration{*optExpiry}}))
-        {
-            JLOG(ctx.j.warn()) << "Creating a check that has already expired.";
-            return tecEXPIRED;
-        }
+        JLOG(ctx.j.warn()) << "Creating a check that has already expired.";
+        return tecEXPIRED;
     }
     return tesSUCCESS;
 }

src/ripple/app/tx/impl/CreateOffer.cpp

@@ -42,8 +42,7 @@ CreateOffer::makeTxConsequences(PreflightContext const& ctx)
 NotTEC
 CreateOffer::preflight(PreflightContext const& ctx)
 {
-    auto const ret = preflight1(ctx);
-    if (!isTesSuccess(ret))
+    if (auto const ret = preflight1(ctx); !isTesSuccess(ret))
         return ret;
 
     auto& tx = ctx.tx;
@@ -173,14 +172,7 @@ CreateOffer::preclaim(PreclaimContext const& ctx)
         return temBAD_SEQUENCE;
     }
 
-    using d = NetClock::duration;
-    using tp = NetClock::time_point;
-    auto const expiration = ctx.tx[~sfExpiration];
-
-    // Expiration is defined in terms of the close time of the parent ledger,
-    // because we definitively know the time that it closed but we do not
-    // know the closing time of the ledger that is under construction.
-    if (expiration && (ctx.view.parentCloseTime() >= tp{d{*expiration}}))
+    if (hasExpired(ctx.view, ctx.tx[~sfExpiration]))
     {
         // Note that this will get checked again in applyGuts, but it saves
         // us a call to checkAcceptAsset and possible false negative.
@@ -951,13 +943,8 @@ CreateOffer::applyGuts(Sandbox& sb, Sandbox& sbCancel)
     }
 
     auto const expiration = ctx_.tx[~sfExpiration];
-    using d = NetClock::duration;
-    using tp = NetClock::time_point;
 
-    // Expiration is defined in terms of the close time of the parent ledger,
-    // because we definitively know the time that it closed but we do not
-    // know the closing time of the ledger that is under construction.
-    if (expiration && (sb.parentCloseTime() >= tp{d{*expiration}}))
+    if (hasExpired(sb, expiration))
     {
         // If the offer has expired, the transaction has successfully
         // done nothing, so short circuit from here.