From 7736af31277735359f1cdf61341ca5658e2f0388 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 25 Jul 2024 12:42:42 +0000 Subject: [PATCH 01/74] initial fee estimation logic + a python notebook detailing a challenge --- mining/src/block_template/selector.rs | 2 +- mining/src/feerate/fee_estimation.ipynb | 369 ++++++++++++++++++++++++ mining/src/feerate/mod.rs | 78 +++++ mining/src/lib.rs | 1 + 4 files changed, 449 insertions(+), 1 deletion(-) create mode 100644 mining/src/feerate/fee_estimation.ipynb create mode 100644 mining/src/feerate/mod.rs diff --git a/mining/src/block_template/selector.rs b/mining/src/block_template/selector.rs index a55ecb93d..0059ad92e 100644 --- a/mining/src/block_template/selector.rs +++ b/mining/src/block_template/selector.rs @@ -18,7 +18,7 @@ use kaspa_consensus_core::{ /// candidate transactions should be. A smaller alpha makes the distribution /// more uniform. ALPHA is used when determining a candidate transaction's /// initial p value. -const ALPHA: i32 = 3; +pub(crate) const ALPHA: i32 = 3; /// REBALANCE_THRESHOLD is the percentage of candidate transactions under which /// we don't rebalance. Rebalancing is a heavy operation so we prefer to avoid diff --git a/mining/src/feerate/fee_estimation.ipynb b/mining/src/feerate/fee_estimation.ipynb new file mode 100644 index 000000000..67d69e511 --- /dev/null +++ b/mining/src/feerate/fee_estimation.ipynb @@ -0,0 +1,369 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "ALPHA = 3.0" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [], + "source": [ + "feerates = [1.0, 1.1, 1.2]*10 + [1.5]*3000 + [2]*3000 + [2.1]*3000 + [3, 4, 5]*10\n", + "# feerates = [1.0, 1.1, 1.2] + [1.1]*100 + [1.2]*100 + [1.3]*100 # + [3, 4, 5, 100]" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total mempool weight: 64108.589999995806\n" + ] + } + ], + "source": [ + "total_weight = sum(np.array(feerates)**ALPHA)\n", + "print('Total mempool weight: ', total_weight)" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [], + "source": [ + "avg_mass = 2000\n", + "bps = 1\n", + "block_mass_limit = 500_000\n", + "network_mass_rate = bps * block_mass_limit" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inclusion time: 0.004\n" + ] + } + ], + "source": [ + "print('Inclusion time: ', avg_mass/network_mass_rate)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [], + "source": [ + "class FeerateBucket:\n", + " def __init__(self, feerate, estimated_seconds):\n", + " self.feerate = feerate\n", + " self.estimated_seconds = estimated_seconds\n", + " \n", + "\n", + "class FeerateEstimations:\n", + " def __init__(self, low_bucket, normal_bucket, priority_bucket):\n", + " self.low_bucket = low_bucket \n", + " self.normal_bucket = normal_bucket\n", + " self.priority_bucket = priority_bucket\n", + " \n", + " def __repr__(self):\n", + " return 'Feerates:\\t{}, {}, {} \\nTimes:\\t\\t{}, {}, {}'.format(self.low_bucket.feerate, \n", + " self.normal_bucket.feerate,\n", + " self.priority_bucket.feerate, \n", + " self.low_bucket.estimated_seconds, \n", + " self.normal_bucket.estimated_seconds, \n", + " self.priority_bucket.estimated_seconds)\n", + " def feerates(self):\n", + " return np.array([\n", + " self.low_bucket.feerate, \n", + " self.normal_bucket.feerate,\n", + " self.priority_bucket.feerate\n", + " ])\n", + " \n", + " def times(self):\n", + " return np.array([\n", + " self.low_bucket.estimated_seconds, \n", + " self.normal_bucket.estimated_seconds,\n", + " self.priority_bucket.estimated_seconds\n", + " ])\n", + " \n", + "class FeerateEstimator:\n", + " \"\"\"\n", + " `total_weight`: The total probability weight of all current mempool ready \n", + " transactions, i.e., Σ_{tx in mempool}(tx.fee/tx.mass)^ALPHA\n", + " \n", + " 'inclusion_time': The amortized time between transactions given the current \n", + " transaction masses present in the mempool, i.e., the inverse \n", + " of the transaction inclusion rate. For instance, if the average \n", + " transaction mass is 2500 grams, the block mass limit is 500,000\n", + " and the network has 10 BPS, then this number would be 1/2000 seconds.\n", + " \"\"\"\n", + " def __init__(self, total_weight, inclusion_time):\n", + " self.total_weight = total_weight\n", + " self.inclusion_time = inclusion_time\n", + "\n", + " \"\"\"\n", + " Feerate to time function: f(feerate) = inclusion_time * (1/p(feerate))\n", + " where p(feerate) = feerate^ALPHA/(total_weight + feerate^ALPHA) represents \n", + " the probability function for drawing `feerate` from the mempool\n", + " in a single trial. The inverse 1/p is the expected number of trials until\n", + " success (with repetition), thus multiplied by inclusion_time it provides an\n", + " approximation to the overall expected waiting time\n", + " \"\"\"\n", + " def feerate_to_time(self, feerate):\n", + " c1, c2 = self.inclusion_time, self.total_weight\n", + " return c1 * c2 / feerate**ALPHA + c1\n", + "\n", + " \"\"\"\n", + " The inverse function of `feerate_to_time`\n", + " \"\"\"\n", + " def time_to_feerate(self, time):\n", + " c1, c2 = self.inclusion_time, self.total_weight\n", + " return ((c1 * c2 / time) / (1 - c1 / time))**(1 / ALPHA)\n", + " \n", + " \"\"\"\n", + " The antiderivative function of \n", + " feerate_to_time excluding the constant shift `+ c1`\n", + " \"\"\"\n", + " def feerate_to_time_antiderivative(self, feerate):\n", + " c1, c2 = self.inclusion_time, self.total_weight\n", + " return c1 * c2 / (-2.0 * feerate**(ALPHA - 1))\n", + " \n", + " \"\"\"\n", + " Returns the feerate value for which the integral area is `frac` of the total area.\n", + " See figures below for illustration\n", + " \"\"\"\n", + " def quantile(self, lower, upper, frac):\n", + " c1, c2 = self.inclusion_time, self.total_weight\n", + " z1 = self.feerate_to_time_antiderivative(lower)\n", + " z2 = self.feerate_to_time_antiderivative(upper)\n", + " z = frac * z2 + (1.0 - frac) * z1\n", + " return ((c1 * c2) / (-2 * z))**(1.0 / (ALPHA - 1.0))\n", + " \n", + " def calc_estimations(self):\n", + " # Choose `high` such that it provides sub-second waiting time\n", + " high = self.time_to_feerate(1.0)\n", + " # Choose `low` feerate such that it provides sub-hour waiting time AND it covers (at least) the 0.25 quantile\n", + " low = max(self.time_to_feerate(3600.0), self.quantile(1.0, high, 0.25))\n", + " # Choose `mid` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.5 quantile\n", + " mid = max(self.time_to_feerate(60.0), self.quantile(1.0, high, 0.5))\n", + " # low = self.quantile(1.0, high, 0.25)\n", + " # mid = self.quantile(1.0, high, 0.5)\n", + " return FeerateEstimations(\n", + " FeerateBucket(low, self.feerate_to_time(low)),\n", + " FeerateBucket(mid, self.feerate_to_time(mid)),\n", + " FeerateBucket(high, self.feerate_to_time(high)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Feerate estimation\n", + "\n", + "The figure below illustrates the estimator selectoin. We first estimate the `feerate_to_time` function and then select 3 meaningfull points by analyzing the curve and its integral (see `calc_estimations`). " + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHTlJREFUeJzt3XuUnHWd5/H3t+7V9066k3TSucEE5GICoY2szDqMDCqKgq7L4Jxx0OOezDniqmfnnDnq2XVcj5z1rKvs4io7ICiut2UUVxzwwkZGiCChgxDIBUjIpTu37tz7kr5U13f/qKdDJ3SnO91Vebqe+rzOqfM8z6+ep+pbXD7P07/6Pb8yd0dERKIrFnYBIiJSWgp6EZGIU9CLiEScgl5EJOIU9CIiEaegFxGJOAW9iEjEKehFRCJOQS8iEnGJsAsAaGpq8mXLloVdhohIWdm4ceMhd2+ebL9ZEfTLli2jvb097DJERMqKme2eyn7quhERiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4so66F8+0MNXf72NY/1DYZciIjJrlXXQ7z7cxzcf30HHkZNhlyIiMmtNGvRmttjMHjezrWa22cw+HbR/0cz2mtnzweM9Y475nJltN7OXzexdpSp+fl0GgIMnBkr1FiIiZW8qUyDkgL9z9+fMrBbYaGaPBc/d6e7/bezOZnYpcCtwGbAQ+H9mdpG7jxSzcBgT9D0KehGRiUx6Re/u+939uWC9B9gKLDrLITcBP3b3QXffCWwH1hSj2DM11aQw4OCJwVK8vIhIJJxTH72ZLQOuBJ4Jmj5pZpvM7H4zawzaFgEdYw7r5OwnhmlLxGM0VCXpUteNiMiEphz0ZlYD/BT4jLufAO4GLgSuAPYDXxvddZzDfZzXW2tm7WbW3t3dfc6Fj6rPJtVHLyJyFlMKejNLUgj5H7j7QwDuftDdR9w9D9zL690zncDiMYe3AvvOfE13v8fd29y9rbl50umUJ1SfTXJAQS8iMqGpjLox4D5gq7t/fUx7y5jdPgC8FKw/DNxqZmkzWw6sADYUr+TTNVQl1UcvInIWUxl1cw3wEeBFM3s+aPs88GEzu4JCt8wu4G8B3H2zmT0IbKEwYuf2Uoy4GdVQleJI3xBDuTypRFnfFiAiUhKTBr27r2f8fvdHz3LMHcAdM6hryhqqkgB09w6yqCF7Pt5SRKSslP0lcGMQ9PpCVkRkfGUf9A3ZFICGWIqITKD8g/7UFb2+kBURGU/ZB31NJkHM1HUjIjKRsg/6mBk16YSu6EVEJlD2QQ9QnU7QpYnNRETGFYmgr0rF2X9MQS8iMp5IBH11OqFpEEREJhCJoK9JJ+gdzNE/lAu7FBGRWScSQV+bKdzgu0/dNyIibxCNoE8XxtLvO6bfjhUROVM0gv7UFb2CXkTkTJEI+up0AkNBLyIynkgEfTxm1GYS7DuuPnoRkTNFIuihMBWCruhFRN4oOkGfSrD3qIJeRORM0Qn6TIL9xwdwf8PvkIuIVLTIBH1tJsnQSJ7DfUNhlyIiMqtEKOg1xFJEZDzRCfq07o4VERlPdII+o7tjRUTGE5mgzyRjJOOmoBcROUNkgt7MqMsk2a+bpkREThOZoAeoTsfpPNofdhkiIrNKpIK+NpOkUzdNiYicJlJBX5dNcrhvSD9AIiIyRqSCvj4YeaOrehGR10Ur6LOFoN9zWP30IiKjIhX0ddnCTVMd+kJWROSUSAV9NhknlYix54iCXkRk1KRBb2aLzexxM9tqZpvN7NNB+xwze8zMXg2WjUG7mdldZrbdzDaZ2epSf4gxtVKfTdKhoBcROWUqV/Q54O/c/RLgauB2M7sU+Cywzt1XAOuCbYAbgBXBYy1wd9GrPovadEJX9CIiY0wa9O6+392fC9Z7gK3AIuAm4IFgtweAm4P1m4DvecEfgAYzayl65ROoyybZc6Rf89KLiATOqY/ezJYBVwLPAPPdfT8UTgbAvGC3RUDHmMM6g7YzX2utmbWbWXt3d/e5Vz6B+mySgWHNSy8iMmrKQW9mNcBPgc+4+4mz7TpO2xsur939Hndvc/e25ubmqZYxqdGRN+q+EREpmFLQm1mSQsj/wN0fCpoPjnbJBMuuoL0TWDzm8FZgX3HKndzoTVP6QlZEpGAqo24MuA/Y6u5fH/PUw8BtwfptwM/HtP9NMPrmauD4aBfP+VCXVdCLiIyVmMI+1wAfAV40s+eDts8DXwEeNLOPA3uAfxs89yjwHmA70A98rKgVTyIZj1GjkTciIqdMGvTuvp7x+90Brhtnfwdun2FdM1KXSbBb0yCIiAARuzN2VH1VktcO9YVdhojIrBDJoG+sStHdM0jfoKYrFhGJZNA3VBW+kN2pq3oRkWgGfWNVClDQi4hARIO+IasrehGRUZEM+kQ8Rn02qaAXESGiQQ+FOW9e6+4NuwwRkdBFNugbsoUhlprFUkQqXXSDvipJz0COI5rFUkQqXGSDXiNvREQKIhv0o2PpdYesiFS6yAZ9XSZJ3ExX9CJS8SIb9LGY0VCVZHuXRt6ISGWLbNADNFanePlAT9hliIiEKtJBP7c6RceRfgaGR8IuRUQkNJEO+neNPMGTqU+RvmMu3Hk5bHow7JJERM67qfzCVFm6uOuX/EX310jFBgsNxzvgF58qrK+8JbzCRETOs8he0f/pnm+R8sHTG4dPwrovhVOQiEhIIhv0tYMHx3/ieOf5LUREJGSRDfqe9Pzxn6hvPb+FiIiELLJBv37JJxiOZU5vTGbhui+EU5CISEgi+2Xsy/NuAOCtO/8njcPd5GoXknrnF/VFrIhUnMgGPRTC/qmqd/D9Z/Zw582r+MBKdduISOWJbNfNqIaqFHEzXj6gqRBEpDJFPujjMWNuTYptB06EXYqISCgiH/QAc2tSvLT3eNhliIiEoiKCvrkmzaHeIbp6BsIuRUTkvKuMoK9NA7B1v2ayFJHKUxlBX1MI+s371H0jIpVn0qA3s/vNrMvMXhrT9kUz22tmzweP94x57nNmtt3MXjazd5Wq8HORTsZpyCbZsk9fyIpI5ZnKFf13gXeP036nu18RPB4FMLNLgVuBy4JjvmVm8WIVOxNza1JsVtCLSAWaNOjd/QngyBRf7ybgx+4+6O47ge3AmhnUVzTNNWl2HeqjbzAXdikiIufVTProP2lmm4KuncagbRHQMWafzqDtDcxsrZm1m1l7d3f3DMqYmubaNA5s008LikiFmW7Q3w1cCFwB7Ae+FrTbOPv6eC/g7ve4e5u7tzU3N0+zjKkbHXmzZb+6b0Skskwr6N39oLuPuHseuJfXu2c6gcVjdm0F9s2sxOKoSSeoSsV5sfNY2KWIiJxX0wp6M2sZs/kBYHREzsPArWaWNrPlwApgw8xKLA4zY15tmuc7FPQiUlkmnb3SzH4EXAs0mVkn8A/AtWZ2BYVumV3A3wK4+2YzexDYAuSA2919pDSln7v5dRk27DxC72COmnSkJ+4UETll0rRz9w+P03zfWfa/A7hjJkWVyoK6DA68tPc4V18wN+xyRETOi4q4M3bU/LrCL069oO4bEakgFRX02VScxqqk+ulFpKJUVNBDYZjlHxX0IlJBKi7oF9RlOHB8gK4TmrJYRCpDxQX9aD+9um9EpFJUXNDPq00TMwW9iFSOigv6RDzGvNoMz+6a6jxtIiLlreKCHmBhQ4bnO44xMDxr7uUSESmZCg36LMMjzqZO/eKUiERfxQY9oO4bEakIFRn02WScppoUG3Yq6EUk+ioy6AEW1GfYuPsoI/lxp8sXEYmMig36RQ1ZegdzbNUPkYhIxFVs0KufXkQqRcUGfV0mSX02ydM7DoddiohISVVs0AO0NmZ5asdhciP5sEsRESmZig76JXOq6B3M8eJejacXkeiq6KBvbSz0069/9VDIlYiIlE5FB31VKsG8ujTrtyvoRSS6KjroARY3VLFx91H6h3JhlyIiUhIK+jlZcnnnGd0lKyIRVfFBv6ghSyJm6qcXkciq+KBPxGMsasjy221dYZciIlISFR/0AMuaqtl5qI+dh/rCLkVEpOgU9MAFTdUArNt6MORKRESKT0EP1GWTNNekWbdV3TciEj0K+sDSuVVs2HWE4yeHwy5FRKSoFPSB5U3VjOSdJ17pDrsUEZGiUtAHFtRnqErF1U8vIpEzadCb2f1m1mVmL41pm2Nmj5nZq8GyMWg3M7vLzLab2SYzW13K4ospZsayudU8tvUgg7mRsMsRESmaqVzRfxd49xltnwXWufsKYF2wDXADsCJ4rAXuLk6Z58eK+TX0DY7w5Cu6eUpEomPSoHf3J4Az5we4CXggWH8AuHlM+/e84A9Ag5m1FKvYUlvcWEU2GeeRF/eHXYqISNFMt49+vrvvBwiW84L2RUDHmP06g7Y3MLO1ZtZuZu3d3bPjC9B4zFjeVM1jWw4yMKzuGxGJhmJ/GWvjtPl4O7r7Pe7e5u5tzc3NRS5j+i6aX0PvYI4nNfeNiETEdIP+4GiXTLAcvdOoE1g8Zr9WYN/0yzv/Wke7bzaVVdkiIhOabtA/DNwWrN8G/HxM+98Eo2+uBo6PdvGUi3jMuKC5mt9sOag56kUkEqYyvPJHwNPAxWbWaWYfB74CXG9mrwLXB9sAjwKvAduBe4FPlKTqErtkQR39QyP86qUDYZciIjJjicl2cPcPT/DUdePs68DtMy0qbAsbMjRUJfmn9k4+uLo17HJERGZEd8aOw8x404Jann7tMJ1H+8MuR0RkRhT0E7hkQR0ADz23N+RKRERmRkE/gbpsksWNWf6pvYNCj5SISHlS0J/FJS11dBw9yVM7DoddiojItCnoz2LFvBqqUnG+9/SusEsREZk2Bf1ZJOIxLmmp47EtB9l37GTY5YiITIuCfhIrF9XjDj98Zk/YpYiITIuCfhJ12STLm6r54YY9mqdeRMqSgn4KVrbWc6RviH9+oaxmcxARART0U7JkThVNNSn+1+92kM9rqKWIlBcF/RSYGVctaeTVrl4ef7lr8gNERGYRBf0UrZhfS302yd3/siPsUkREzomCforiMeOKxQ207z7Ks7vO/GVFEZHZS0F/Di5bWEdVKs431r0adikiIlOmoD8HyXiM1UsaeeLVQ2zYqat6ESkPCvpztLK1npp0gq/+epsmOxORsqCgP0fJeIy2ZY08u+soT+gHxEWkDCjop+HyhfXUZ5P8119t07h6EZn1FPTTEI8Zb10+h837TvDQH/XDJCIyuynop+lNC2ppqc/wlV9upXcwF3Y5IiITUtBPk5nx9hXNHOod4hu/1XBLEZm9FPQzsKA+w6Uttdz35E52HuoLuxwRkXEp6GfobRc2cVP899R86wr8iw1w5+Ww6cGwyxIROSURdgHlbvXxx7gucS/p/GCh4XgH/OJThfWVt4RXmIhIQFf0M/Sne75F2gdPbxw+Ceu+FE5BIiJnUNDPUO3gwfGfON55fgsREZmAgn6GetLzx3+ivvX8FiIiMgEF/QytX/IJhmOZ09pOeop9bX8fUkUiIqdT0M/Qy/Nu4LELP8+J9AIc41hqAV9gLR9tX8rAsH5MXETCN6NRN2a2C+gBRoCcu7eZ2Rzg/wDLgF3ALe5+dGZlzm4vz7uBl+fdcGp7+HAfrzy/j//4f1/iqx9aiZmFWJ2IVLpiXNH/ubtf4e5twfZngXXuvgJYF2xXlKVzq1mzfA4/2djJ957eHXY5IlLhStF1cxPwQLD+AHBzCd5j1rt6+RwuaKrmS7/YwtM7DoddjohUsJkGvQO/MbONZrY2aJvv7vsBguW8Gb5HWTIz3nnZfBqqknziBxs1RYKIhGamQX+Nu68GbgBuN7O3T/VAM1trZu1m1t7d3T3DMmandCLOe1e2MJjL85H7nqGrZyDskkSkAs0o6N19X7DsAn4GrAEOmlkLQLDsmuDYe9y9zd3bmpubZ1LGrNZYleJ9qxbSdWKQ2+7fQM/AcNgliUiFmXbQm1m1mdWOrgPvBF4CHgZuC3a7Dfj5TIssdwvqMrznzQt4+UAP/+6BdvqHNH+9iJw/M7minw+sN7MXgA3AI+7+K+ArwPVm9ipwfbBd8ZbOreadly5gw64jfPT+Z+nTj5WIyHky7XH07v4asGqc9sPAdTMpKqouXlALwK+3HOCj39nAdz+2huq0JhAVkdLSnbHn2cULann3ZQvYuPsof3XvHzjcOzj5QSIiM6CgD8FF82t5z5tb2LzvBB/81lPsPqyhlyJSOgr6kFzYXMMHVy+iu3eQm7/5e57bE+lZIkQkRAr6ELXUZ/nQ6lYc+Mt/fJofbdgTdkkiEkEK+pA1Vqf4y7bFLGzI8rmHXuRzD21iMKdZL0WkeBT0s0AmGef9qxbylmWN/GhDBx/45lNs7+oJuywRiQgF/SwRM+NtFzbxvpUt7Drcx3vvWs//fnoX7h52aSJS5hT0s8wFzTX81ZoltNRn+E8/38xHv/MsHUf6wy5LRMqYgn4Wqk4neP+qhVx7UTNP7zjM9Xf+jn/83Q6GR/JhlyYiZUhBP0uZGasWN/DXVy9hUUOW//LLbdz4jfWa215EzpmCfparzSS5ceVCblzZwv5jJ/nwvX/g4w88qy9rRWTKNNFKmbiwuYalc6p4vuMY6189xLu2Pcktb1nM7X9+Ia2NVWGXJyKzmIK+jCTiMdqWzeHShXVs2HmEB5/t4MH2Dv7N6kV84to/YVlTddglisgspKAvQ1WpBNdePI+rljaycfdRHnpuLz/Z2MmNKxfysWuWceWSxrBLFJFZREFfxmozSa69eB5vWTaH5/Yc5debD/DwC/tYuaiej16zjPeubCGdiIddpoiEzGbDDTltbW3e3t4+rWN/u+0gL3QcL3JF5Wkol2fr/hNs2nucI31DNFYl+cCVrXzoqlYuXVgXdnkiUmRmttHd2ybbT1f0EZJKxFi1uIGVrfXsOdLPS/tO8MDTu7j/9zt504JaPnRVK+9ftZB5dZmwSxWR80hBH0FmxtK51SydW83J4RFeOdDDtgM9fPmRrdzxyFZWL23khssX8K7LFrB4jkbsiESdgj7issk4qxY3sGpxA0f6htje1cuO7l6+/MhWvvzIVi5fWMc7LpnPn13UzKrWehJx3VohEjUK+goypzrFmuVzWLN8Dsf6h9jR3ceO7l6+8dtXuWvdq9RmEvzrFU382UXNvO3CJlobs5hZ2GWLyAwp6CtUQ1WKq5amuGppIwPDI+w50s/uw/08+cohHn3xAAAL6jKsWT6Htyyfw5plc1gxr4ZYTMEvUm4U9EImGeei+bVcNL8Wd+dw3xB7j55k77GTPL6ti4df2AdAfTbJqtZ6VrY28ObWela21rOgLqOrfpFZTkEvpzEzmmrSNNWkWbW4AXfnxECOvcdOsu/YSbYd6GH99kPkg1G5c6pTrGqt5/JF9ayYX8vF82tZ3lRNKqG+fpHZQkEvZ2Vm1GeT1GeTXNpSGIufG8nT3TtI14lBDvYM8OLe4/zule5T4R+PGcubqrk4+CvhT+bVsKypiqVzq6lJ6z85kfNN/9fJOUvEY7TUZ2mpz55qy43kOdo/zOG+QY70DXG4d4indhzi0Rf3M/aWvLnVKZY3FYZ+LptbxdKmapbOqaKlIUNTdVrfAYiUgIJeiiIRj9Fcm6a5Nn1a+/BInmP9wxzrH+LYyWGOnxzmwIkBXjnYw4mB3Gn7JuNGS32WhQ0ZFjZkWVifZWFDlpaGDC31GZpr0jRWpXQyEDlHCnopqeQEJwAonASOB+HfO5CjZzBHz8AwHUf62bLvBL2DuVPdQaPiZsytSZ16zeaawrJpzLKxOkljVYqGqqTm+hFBQS8hSsZjp774HU8+7/QN5egZyNE7mKN/aIT+oRx9gyP0DeY43DvEc0NHxz0hjMom4zRUFYK/sTpJQ1WKxmC7PlvYrs0kCo90ktpMgppgWycJiQoFvcxasZhRm0lSm0medT93Z2A4XzgJDI0wMDz6yJ9a7x/KcbR/iMFcDwPDI5wcGmGy6fxS8RjV6Ti1mSR12dNPBDXpBNlUnOpUgqpUnGwqXlgmC9uvt415PhnXnccSipIFvZm9G/gfQBz4trt/pVTvJZXNzMgGYTp3ise4O4O5wolgKJdnMJdnaCTPUC7/+nYuz+BI4fnegRxH+oYYzvnr+43kGZnoT4kJJONGNlmoNZ2Ik07EyCTjZJNx0skY6UTsVHthO1hPxEgnx6wn3rh/MhEjETOS8Rip8dYTMVLxwno8Zrr/IQybHoR1X4LjnVDfCtd9AVbeUvK3LUnQm1kc+CZwPdAJPGtmD7v7llK8n8i5MjMyyTiZ5My6Z0byTi6fZ3jEGR7JkwuWhYeTC5bD+Te25fKFE8WJgWGO9g+Rd2ck78FrOiMjheXo6xeTUeg6S8QLJ4Nk3EjEYiQTVjgZxAsnhdGTxOgyHosRj0EiFiMeMxIxIxYs37j9+kklPub509djk7xGoa5YrPD9TCxmxMyIGcRs9IRVGNI7tv3UI8a463EzLNiOW+E1Rl8vZpTmJLjpQfjFp2D4ZGH7eEdhG0oe9qW6ol8DbHf31wDM7MfATYCCXiKlEFpxSn17gLuTd06dHHKjJ4QRP3WyyXvhxJN3J593RtzJ5wmWfsby9fZTJ5jgPfJ5ZziXZ3B45NS+eS/s507hQbBv8BrukOeN++aD1yxHY08Ap50wTrW9flKI2+vrMePUCcl4ve37PZ9nvp88/U2GTxau8Ms06BcBHWO2O4G3luKN5lanWa7fShWZtUZPUu7+hhPH6IllJDgxjD35jN1vZMxJ5tQJBz9je8z6mPc8vT1YZ2r7TPR6p72vO/ng9caeCEdrGj2m2Q+N/w/oeGfJ/x2UKujH+7vntPO6ma0F1gIsWbJk2m80OgWviMisdmdrobvmTPWtJX/rUg0B6AQWj9luBfaN3cHd73H3Nndva25uLlEZIiKzxHVfgGT29LZkttBeYqUK+meBFWa23MxSwK3AwyV6LxGR2W/lLfC+u6B+MWCF5fvuKt9RN+6eM7NPAr+mMLzyfnffXIr3EhEpGytvOS/BfqaSjRVw90eBR0v1+iIiMjW6TU9EJOIU9CIiEaegFxGJOAW9iEjEKehFRCJOQS8iEnEKehGRiDP38KeWM7NuYPc0D28CJpgtKDKi/hn1+cqbPl94lrr7pHPIzIqgnwkza3f3trDrKKWof0Z9vvKmzzf7qetGRCTiFPQiIhEXhaC/J+wCzoOof0Z9vvKmzzfLlX0fvYiInF0UruhFROQsyjbozex+M+sys5fCrqUUzGyxmT1uZlvNbLOZfTrsmorJzDJmtsHMXgg+338Ou6ZSMLO4mf3RzP457FpKwcx2mdmLZva8mbWHXU+xmVmDmf3EzLYF/y/+q7Brmo6y7boxs7cDvcD33P3ysOspNjNrAVrc/TkzqwU2Aje7+5aQSysKMzOg2t17zSwJrAc+7e5/CLm0ojKz/wC0AXXufmPY9RSbme0C2twn+uXr8mZmDwBPuvu3g1/Lq3L3Y2HXda7K9ore3Z8AjoRdR6m4+353fy5Y7wG2AovCrap4vKA32EwGj/K86piAmbUC7wW+HXYtcu7MrA54O3AfgLsPlWPIQxkHfSUxs2XAlcAz4VZSXEG3xvNAF/CYu0fq8wH/Hfh7IB92ISXkwG/MbKOZrQ27mCK7AOgGvhN0v33bzKrDLmo6FPSznJnVAD8FPuPuJ8Kup5jcfcTdrwBagTVmFpkuODO7Eehy941h11Ji17j7auAG4PagSzUqEsBq4G53vxLoAz4bbknTo6CfxYK+658CP3D3h8Kup1SCP4f/BXh3yKUU0zXA+4M+7B8D7zCz74dbUvG5+75g2QX8DFgTbkVF1Ql0jvlL8ycUgr/sKOhnqeDLyvuAre7+9bDrKTYzazazhmA9C/wFsC3cqorH3T/n7q3uvgy4Ffitu/91yGUVlZlVBwMFCLo03glEZhScux8AOszs4qDpOqAsB0Mkwi5guszsR8C1QJOZdQL/4O73hVtVUV0DfAR4MejHBvi8uz8aYk3F1AI8YGZxChccD7p7JIcgRth84GeFaxISwA/d/VfhllR0/x74QTDi5jXgYyHXMy1lO7xSRESmRl03IiIRp6AXEYk4Bb2ISMQp6EVEIk5BLyIScQp6EZGIU9CLiEScgl5EJOL+P2hKqYG7wKxGAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Feerates:\t1.1499744606513134, 1.6228733928138133, 6.361686926992798 \n", + "Times:\t\t168.62498827393395, 59.999999999999986, 1.0000000000000004" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "estimator = FeerateEstimator(total_weight=total_weight, \n", + " inclusion_time=avg_mass/network_mass_rate)\n", + "\n", + "pred = estimator.calc_estimations()\n", + "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", + "y = estimator.feerate_to_time(x)\n", + "plt.figure()\n", + "plt.plot(x, y)\n", + "plt.fill_between(x, estimator.inclusion_time, y2=y, alpha=0.5)\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "pred" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Interpolating the original function using two of the points\n", + "\n", + "The code below reverse engineers the original curve using only 2 of the estimated points" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHS1JREFUeJzt3XlwXOWZ7/Hv04t2WYslG1uSsQFj9tiOMGQ8YZIwhG0qmCxzSSaEbOXcDLmVzORyb8i9U8nUrdRQxUwySyUwBLiBbFyGEMIkJIYQwpIQbBmMjTEGGxssy7ZkeZOstdXP/aOPHNmWrK1bRzr9+1R19em3z+l+muV3Xr399nvM3RERkeiKhV2AiIjkloJeRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRFwi7AIAampqfOHChWGXISIyo6xfv36/u9eOtt+0CPqFCxfS1NQUdhkiIjOKmb01lv00dCMiEnEKehGRiFPQi4hEnIJeRCTiFPQiIhGnoBcRiTgFvYhIxM3ooN+6t4Pb17zGoa6+sEsREZm2ZnTQv9V+lG8/tZ1dB7rDLkVEZNoaNejNrMHMnjKzLWa22cy+GLR/3cx2m9mG4HbNkGNuNbNtZrbVzK7MVfFzZxUBsO9IT67eQkRkxhvLEggp4Mvu/qKZlQPrzeyJ4Llvufs/Dt3ZzM4DbgDOB+YDvzazs919IJuFw5Cg71DQi4iMZNQevbvvcfcXg+0OYAtQd4pDrgMecPded98BbANWZKPYE9WUFWAG+4705uLlRUQiYVxj9Ga2EFgGvBA0fcHMNprZvWZWFbTVAbuGHNbMqU8ME5aIx6gpK6RVQzciIiMac9CbWRnwE+BL7n4EuAM4E1gK7AH+aXDXYQ73YV5vtZk1mVlTW1vbuAsfNHdWocboRUROYUxBb2ZJMiH/Q3d/GMDd97n7gLunge/yx+GZZqBhyOH1QMuJr+nud7l7o7s31taOupzyiOaWF2noRkTkFMYy68aAe4At7v7NIe3zhux2PfBKsP0ocIOZFZrZImAxsDZ7JR9vzqwiWvVlrIjIiMYy62YlcCOwycw2BG1fBT5qZkvJDMvsBD4H4O6bzexB4FUyM3ZuzsWMm0FzZxWyv7OPvlSagsSM/lmAiEhOjBr07v4cw4+7P3aKY74BfGMSdY3Z4BTLts5e6iqLp+ItRURmlBnfBT5NP5oSETmlGR/0c2YVAmiKpYjICGZ80P9xGQTNvBERGc6MD/rqkgISMdPQjYjICGZ80MdixpzyQvXoRURGMOODHjSXXkTkVCIR9FoGQURkZBEJ+iL2HlbQi4gMJxJBf1pFEUd6UnT1pcIuRURk2olE0M+vyPwituWQevUiIieKRtBXDga9rh0rInKiiAR95kdTCnoRkZNFIujnzirCDFr0hayIyEkiEfTJeIy55UXq0YuIDCMSQQ+Z4RsFvYjIySIU9MXs0dCNiMhJIhX0uw91437SdchFRPJadIK+ooi+VJr2o31hlyIiMq1EJ+g1l15EZFgRDHqN04uIDBXBoFePXkRkqMgEfVVJkqJkTEEvInKCyAS9mTG/QlMsRUROFJmgh8zwTbN69CIix4lU0NdVFrP7oIJeRGSoSAX9gtkl7O/s1QVIRESGiFTQN1SXANCsXr2IyDHRCvqqzBTLt9u7Qq5ERGT6iFTQLwh69LsOKuhFRAZFKuirSwsoKYjz9gEFvYjIoFGD3swazOwpM9tiZpvN7ItBe7WZPWFmbwT3VUG7mdm/mtk2M9toZstz/SGG1MqC6hJ2HdAYvYjIoLH06FPAl939XOBS4GYzOw/4CvCkuy8GngweA1wNLA5uq4E7sl71KTRUl7BLPXoRkWNGDXp33+PuLwbbHcAWoA64Drgv2O0+YFWwfR1wv2f8Aag0s3lZr3wEDVUlvH2gS+vSi4gExjVGb2YLgWXAC8Bcd98DmZMBMCfYrQ7YNeSw5qDtxNdabWZNZtbU1tY2/spHsKC6mO7+Aa1LLyISGHPQm1kZ8BPgS+5+5FS7DtN2Uvfa3e9y90Z3b6ytrR1rGaManEuvL2RFRDLGFPRmliQT8j9094eD5n2DQzLBfWvQ3gw0DDm8HmjJTrmjOzbFUkEvIgKMbdaNAfcAW9z9m0OeehS4Kdi+CfjZkPZPBLNvLgUODw7xTIX6KgW9iMhQiTHssxK4EdhkZhuCtq8CtwEPmtlngLeBjwTPPQZcA2wDuoBPZbXiURQXxKktL9QUSxGRwKhB7+7PMfy4O8Dlw+zvwM2TrGtSFlSXsLP9aJgliIhMG5H6ZeygRTWlCnoRkUBkg37fkV6O9mq5YhGRSAb9GTWlAOzYr169iEgkg35RrYJeRGRQJIN+4WwFvYjIoEgGfVEyTl1lsYJeRISIBj1kvpB9U0EvIhLtoN/R1qlVLEUk70U66I/0pDigVSxFJM9FN+g180ZEBIhw0A/Opdc4vYjku8gGfV1lMcm4qUcvInkvskGfiMc4fXYp21o7wy5FRCRUkQ16gMVzyhT0IpL3oh30c8t5q/0oPf0DYZciIhKaSAf94a4+0g7n/t2vWHnbb3jkpd1hlyQiMuUiG/SPvLSbB9btAjJXJt99qJtbH96ksBeRvBPZoL99zVZ6U+nj2rr7B7h9zdaQKhIRCUdkg77l0PDXjB2pXUQkqiIb9PMri8fVLiISVZEN+luuXEJxMn5cW3Eyzi1XLgmpIhGRcCTCLiBXVi2rA+Dv/3MzB7v6qS0v5H9dc+6xdhGRfBHZHj1kwv7Bz70LgK9ec45CXkTyUqSDHmBhTSnJuLF1r34hKyL5KfJBn4zHOLO2jNf2Hgm7FBGRUEQ+6AHOmz+LV1sU9CKSn/Ii6M+fX0FrRy9tHb1hlyIiMuXyIujPmzcLgFf3qFcvIvknv4JewzcikodGDXozu9fMWs3slSFtXzez3Wa2IbhdM+S5W81sm5ltNbMrc1X4eFSUJKmvKmZzy+GwSxERmXJj6dF/D7hqmPZvufvS4PYYgJmdB9wAnB8c8x0ziw9z7JQ7b94sDd2ISF4aNejd/RngwBhf7zrgAXfvdfcdwDZgxSTqy5rz5s9ix/6jdPWlwi5FRGRKTWaM/gtmtjEY2qkK2uqAXUP2aQ7aTmJmq82sycya2traJlHG2Jw/vwJ3eG1vR87fS0RkOplo0N8BnAksBfYA/xS02zD7+nAv4O53uXujuzfW1tZOsIyxO29+5gvZzfpCVkTyzISC3t33ufuAu6eB7/LH4ZlmoGHIrvVAy+RKzI75FUVUlSR5pVlfyIpIfplQ0JvZvCEPrwcGZ+Q8CtxgZoVmtghYDKydXInZYWa8o6GSl5sPhV2KiMiUGnWZYjP7MfAeoMbMmoGvAe8xs6VkhmV2Ap8DcPfNZvYg8CqQAm5294HclD5+76iv5JnX3+Bob4rSwsiu0CwicpxR087dPzpM8z2n2P8bwDcmU1SuLG2oJO2wafdhLj1jdtjliIhMibz4ZeygdzRUAvDyLg3fiEj+yKugry4tYEF1icbpRSSv5FXQQ6ZX//IuzbwRkfyRd0G/tKGS3Ye6ae3oCbsUEZEpkYdBXwGgXr2I5I28C/rz51eQiBkvvX0w7FJERKZE3gV9UTLO+XUVNO1U0ItIfsi7oAdYsbCKDc2H6E1Nm99yiYjkTF4GfePCavpSaTZq3RsRyQN5GfQXL6wGYO2OsS6zLyIyc+Vl0FeXFnDWnDLW7VTQi0j05WXQQ6ZXv37nQQbSwy6XLyISGXkb9CsWVdHRm+K1vboQiYhEW94G/eA4/TqN04tIxOVt0NdXlVBXWczzb7aHXYqISE7lbdAD/OlZNfx+e7vG6UUk0vI66FcurqGjJ8VGLVssIhGW30F/ZuYqU7/btj/kSkREcievg352WSHnzZvFcwp6EYmwvA56gHcvrmH9Wwfp6kuFXYqISE7kfdCvPKuG/gHXcggiEll5H/QrFlVTkIjx3BsavhGRaMr7oC9KxrlkUTVPbW0NuxQRkZzI+6AHuPycOWxvO8rO/UfDLkVEJOsU9MDl584F4MnX1KsXkehR0AMN1SWcPbeMJ7fsC7sUEZGsU9AH3nfOXNbuOMCRnv6wSxERySoFfeDPz51DKu0883pb2KWIiGSVgj6wbEEVVSVJntyicXoRiZZRg97M7jWzVjN7ZUhbtZk9YWZvBPdVQbuZ2b+a2TYz22hmy3NZfDbFY8bl587l11v20ZsaCLscEZGsGUuP/nvAVSe0fQV40t0XA08GjwGuBhYHt9XAHdkpc2pce+E8OnpSWuRMRCJl1KB392eAE9cHuA64L9i+D1g1pP1+z/gDUGlm87JVbK6tPKuGWUUJfr5xT9iliIhkzUTH6Oe6+x6A4H5O0F4H7BqyX3PQdhIzW21mTWbW1NY2Pb4ALUjEuPL803his4ZvRCQ6sv1lrA3TNuzlm9z9LndvdPfG2traLJcxcddeNI+O3hTPvq7hGxGJhokG/b7BIZngfnCqSjPQMGS/eqBl4uVNvZVn1VBRnOQXmzR8IyLRMNGgfxS4Kdi+CfjZkPZPBLNvLgUODw7xzBTJeIyrzj+Nxzfv1Rr1IhIJY5le+WPgeWCJmTWb2WeA24ArzOwN4IrgMcBjwJvANuC7wF/npOocu355HUf7BlizeW/YpYiITFpitB3c/aMjPHX5MPs6cPNkiwrbioXVNFQX89D6Zq5fVh92OSIik6Jfxg4jFjM+tLye329vZ/eh7rDLERGZFAX9CD60vB53+OmLzWGXIiIyKQr6ETRUl3DpGdU8tL6ZzIiUiMjMpKA/hY+8s4Gd7V08/2Z72KWIiEyYgv4Urr1oHpUlSb7//FthlyIiMmEK+lMoSsb5Lxc38Pir+9hzWF/KisjMpKAfxccvOZ20Oz964e2wSxERmRAF/Sgaqkt435I5/HjtLvpS6bDLEREZNwX9GNz4rtPZ39nLLzbNqGV7REQABf2YXLa4lrPnlvHvT7+pqZYiMuMo6McgFjM+d9mZvLa3g99unR5r54uIjJWCfow+sHQ+8yuKuOO328MuRURkXBT0Y5SMx/jsu89g7c4DrH/rxCsriohMXwr6cbhhRQNVJUn+7Tfbwi5FRGTMFPTjUFKQYPVlZ/LbrW007VSvXkRmBgX9ON30J6dTU1bI7Wu2agaOiMwICvpxKilI8IX3nskLOw7w3DZdQFxEpj8F/QR89JIF1FUWc/uaraTT6tWLyPSmoJ+AwkScv7nibDY2H+aRDbvDLkdE5JQU9BP0wWV1vKO+gtt++RpHe1NhlyMiMiIF/QTFYsbXPnA+rR29fPspTbcUkelLQT8JyxdU8cFlddz97A7eaj8adjkiIsNKhF3ATPc/rz6HX2xs4f3feoa+VJr5lcXccuUSVi2rC7s0ERFAQT9pz29vJw30B2vV7z7Uza0PbwJQ2IvItKChm0m6fc1W+geOn2LZ3T/A7Wu2hlSRiMjxFPST1HJo+GvJjtQuIjLVFPSTNL+yeFztIiJTTUE/SbdcuYTiZPyk9hsvXRBCNSIiJ1PQT9KqZXX8wwcvpK6yGANOm1VEaUGcRza00NM/EHZ5IiKTm3VjZjuBDmAASLl7o5lVA/8PWAjsBP7S3Q9OrszpbdWyuuNm2Dz1Wiuf+t46vv7oZv7hgxdiZiFWJyL5Lhs9+ve6+1J3bwwefwV40t0XA08Gj/PKe8+Zw83vPZMH1u3ihy+8HXY5IpLncjF0cx1wX7B9H7AqB+8x7f3tFUt475Javv7oZtbu0EVKRCQ8kw16Bx43s/Vmtjpom+vuewCC+zmTfI8ZKR4z/vmGZSyoLuHzP1ivJRJEJDSTDfqV7r4cuBq42cwuG+uBZrbazJrMrKmtrW2SZUxPFcVJvntTIwPufOLetbR19IZdkojkoUkFvbu3BPetwE+BFcA+M5sHENy3jnDsXe7e6O6NtbW1kyljWjuztox7P3kx+4708OnvraNTSxqLyBSbcNCbWamZlQ9uA+8HXgEeBW4KdrsJ+Nlki5zpli+o4jt/tZxX9xxh9f1NdPdp2qWITJ3J9OjnAs+Z2cvAWuAX7v4r4DbgCjN7A7gieJz33nfOXP7xIxfx/JvtfOa+dQp7EZkyE55H7+5vAu8Ypr0duHwyRUXV9cvqAfjygy/z6e+t455PNlJSoAVERSS39MvYKXb9snq++ZdLeWFHOzfes5aDR/vCLklEIk5BH4JVy+r49seWs2n3YT505+/ZdaAr7JJEJMIU9CG5+sJ5/OAzl9De2cf13/k9G5sPhV2SiESUgj5EKxZV85PPv4vCRIwP3/k8D61vDrskEYkgBX3IzppTzqNfWEnj6VX89/94mb975BX6gssSiohkg4J+GphdVsj9n17B6svO4Pt/eIuP3Pl7duzXkgkikh0K+mkiEY/x1WvO5c6PL2dnexfX/Muz/Hjt27j76AeLiJyCgn6aueqCeaz50mUsP72SWx/exGfva9L1Z0VkUhT009BpFUV8/9OX8L+vPZffbd/PFd98mnue20FqQGP3IjJ+CvppKhYzPvvuM3jib/6MFYuq+T8/f5VV3/kd63ZqbXsRGR8F/TTXUF3CvZ+8mO/81XLaOnr5yJ3P81+/v15f1orImGmhlRnAzLjmwnm8Z0ktdz+7gzuf3s6vt+zjY5cs4K/fcxanVRSFXaKITGM2HWZ1NDY2elNTU9hlzBitHT1864k3eLBpF3EzPtxYz+f/7EwaqkvCLk1EppCZrR9yve6R91PQz1y7DnRxx9PbeaipmbQ7H1g6n0+vXMQFdRVhlyYiU0BBn0f2HO7m359+kwebdtHVN8A7T6/ik3+ykKsuOI1kXF/DiESVgj4PHe7u56H1zdz//E7eau+itryQDy6r40PvrOfsueVhlyciWaagz2PptPPb11v50Qu7eGprKwNp56L6Cj78znquvXAes8sKwy5RRLJAQS8A7O/s5WcbWnhofTNb9hwhZplVM6++YB5Xnn+aZuyIzGAKejnJlj1H+OWmPfzylb280doJwLIFlVx+zhwuO7uWC+ZXEItZyFWKyFgp6OWUtrV28qtX9rBm8z427T4MQHVpAe9eXMNli2v508U1zJ2l3r7IdKaglzHb39nLc2/s5+nX23j2jTb2d2auY7uguoSLF1azYlEVFy+sZlFNKWbq8YtMFwp6mZB02nl1zxFe2HGAtTvaadp5kPbgAuY1ZQUsbajkwrpKLqqv4IK6CmrL9cWuSFgU9JIV7s72tqOs23mAdTsPsLH5MNvbOhn8z2ZeRREX1lVw/vwKlpxWxtlzyzl9dilxjfWL5NxYg15r3cgpmRlnzSnjrDllfHTFAgA6e1Ns3n2YTcFtY/NhHn9137FjChIxzqwtY8ncMhbPLWfxnDIW1ZTSUF1CUTIe1kcRyVsKehm3ssIEl5wxm0vOmH2srasvxbbWTrbu7eCN1k5e39fB2h0HeGRDy7F9zGDerCJOn13KwpqSzP3sEhqqS6irLKaiOKnvAERyQEEvWVFSkOCi+kouqq88rr2jp5/tbUd5q/0oO/d3Ze7bj/L45n3Hxv4HFSfjzK8sYn5lMfMripk3ZPu0iiJqywuZVZTQyUBknBT0klPlRUmWNlSytKHypOeO9PTzdnsXbx/oouVQN3sO99ByqJuWwz28treVto7ek44pSMSoLSuktryQmuD+2K2skNryAipLCqgqKaCiOKnvCkRQ0EuIZhUluaCuYsTVNntTA+w73EvL4W72Hu5hf2cvbR3BrbOX5oNdbNiVmRU00pyCWUUJqkoHwz957ARQVVJAVWmSiuIks4qSlBUlKC9KUFaYoLwoSVlhQicJiQwFvUxbhYk4C2aXsGD2qdfZTw2kOXC0j9aOXvZ39nKoq5+DXX0c7Orn0JD79s4+trV2cqirn87e1KjvX1oQp7womTkBFCWObZcXJigtTFBSEKe4IE5JMk5JQSKzPdhWEDyfzLSVFCQoSsY07CShyFnQm9lVwL8AceBud78tV+8l+S0RjzFnVhFzxvFL3r5UmkPdfRzp7udIT4rOnhQdPSk6e/vp6EkNacucFDp6Uhzu6qP5YBcdPSmO9qbo7h8Y8S+J4ZhxLPiLC+IUJeIUJmMUJuIUJmLBbbBtSPsJ+xQlTzwuTkEiRjJuJOOx4HbCdiJGMpbZjsdMJ5yQPPLSbm5fs5WWQ93MryzmliuXsGpZXc7fNydBb2Zx4NvAFUAzsM7MHnX3V3PxfiLjVZCIMae8iDnlE1/mwd3p6U/T1Zeiq2+A7v4BuvoG6OpL0d2X2e4OHnf1Dxxry7Sn6BtI09ufpjeVpjc1QEdPit7UQOZxf/rYdk//AOks/tzFjGOhn0zESMRiFBzbzpwgCobZTsQz9/Eht8zj49tH2ice47h9EzEjNmT/xDD7DH0Ns8x2zCBmlrnFhmwbwfOGnbh9bJ8Tjjnh+FyeAB95aTe3PryJ7v4BAHYf6ubWhzcB5Dzsc9WjXwFsc/c3AczsAeA6QEEvkWFmFAe989mj7z4pqYHBE0JwAhhyghg8MfSn0/Sn0vQPOKl0mr5gu38gHdwy26mBNH0jbPcPOH0nbB/tTQVtTtqdVNoZSPswj9OkHVLpdOZx2sf1F890YMFJJB6cIGI2+BfQH08cx51oDGKxk7cNjp1kBu9f39dB/8Dx/0C6+we4fc3WGRv0dcCuIY+bgUty9F4ikZeIx0jEY5TOsBUn0mlnwP1Y8A8MZB4fOxmc8uSRObkM+B/b3SEdvF462M7cMu813PaAO+4e1JL5SyzzGgSvecL2cK+RDl7juNcj2DdT10DwWXGOqzUdvOfmliPD/jNqOdSd838PuQr64f7+Oe5UZmargdUACxYsyFEZIhKmWMyIYegH0bDytt+we5hQn19ZnPP3ztUFRZuBhiGP64GWoTu4+13u3ujujbW1tTkqQ0RkerjlyiUUn3DGK07GueXKJTl/71z16NcBi81sEbAbuAH4WI7eS0Rk2hsch4/MrBt3T5nZF4A1ZKZX3uvum3PxXiIiM8WqZXVTEuwnytk8end/DHgsV68vIiJjk6sxehERmSYU9CIiEaegFxGJOAW9iEjEKehFRCJOQS8iEnEKehGRiDOfBsvLmVkb8NYED68B9mexnOko6p9Rn29m0+cLz+nuPuoaMtMi6CfDzJrcvTHsOnIp6p9Rn29m0+eb/jR0IyIScQp6EZGIi0LQ3xV2AVMg6p9Rn29m0+eb5mb8GL2IiJxaFHr0IiJyCjM26M3sXjNrNbNXwq4lF8yswcyeMrMtZrbZzL4Ydk3ZZGZFZrbWzF4OPt/fh11TLphZ3MxeMrOfh11LLpjZTjPbZGYbzKwp7HqyzcwqzewhM3st+H/xXWHXNBEzdujGzC4DOoH73f2CsOvJNjObB8xz9xfNrBxYD6xy91dDLi0rzMyAUnfvNLMk8BzwRXf/Q8ilZZWZ/S3QCMxy978Iu55sM7OdQKO7T9d55pNiZvcBz7r73WZWAJS4+6Gw6xqvGdujd/dngANh15Er7r7H3V8MtjuALcDUX5omRzyjM3iYDG4zs9cxAjOrB64F7g67Fhk/M5sFXAbcA+DufTMx5GEGB30+MbOFwDLghXArya5gWGMD0Ao84e6R+nzAPwP/A0iHXUgOOfC4ma03s9VhF5NlZwBtwP8Nht/uNrPSsIuaCAX9NGdmZcBPgC+5+5Gw68kmdx9w96VAPbDCzCIzBGdmfwG0uvv6sGvJsZXuvhy4Grg5GFKNigSwHLjD3ZcBR4GvhFvSxCjop7Fg7PonwA/d/eGw68mV4M/h3wJXhVxKNq0EPhCMYT8AvM/MfhBuSdnn7i3BfSvwU2BFuBVlVTPQPOQvzYfIBP+Mo6CfpoIvK+8Btrj7N8OuJ9vMrNbMKoPtYuDPgdfCrSp73P1Wd69394XADcBv3P3jIZeVVWZWGkwUIBjSeD8QmVlw7r4X2GVmS4Kmy4EZORkiEXYBE2VmPwbeA9SYWTPwNXe/J9yqsmolcCOwKRjHBviquz8WYk3ZNA+4z8ziZDocD7p7JKcgRthc4KeZPgkJ4Efu/qtwS8q6/wb8MJhx8ybwqZDrmZAZO71SRETGRkM3IiIRp6AXEYk4Bb2ISMQp6EVEIk5BLyIScQp6EZGIU9CLiEScgl5EJOL+P0ZSdzmC3LhuAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(array([168.62498827, 59.99018107, 0.98484788]),\n", + " array([168.62498827, 60. , 1. ]))" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x1, x2 = pred.low_bucket.feerate**ALPHA, pred.normal_bucket.feerate**ALPHA\n", + "y1, y2 = pred.low_bucket.estimated_seconds, pred.normal_bucket.estimated_seconds\n", + "b2 = (y1 - y2*x2/x1) / (1 - x1/x2)\n", + "b1 = (y1 - b2) * x1\n", + "def p(ff):\n", + " return b1/ff**ALPHA + b2\n", + "\n", + "plt.figure()\n", + "plt.plot(x, p(x))\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "p(pred.feerates()), pred.times()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Challenge: outliers\n", + "\n", + "The segment below illustrates a challenge in the current approach. It is sufficient to add a single outlier \n", + "to the total weight (with `feerate=100`), and the `feerate_to_time` function is notably influenced. In truth, this tx should not affect our prediction because it only captures the first slot of each block, however because we sample with repetition it has a significant impact on the function. The following figure shows the `feerate_to_time` function with such an outlier " + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGwdJREFUeJzt3XtwXOWZ5/Hvc/qmiy1ZtuWrjG0SA8GEQOIBzzK7m8XhmhSmapMsuzOJa5ZaqjbMJLM7u5OQVJHdXKbIbCpkUrlskeDgzKZCXIQMDBU2eLjshAwQzP1ijI1NsHyVb7JsS7KkfvaPfiW3pW6pbaQ+7T6/T5Wqz3nP26efli399J73nD7m7oiISPJEcRcgIiLxUACIiCSUAkBEJKEUACIiCaUAEBFJKAWAiEhCKQBERBJKASAiklAKABGRhErHXcB4Zs+e7UuWLIm7DBGRs8pzzz23393bJ+pX0wGwZMkSNm7cGHcZIiJnFTP7fSX9dAhIRCShFAAiIgmlABARSSgFgIhIQikAREQSSgEgIpJQCgARkYSqywDY3d3Ltx7ZzLauo3GXIiJSs+oyAPb3nOA7j23lra5jcZciIlKz6jIAGjKFt9U3MBRzJSIitatOAyAFQK8CQESkrLoMgMZsIQA0AhARKa8uA2B4BKAAEBEprz4DIF14W70n8jFXIiJSu+oyANKpiEzKNAcgIjKOugwAgFw6pUNAIiLjqDgAzCxlZi+Y2UNhfamZPWNmW8zs52aWDe25sL41bF9StI/bQvtmM7tmst9MsVw6UgCIiIzjdEYAnwM2Fa1/A7jT3ZcBh4CbQ/vNwCF3fy9wZ+iHmV0I3AQsB64Fvm9mqXdXfnkKABGR8VUUAGbWAXwU+FFYN+BK4L7QZR1wY1heHdYJ21eF/quBe9293923A1uByybjTZSSTUeaAxARGUelI4BvA38FDJ9WMws47O6DYb0TWBiWFwI7AML27tB/pL3EcyZdIQB0FpCISDkTBoCZfQzY5+7PFTeX6OoTbBvvOcWvd4uZbTSzjV1dXROVV1Y2pUNAIiLjqWQEcAVwg5m9DdxL4dDPt4EZZpYOfTqAXWG5E1gEELa3AgeL20s8Z4S73+XuK9x9RXt7+2m/oWGZdETvCQWAiEg5EwaAu9/m7h3uvoTCJO5j7v7HwOPAx0O3NcADYfnBsE7Y/pi7e2i/KZwltBRYBvxu0t7JKDnNAYiIjCs9cZeyPg/ca2ZfA14A7g7tdwN/Z2ZbKfzlfxOAu79mZuuB14FB4FZ3n7Lf0DoEJCIyvtMKAHd/AngiLG+jxFk87t4HfKLM878OfP10izwTOR0CEhEZV91eCZzVlcAiIuOq4wCI6BvUaaAiIuXUbQDk0hFDeWdgSCEgIlJKXQcA6K5gIiLl1G0AZNO6L7CIyHjqNgCGRwB9uimMiEhJdRsA2bRuDC8iMp46DgAdAhIRGU/dBoAmgUVExqcAEBFJqLoNgGyq8Nb6FQAiIiXVbQBoBCAiMr66DYDhSeDj+kA4EZGS6jYAGjLhNFAFgIhISXUfAMf6FQAiIqXUbQCkIiMdGccHBifuLCKSQHUbAFCYBziuEYCISEn1HQCpiGMnNAIQESmlrgMgk9IIQESknLoOgHTKNAIQESmj7gNA1wGIiJRW1wGQiSKO9WsEICJSSn0HQFoBICJSTn0HQMo4pkNAIiIl1XUAZFMRxzUJLCJSUl0HQCYV0TeQJ5/3uEsREak5dR8AoI+EFhEppc4DwAB0LYCISAl1HQDDdwXT1cAiImPVdQBkwk1hNAIQERmrvgMgpbuCiYiUU+cBEOYAdDGYiMgYdR4AGgGIiJRT1wEwPAmsEYCIyFh1HQAaAYiIlFfnAaDrAEREyqnrAEhFRmQ6BCQiUsqEAWBmDWb2OzN7ycxeM7P/GdqXmtkzZrbFzH5uZtnQngvrW8P2JUX7ui20bzaza6bqTRW9Hrl0iqN9CgARkdEqGQH0A1e6+weAS4BrzWwl8A3gTndfBhwCbg79bwYOuft7gTtDP8zsQuAmYDlwLfB9M0tN5pspJZeO6FEAiIiMMWEAeMHRsJoJXw5cCdwX2tcBN4bl1WGdsH2VmVlov9fd+919O7AVuGxS3sU4MumIHh0CEhEZo6I5ADNLmdmLwD5gA/AWcNjdh3+zdgILw/JCYAdA2N4NzCpuL/GcKZNNRfT0DUz1y4iInHUqCgB3H3L3S4AOCn+1v69Ut/BoZbaVaz+Fmd1iZhvNbGNXV1cl5Y0rm4440qsRgIjIaKd1FpC7HwaeAFYCM8wsHTZ1ALvCciewCCBsbwUOFreXeE7xa9zl7ivcfUV7e/vplFdSNhXR068RgIjIaJWcBdRuZjPCciPwEWAT8Djw8dBtDfBAWH4wrBO2P+buHtpvCmcJLQWWAb+brDdSTjYd6SwgEZES0hN3YT6wLpyxEwHr3f0hM3sduNfMvga8ANwd+t8N/J2ZbaXwl/9NAO7+mpmtB14HBoFb3X3KL9EdPgvI3SnMRYuICFQQAO7+MnBpifZtlDiLx937gE+U2dfXga+ffplnLpuOGMw7/YN5GjJTftapiMhZo66vBIZCAAC6FkBEZJS6D4DcSABoIlhEpFjdB8AfHnuMJ7OfZen3OuDOi+Dl9XGXJCJSEyqZBD5rnb/vYT6y95tko/5CQ/cO+IfPFpYv/mR8hYmI1IC6HgH80TvfJ+v9pzYO9MKjX4mnIBGRGlLXATC9f2/pDd2d1S1ERKQG1XUA9OTmlt7Q2lHdQkREalBdB8CT53yGgajh1MZMI6y6PZ6CRERqSF1PAm+ecx0AH3jzOyywA1hrR+GXvyaARUTqOwCgEAKf33IB1180n298/OK4yxERqRl1fQhomD4RVERkrEQEQC4d0d2rABARKZaMAMikOHRMASAiUiwRAdCQjjh8/ETcZYiI1JREBEAuk9IhIBGRURIRAA2ZiGMnhhgYysddiohIzUhGAKQLN4LRKEBE5KRkBEC4E9jh4woAEZFhCQmAwtvs7tVEsIjIsEQEQE4jABGRMRIRAA3htpAKABGRk5IRAMMjAE0Ci4iMSEQA5NIRBnTrYjARkRGJCAAzoyGT0ghARKRIIgIAoDGT0hyAiEiRxARALhNpBCAiUiQxAZBNRRw+pjkAEZFhiQmAXCbioCaBRURGJCYAmrJpDmkEICIyIjEB0JhNcezEEH0DQ3GXIiJSExITAE3hYrADGgWIiABJCoBsCICj/TFXIiJSGxITAI1ZjQBERIolJwCGDwEdVQCIiECCAqApmwZ0CEhEZFhiAiCTMtKR6RCQiEiQmAAwM5pzafZrBCAiAlQQAGa2yMweN7NNZvaamX0utM80sw1mtiU8toV2M7PvmNlWM3vZzD5YtK81of8WM1szdW+rtMZMSnMAIiJBJSOAQeAv3f19wErgVjO7EPgC8Ki7LwMeDesA1wHLwtctwA+gEBjAl4HLgcuALw+HRrU0ZCKNAEREggkDwN13u/vzYbkH2AQsBFYD60K3dcCNYXk18BMveBqYYWbzgWuADe5+0N0PARuAayf13UygMZtSAIiIBKc1B2BmS4BLgWeAue6+GwohAcwJ3RYCO4qe1hnayrWPfo1bzGyjmW3s6uo6nfIm1JRJc/DYCdx9UvcrInI2qjgAzGwa8AvgL9z9yHhdS7T5OO2nNrjf5e4r3H1Fe3t7peVVpDGbYmDI6ekfnNT9ioicjSoKADPLUPjl/1N3vz807w2HdgiP+0J7J7Co6OkdwK5x2qumOVe4GGzfkb5qvqyISE2q5CwgA+4GNrn7t4o2PQgMn8mzBnigqP3T4WyglUB3OET0a+BqM2sLk79Xh7aqmZYrXAy294jmAURE0hX0uQL4FPCKmb0Y2r4I3AGsN7ObgXeAT4RtvwKuB7YCx4E/BXD3g2b2VeDZ0O8r7n5wUt5FhZpDAOzp1ghARGTCAHD3Jyl9/B5gVYn+DtxaZl9rgbWnU+BkGhkB9CgAREQScyUwQCYV0ZCJ2KsRgIhIsgIACqMAzQGIiCQwAJqyafboLCARkeQFQHMupQAQESGJAZBN09XTTz6vq4FFJNkSFwDTcmmG8q77AohI4iUvABqGLwbTYSARSbbEBUBzVheDiYhAAgNgehgB7DzcG3MlIiLxSlwANGVTpCOj89DxuEsREYlV4gLAzGhtzNB5SCMAEUm2xAUAFCaCd2gEICIJl8gAaGnI0HlQIwARSbaEBkCaw70DHNWdwUQkwZIZAI0ZAHZqHkBEEiyZAdBQCACdCSQiSZbIABi+FkBnAolIkiUyAJqyKTIpY8dBjQBEJLkSGQCFawGyvH1AASAiyZXIAABobUzzVtfRuMsQEYlNYgOgrSnLOwePMziUj7sUEZFYJDoAhvLODk0Ei0hCJTYAZjQVTgXdvl+HgUQkmRIbAG3NWQC2dR2LuRIRkXgkNgAaMykaMym27VcAiEgyJTYAoHAYaLtGACKSUIkPgC37euIuQ0QkFokOgNnTcuw/eoIDR/vjLkVEpOoSHwAAm/doFCAiyZPwACicCbRJASAiCZToAGjKppmWS/PG7iNxlyIiUnWJDgCAmc1ZNikARCSBEh8As6dl2bLvqD4TSEQSRwEwLUf/YJ63D+h6ABFJlsQHQPv0wplAL3d2x1yJiEh1JT4AZjZnyaYjXtxxOO5SRESqKvEBEJkxd3qOF95RAIhIskwYAGa21sz2mdmrRW0zzWyDmW0Jj22h3czsO2a21cxeNrMPFj1nTei/xczWTM3bOTNzWhrYtPsIfQNDcZciIlI1lYwA7gGuHdX2BeBRd18GPBrWAa4DloWvW4AfQCEwgC8DlwOXAV8eDo1aMK+lgcG889ounQ4qIskxYQC4+z8BB0c1rwbWheV1wI1F7T/xgqeBGWY2H7gG2ODuB939ELCBsaESm3mtDQC8pHkAEUmQM50DmOvuuwHC45zQvhDYUdSvM7SVax/DzG4xs41mtrGrq+sMyzs903JpWhrSPPf7Q1V5PRGRWjDZk8BWos3HaR/b6H6Xu69w9xXt7e2TWtx4Fsxo5J/f2o97ybJEROrOmQbA3nBoh/C4L7R3AouK+nUAu8ZprxkdbY0cOj7Am3t1j2ARSYYzDYAHgeEzedYADxS1fzqcDbQS6A6HiH4NXG1mbWHy9+rQVjMWtTUB8NRb+2OuRESkOio5DfRnwFPA+WbWaWY3A3cAV5nZFuCqsA7wK2AbsBX4IfAZAHc/CHwVeDZ8fSW01YyWxgytjRme2nYg7lJERKoiPVEHd//3ZTatKtHXgVvL7GctsPa0qquyhTMaeeqtAwzlnVRUatpCRKR+JP5K4GKLZzVxpG+Q59/R2UAiUv8UAEUWz2oiMvjHTXvjLkVEZMopAIrk0ik62pr4x9cVACJS/xQAoyyd3cxbXcd4e7/uDyAi9U0BMMrS2c0A/Pq1PTFXIiIytRQAo7Q2ZpjX0sDfv7gz7lJERKaUAqCE8+dNZ9PuHrbs7Ym7FBGRKaMAKGHZnGlEhkYBIlLXFAAlNOfSLJrZxP3P72Qorw+HE5H6pAAoY/n8FnZ39/H4G/sm7iwichZSAJRxbvs0pjekWffU23GXIiIyJRQAZaQi46IFrfxmy362dekjokWk/igAxrF8QQupyPjhb7bFXYqIyKRTAIyjOZdm+fwW1m/spPPQ8bjLERGZVAqACaxY0gbA9594K+ZKREQmlwJgAtMbMlw4v4X1z+5guz4fSETqiAKgApcvnUkqMr760OtxlyIiMmkUABVozqX5gyUzeeyNfTyxWdcFiEh9UABU6JJFM5jZnOVLv3yVnr6BuMsREXnXFAAVSkXGqgvmsKu7l689tCnuckRE3jUFwGlYMKORD53Txs837uDhV3bHXY6IyLuiADhNK8+dxfzWBv7r+pfYvEcfFy0iZy8FwGlKRcb1F80nFRn/6Scb2X+0P+6SRETOiALgDExrSHP9++exu7uXT939DN29mhQWkbOPAuAMzW9t5KPvn8+be4/y6bXPcPj4ibhLEhE5LQqAd2HxrGauu2ger+48wr/9wT+z83Bv3CWJiFRMAfAuvad9GjdesoCdh3pZ/d0neXrbgbhLEhGpiAJgEnS0NfHxD3WQd/gPP3ya7z62hcGhfNxliYiMSwEwSWZNy/HvVixi2ZxpfPORN7nxe7/l1Z3dcZclIlKWAmASZdMR1yyfx/UXzWP7gWPc8N0n+eIvX2FPd1/cpYmIjJGOu4B6Y2YsmzudRTObeHrbAX7+7A5+8Vwnf7JyMX96xRI62priLlFEBFAATJmGTIoPnz+HS89p45ltB/jxb7fz499u55rl8/jUysWsPHcWUWRxlykiCaYAmGKtjRmuXj6Ple+Zxcud3TyxuYuHX93D3JYcN3xgAR+7eAHvX9iqMBCRqlMAVElLQ4Y/eu9sLl86k+37j7F5Tw9rf/s2P/zNdmY2Z/nw+e18+Pw5rFw6kzktDXGXKyIJoACoskwq4ry50zlv7nR6B4b4/f5jvH3gOA+/sof7n98JwMIZjaxY0saHFrexfEEL582dzvSGzMQ7f3k9PPoV6O6E1g5YdTtc/MkpfkcicrZSAMSoMZPigvktXDC/hbw7+470s6u7l93dfTy6aR8PvLhrpO+CGQ1cOL+FZXOns3hmE+fMauKcmU3Mb20kFVnhl/8/fBYGwtXI3TsK66AQEJGSqh4AZnYt8LdACviRu99R7RpqUWTGvNYG5rUWDv+4Oz19g+w/2s/+Yyc40NPPizsO89gb+8j7yeelI2NhWyP39X2J9qFRH0Ux0MvAI/+Do++5kZbGTCEoRESCqgaAmaWA7wFXAZ3As2b2oLvrbuujmBktjRlaGjOc236yPZ93evoH6e4dGPk60jvArKGukvtJ9ezi0q9uAGB6Q5oZjRnamrPMaMoyozFDa2OG5lya5myKplyaabkUTdk0zeFxWi5NUzZFcy5NLh2RS6fIpSNNWotMppgO31Z7BHAZsNXdtwGY2b3AakABUKEoMlrDL+5iRzfOpaV/z5j+hzLt/OvF7fQNDBW+BvN09w7Q1dNP38AQ/YN5TgzmGSweVlQgHRm5dER2OBQyEQ3pFNl0REPmZFBkUhHplJGOjHQqIpMyUpGRjqKRtsKjkUlFYZuN2lZ4npmRMiOywvchKrGcstAvKqwXLxf6GFF0sl9khO0W9lPoZwbG8GNhP4XHQjtG2W0WsnF4fbiOkT6m8JQiMR6+rXYALAR2FK13ApdXuYa69OQ5n+Gqt/6aTP7kVccDUQPPLP0zLpkzY8LnD+WdgaE8A0OFQBgYKlofyjMw6Azm8wzlncG8n/JYWM4zmM/T1zfEoeMn2/Pu5L1wSGso77jDkIf2fFjOO6cXP/VhTKBQPkSGw4YSwXSy38mgGXmNUVlzyuqojeWed+oeR287uVBRvxINxc8br97i4By9v0rrHa+93P7H1lRZvZW8qAF3H/oic/JjD9/y6FfqLgBKfX9O+dk3s1uAWwDOOeecM36hFUtm8qHFbWf8/LPPMnhlPl40jEyvup3r3v8Jrou7tArkQ6AUgsQZHHIGh8JyCCd3yBcFyfDySMCEsMmPBE8hZPJF23z0c8LySP+i57uDM/wI7sAp6z7SPrzOyPrJfvl86fbidcrsb+R1x9nmFBqLayhUeqqiTSW2lY7g0c3FUX3K/sb0K79vL7My+s+AcvVWWtPY1zr97814tY/5lpX5HpZ7zWHtB/aX3tDdOe7+JkO1A6ATWFS03gHsKu7g7ncBdwGsWLHiXf1hmLih9sWfPGvP+EmljFQKcqTiLkWkuu7sKBz2Ga21Y8pfutofBvcssMzMlppZFrgJeLDKNYiI1I5Vt0Om8dS2TGOhfYpVdQTg7oNm9mfArymcBrrW3V+rZg0iIjVleNSegLOAcPdfAb+q9uuKiNSsmA7f6n4AIiIJpQAQEUkoBYCISEIpAEREEkoBICKSUAoAEZGEUgCIiCSUAkBEJKGs3IdA1QIz6wJ+H3cdwWygzKc21QTVd+ZquTao7fpquTao7fqmsrbF7t4+UaeaDoBaYmYb3X1F3HWUo/rOXC3XBrVdXy3XBrVdXy3UpkNAIiIJpQAQEUkoBUDl7oq7gAmovjNXy7VBbddXy7VBbdcXe22aAxARSSiNAEREEkoBMAEzW2Rmj5vZJjN7zcw+F3dNo5lZysxeMLOH4q5lNDObYWb3mdkb4Xv4h3HXVMzM/kv4d33VzH5mZg0x1rLWzPaZ2atFbTPNbIOZbQmPsd3oukx9/yv8275sZr80sxm1VF/Rtv9mZm5ms2upNjP7czPbHP4P/k2161IATGwQ+Et3fx+wErjVzC6MuabRPgdsiruIMv4W+L/ufgHwAWqoTjNbCHwWWOHuF1G4S91NMZZ0D3DtqLYvAI+6+zLg0bAel3sYW98G4CJ3vxh4E7it2kUVuYex9WFmi4CrgHeqXVCRexhVm5n9G2A1cLG7Lwe+We2iFAATcPfd7v58WO6h8AtsYbxVnWRmHcBHgR/FXctoZtYC/CvgbgB3P+Huh+Otaow00GhmaaAJ2BVXIe7+T8DBUc2rgXVheR1wY1WLKlKqPnd/xN0Hw+rTwNTfybyMMt8/gDuBvwJim/AsU9t/Bu5w9/7QZ1+161IAnAYzWwJcCjwTbyWn+DaF/9z5uAsp4VygC/hxOET1IzNrjruoYe6+k8JfXe8Au4Fud38k3qrGmOvuu6HwxwgwJ+Z6xvMfgYfjLqKYmd0A7HT3l+KupYTzgH9pZs+Y2f8zsz+odgEKgAqZ2TTgF8BfuPuRuOsBMLOPAfvc/bm4aykjDXwQ+IG7XwocI95DGKcIx9NXA0uBBUCzmf1JvFWdnczsSxQOl/407lqGmVkT8CXg9rhrKSMNtFE4tPzfgfVmZtUsQAFQATPLUPjl/1N3vz/ueopcAdxgZm8D9wJXmtn/ibekU3QCne4+PGK6j0Ig1IqPANvdvcvdB4D7gX8Rc02j7TWz+QDhseqHCSZiZmuAjwF/7LV1Xvl7KIT7S+FnpAN43szmxVrVSZ3A/V7wOwqj+KpOUisAJhAS+W5gk7t/K+56irn7be7e4e5LKExePubuNfMXrLvvAXaY2fmhaRXweowljfYOsNLMmsK/8ypqaJI6eBBYE5bXAA/EWMsYZnYt8HngBnc/Hnc9xdz9FXef4+5Lws9IJ/DB8P+yFvw9cCWAmZ0HZKnyB9cpACZ2BfApCn9dvxi+ro+7qLPInwM/NbOXgUuAv465nhFhZHIf8DzwCoWfh9iuzjSznwFPAeebWaeZ3QzcAVxlZlsonMlyR43V911gOrAh/Gz87xqrryaUqW0tcG44NfReYE21R1C6ElhEJKE0AhARSSgFgIhIQikAREQSSgEgIpJQCgARkYRSAIiIJJQCQEQkoRQAIiIJ9f8BYJviX8a/T5YAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Feerates:\t1.1539704395225572, 4.139754128892224, 16.2278954349457 \n", + "Times:\t\t2769.889957638353, 60.00000000000002, 1.0000000000000007" + ] + }, + "execution_count": 52, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "estimator = FeerateEstimator(total_weight=total_weight + 100**ALPHA, \n", + " inclusion_time=avg_mass/network_mass_rate)\n", + "\n", + "pred = estimator.calc_estimations()\n", + "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", + "y = estimator.feerate_to_time(x)\n", + "plt.figure()\n", + "plt.plot(x, y)\n", + "plt.fill_between(x, estimator.inclusion_time, y2=y, alpha=0.5)\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "pred" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:gr]", + "language": "python", + "name": "conda-env-gr-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs new file mode 100644 index 000000000..569db12a4 --- /dev/null +++ b/mining/src/feerate/mod.rs @@ -0,0 +1,78 @@ +use crate::block_template::selector::ALPHA; + +#[derive(Clone, Copy, Debug)] +pub struct FeerateBucket { + pub feerate: f64, + pub estimated_seconds: f64, +} + +#[derive(Clone, Debug)] +pub struct FeerateEstimations { + pub low_bucket: FeerateBucket, + pub normal_bucket: FeerateBucket, + pub priority_bucket: FeerateBucket, +} + +pub struct FeerateEstimator { + /// The total probability weight of all current mempool ready transactions, i.e., Σ_{tx in mempool}(tx.fee/tx.mass)^alpha + total_weight: f64, + + /// The amortized time between transactions given the current transaction masses present in the mempool, i.e., + /// the inverse of the transaction inclusion rate. For instance, if the average transaction mass is 2500 grams, + /// the block mass limit is 500,000 and the network has 10 BPS, then this number would be 1/2000 seconds. + inclusion_time: f64, +} + +impl FeerateEstimator { + fn feerate_to_time(&self, feerate: f64) -> f64 { + let (c1, c2) = (self.inclusion_time, self.total_weight); + c1 * c2 / feerate.powi(ALPHA) + c1 + } + + fn time_to_feerate(&self, time: f64) -> f64 { + let (c1, c2) = (self.inclusion_time, self.total_weight); + ((c1 * c2 / time) / (1f64 - c1 / time)).powf(1f64 / ALPHA as f64) + } + + /// The antiderivative function of [`feerate_to_time`] excluding the constant shift `+ c1` + fn feerate_to_time_antiderivative(&self, feerate: f64) -> f64 { + let (c1, c2) = (self.inclusion_time, self.total_weight); + c1 * c2 / (-2f64 * feerate.powi(ALPHA - 1)) + } + + fn quantile(&self, lower: f64, upper: f64, frac: f64) -> f64 { + assert!((0f64..=1f64).contains(&frac)); + let (c1, c2) = (self.inclusion_time, self.total_weight); + let z1 = self.feerate_to_time_antiderivative(lower); + let z2 = self.feerate_to_time_antiderivative(upper); + let z = frac * z2 + (1f64 - frac) * z1; + ((c1 * c2) / (-2f64 * z)).powf(1f64 / (ALPHA - 1) as f64) + } + + pub fn calc_estimations(&self) -> FeerateEstimations { + let high = self.time_to_feerate(1f64); + let low = self.time_to_feerate(3600f64).max(self.quantile(1f64, high, 0.25)); + let mid = self.time_to_feerate(60f64).max(self.quantile(low, high, 0.5)); + FeerateEstimations { + low_bucket: FeerateBucket { feerate: low, estimated_seconds: self.feerate_to_time(low) }, + normal_bucket: FeerateBucket { feerate: mid, estimated_seconds: self.feerate_to_time(mid) }, + priority_bucket: FeerateBucket { feerate: high, estimated_seconds: self.feerate_to_time(high) }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_feerate_estimations() { + let estimator = FeerateEstimator { total_weight: 1002283.659, inclusion_time: 0.004f64 }; + let estimations = estimator.calc_estimations(); + assert!(estimations.low_bucket.feerate <= estimations.normal_bucket.feerate); + assert!(estimations.normal_bucket.feerate <= estimations.priority_bucket.feerate); + assert!(estimations.low_bucket.estimated_seconds >= estimations.normal_bucket.estimated_seconds); + assert!(estimations.normal_bucket.estimated_seconds >= estimations.priority_bucket.estimated_seconds); + dbg!(estimations); + } +} diff --git a/mining/src/lib.rs b/mining/src/lib.rs index 2986577ef..ba5f6c6a6 100644 --- a/mining/src/lib.rs +++ b/mining/src/lib.rs @@ -8,6 +8,7 @@ use mempool::tx::Priority; mod block_template; pub(crate) mod cache; pub mod errors; +pub mod feerate; pub mod manager; mod manager_tests; pub mod mempool; From 5b8b393425c612edcf9ccec217da627ac7c256c8 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Fri, 26 Jul 2024 15:12:27 +0000 Subject: [PATCH 02/74] initial usage of btreeset for ready transactions --- mining/src/mempool/model/transactions_pool.rs | 54 ++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/mining/src/mempool/model/transactions_pool.rs b/mining/src/mempool/model/transactions_pool.rs index 988da5ddd..318b22925 100644 --- a/mining/src/mempool/model/transactions_pool.rs +++ b/mining/src/mempool/model/transactions_pool.rs @@ -18,10 +18,48 @@ use kaspa_consensus_core::{ }; use kaspa_core::{time::unix_now, trace, warn}; use std::{ - collections::{hash_map::Keys, hash_set::Iter, HashSet}, + collections::{hash_map::Keys, hash_set::Iter, BTreeSet}, sync::Arc, }; +#[derive(Clone, Debug, Eq, PartialEq)] +struct FeerateTxKey { + fee: u64, + mass: u64, + id: TransactionId, +} + +impl From<&MempoolTransaction> for FeerateTxKey { + fn from(tx: &MempoolTransaction) -> Self { + Self { fee: tx.mtx.calculated_fee.unwrap(), mass: tx.mtx.tx.mass(), id: tx.id() } + } +} + +impl PartialOrd for FeerateTxKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for FeerateTxKey { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + let (feerate, other_feerate) = (self.fee as f64 / self.mass as f64, other.fee as f64 / other.mass as f64); + match feerate.total_cmp(&other_feerate) { + core::cmp::Ordering::Equal => {} + ord => return ord, + } + match self.fee.cmp(&other.fee) { + core::cmp::Ordering::Equal => {} + ord => return ord, + } + // match self.mass.cmp(&other.mass) { + // core::cmp::Ordering::Equal => {} + // ord => return ord, + // } + self.id.cmp(&other.id) + } +} + /// Pool of transactions to be included in a block template /// /// ### Rust rewrite notes @@ -54,7 +92,7 @@ pub(crate) struct TransactionsPool { /// Transactions dependencies formed by outputs present in pool - successor relations. chained_transactions: TransactionsEdges, /// Transactions with no parents in the mempool -- ready to be inserted into a block template - ready_transactions: HashSet, + ready_transactions: BTreeSet, last_expire_scan_daa_score: u64, /// last expire scan time in milliseconds @@ -105,7 +143,7 @@ impl TransactionsPool { let parents = self.get_parent_transaction_ids_in_pool(&transaction.mtx); self.parent_transactions.insert(id, parents.clone()); if parents.is_empty() { - self.ready_transactions.insert(id); + self.ready_transactions.insert((&transaction).into()); } for parent_id in parents { let entry = self.chained_transactions.entry(parent_id).or_default(); @@ -133,18 +171,20 @@ impl TransactionsPool { if let Some(parents) = self.parent_transactions.get_mut(chain) { parents.remove(transaction_id); if parents.is_empty() { - self.ready_transactions.insert(*chain); + let tx = self.all_transactions.get(chain).unwrap(); + self.ready_transactions.insert(tx.into()); } } } } self.parent_transactions.remove(transaction_id); self.chained_transactions.remove(transaction_id); - self.ready_transactions.remove(transaction_id); // Remove the transaction itself let removed_tx = self.all_transactions.remove(transaction_id).ok_or(RuleError::RejectMissingTransaction(*transaction_id))?; + self.ready_transactions.remove(&(&removed_tx).into()); + // TODO: consider using `self.parent_transactions.get(transaction_id)` // The tradeoff to consider is whether it might be possible that a parent tx exists in the pool // however its relation as parent is not registered. This can supposedly happen in rare cases where @@ -165,10 +205,12 @@ impl TransactionsPool { /// These transactions are ready for being inserted in a block template. pub(crate) fn all_ready_transactions(&self) -> Vec { // The returned transactions are leaving the mempool so they are cloned + // self.ready_transactions.range(range) self.ready_transactions .iter() + .rev() // Iterate in descending feerate order .take(self.config.maximum_ready_transaction_count as usize) - .map(|id| CandidateTransaction::from_mutable(&self.all_transactions.get(id).unwrap().mtx)) + .map(|key| CandidateTransaction::from_mutable(&self.all_transactions.get(&key.id).unwrap().mtx)) .collect() } From 5aeb18055e405f1e997e690ab564eac44f7de5fd Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Wed, 31 Jul 2024 22:40:09 +0000 Subject: [PATCH 03/74] initial frontier + sampling logic --- mining/Cargo.toml | 1 + mining/src/mempool/config.rs | 14 +- mining/src/mempool/model/feerate_key.rs | 68 ++++++++ mining/src/mempool/model/frontier.rs | 163 ++++++++++++++++++ mining/src/mempool/model/mod.rs | 2 + mining/src/mempool/model/transactions_pool.rs | 58 ++----- mining/src/mempool/model/tx.rs | 21 --- 7 files changed, 252 insertions(+), 75 deletions(-) create mode 100644 mining/src/mempool/model/feerate_key.rs create mode 100644 mining/src/mempool/model/frontier.rs diff --git a/mining/Cargo.toml b/mining/Cargo.toml index facd45d6a..14b384a82 100644 --- a/mining/Cargo.toml +++ b/mining/Cargo.toml @@ -28,6 +28,7 @@ rand.workspace = true serde.workspace = true smallvec.workspace = true thiserror.workspace = true +indexmap.workspace = true tokio = { workspace = true, features = [ "rt-multi-thread", "macros", "signal" ] } [dev-dependencies] diff --git a/mining/src/mempool/config.rs b/mining/src/mempool/config.rs index aecbc0711..41b677c49 100644 --- a/mining/src/mempool/config.rs +++ b/mining/src/mempool/config.rs @@ -1,7 +1,7 @@ use kaspa_consensus_core::constants::TX_VERSION; -pub(crate) const DEFAULT_MAXIMUM_TRANSACTION_COUNT: u64 = 1_000_000; -pub(crate) const DEFAULT_MAXIMUM_READY_TRANSACTION_COUNT: u64 = 50_000; +pub(crate) const DEFAULT_MAXIMUM_TRANSACTION_COUNT: u32 = 1_000_000; +pub(crate) const DEFAULT_MAXIMUM_READY_TRANSACTION_COUNT: u32 = 10_000; pub(crate) const DEFAULT_MAXIMUM_BUILD_BLOCK_TEMPLATE_ATTEMPTS: u64 = 5; pub(crate) const DEFAULT_TRANSACTION_EXPIRE_INTERVAL_SECONDS: u64 = 60; @@ -29,8 +29,8 @@ pub(crate) const DEFAULT_MAXIMUM_STANDARD_TRANSACTION_VERSION: u16 = TX_VERSION; #[derive(Clone, Debug)] pub struct Config { - pub maximum_transaction_count: u64, - pub maximum_ready_transaction_count: u64, + pub maximum_transaction_count: u32, + pub maximum_ready_transaction_count: u32, pub maximum_build_block_template_attempts: u64, pub transaction_expire_interval_daa_score: u64, pub transaction_expire_scan_interval_daa_score: u64, @@ -52,8 +52,8 @@ pub struct Config { impl Config { #[allow(clippy::too_many_arguments)] pub fn new( - maximum_transaction_count: u64, - maximum_ready_transaction_count: u64, + maximum_transaction_count: u32, + maximum_ready_transaction_count: u32, maximum_build_block_template_attempts: u64, transaction_expire_interval_daa_score: u64, transaction_expire_scan_interval_daa_score: u64, @@ -122,7 +122,7 @@ impl Config { } pub fn apply_ram_scale(mut self, ram_scale: f64) -> Self { - self.maximum_transaction_count = (self.maximum_transaction_count as f64 * ram_scale.min(1.0)) as u64; // Allow only scaling down + self.maximum_transaction_count = (self.maximum_transaction_count as f64 * ram_scale.min(1.0)) as u32; // Allow only scaling down self } } diff --git a/mining/src/mempool/model/feerate_key.rs b/mining/src/mempool/model/feerate_key.rs new file mode 100644 index 000000000..833711f57 --- /dev/null +++ b/mining/src/mempool/model/feerate_key.rs @@ -0,0 +1,68 @@ +use kaspa_consensus_core::tx::TransactionId; + +use super::tx::MempoolTransaction; + +#[derive(Clone, Debug)] +pub struct FeerateTransactionKey { + pub fee: u64, + pub mass: u64, + pub id: TransactionId, +} + +impl Eq for FeerateTransactionKey {} + +impl PartialEq for FeerateTransactionKey { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl FeerateTransactionKey { + pub fn feerate(&self) -> f64 { + self.fee as f64 / self.mass as f64 + } +} + +impl std::hash::Hash for FeerateTransactionKey { + fn hash(&self, state: &mut H) { + // Transaction id is a sufficient identifier for this key + self.id.hash(state); + } +} + +impl PartialOrd for FeerateTransactionKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for FeerateTransactionKey { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // Our first priority is the feerate + match self.feerate().total_cmp(&other.feerate()) { + core::cmp::Ordering::Equal => {} + ord => return ord, + } + + // If feerates are equal, prefer the higher fee in absolute value + match self.fee.cmp(&other.fee) { + core::cmp::Ordering::Equal => {} + ord => return ord, + } + + // + // At this point we don't compare the mass fields since if both feerate + // and fee are equal, mass must be equal as well + // + + // Finally, we compare transaction ids in order to allow multiple transactions with + // the same fee and mass to exist within the same sorted container + self.id.cmp(&other.id) + } +} + +impl From<&MempoolTransaction> for FeerateTransactionKey { + fn from(tx: &MempoolTransaction) -> Self { + Self { fee: tx.mtx.calculated_fee.unwrap(), mass: tx.mtx.tx.mass(), id: tx.id() } + } +} diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs new file mode 100644 index 000000000..0b1b6f059 --- /dev/null +++ b/mining/src/mempool/model/frontier.rs @@ -0,0 +1,163 @@ +use crate::block_template::selector::ALPHA; + +use super::feerate_key::FeerateTransactionKey; +use indexmap::IndexSet; +use itertools::Either; +use kaspa_consensus_core::tx::TransactionId; +use rand::{distributions::Uniform, prelude::Distribution, Rng}; +use std::collections::{BTreeSet, HashSet}; + +/// Management of the transaction pool frontier, that is, the set of transactions in +/// the transaction pool which have no mempool ancestors and are essentially ready +/// to enter the next block template. +#[derive(Default)] +pub struct Frontier { + /// Frontier transactions sorted by feerate order + feerate_order: BTreeSet, + + /// Frontier transactions accessible via random access + index: IndexSet, + + /// Total sampling weight: Σ_{tx in frontier}(tx.fee/tx.mass)^alpha + total_weight: f64, + + /// Total masses: Σ_{tx in frontier} tx.mass + total_mass: u64, +} + +impl Frontier { + // pub fn new() -> Self { + // Self { ..Default::default() } + // } + + pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { + let (weight, mass) = (key.feerate().powi(ALPHA), key.mass); + self.index.insert(key.id); + if self.feerate_order.insert(key) { + self.total_weight += weight; + self.total_mass += mass; + true + } else { + false + } + } + + pub fn remove(&mut self, key: &FeerateTransactionKey) -> bool { + let (weight, mass) = (key.feerate().powi(ALPHA), key.mass); + self.index.swap_remove(&key.id); + if self.feerate_order.remove(key) { + self.total_weight -= weight; + self.total_mass -= mass; + true + } else { + false + } + } + + fn sample_top_bucket<'a, R>(&'a self, rng: &'a mut R, overall_amount: u32) -> impl Iterator + 'a + where + R: Rng + ?Sized, + { + let frontier_length = self.feerate_order.len() as u32; + debug_assert!(overall_amount <= frontier_length); + let sampling_ratio = overall_amount as f64 / frontier_length as f64; + let distr = Uniform::new(0f64, 1f64); + self.feerate_order + .iter() + .rev() + .map_while(move |key| { + let weight = key.feerate().powi(ALPHA); + let exclusive_total_weight = self.total_weight - weight; + let sample_approx_weight = exclusive_total_weight * sampling_ratio; + if weight < exclusive_total_weight / 100.0 { + None // break out map_while + } else { + let p = weight / self.total_weight; + let p_s = weight / (sample_approx_weight + weight); + debug_assert!(p <= p_s); + // Flip a coin with the reversed probability + if distr.sample(rng) < (sample_approx_weight + weight) / self.total_weight { + Some(Some(self.index.get_index_of(&key.id).unwrap() as u32)) + } else { + Some(None) // signals a continue but not a break + } + } + }) + .flatten() + } + + pub fn sample<'a, R>(&'a self, rng: &'a mut R, overall_amount: u32) -> impl Iterator + 'a + where + R: Rng + ?Sized, + { + let frontier_length = self.feerate_order.len() as u32; + if frontier_length <= overall_amount { + return Either::Left(self.index.iter().copied()); + } + + // Based on values taken from `rand::seq::index::sample` + const C: [f32; 2] = [270.0, 330.0 / 9.0]; + let j = if frontier_length < 500_000 { 0 } else { 1 }; + let indices = if (frontier_length as f32) < C[j] * (overall_amount as f32) { + let initial = self.sample_top_bucket(rng, overall_amount).collect::>(); + sample_inplace(rng, frontier_length, overall_amount, initial) + } else { + let initial = self.sample_top_bucket(rng, overall_amount).collect::>(); + sample_rejection(rng, frontier_length, overall_amount, initial) + }; + + Either::Right(indices.into_iter().map(|i| self.index.get_index(i as usize).copied().unwrap())) + } + + pub(crate) fn len(&self) -> usize { + self.index.len() + } +} + +/// Adaptation of `rand::seq::index::sample_inplace` for the case where there exists an +/// initial a priory sample to begin with +fn sample_inplace(rng: &mut R, length: u32, amount: u32, initial: Vec) -> Vec +where + R: Rng + ?Sized, +{ + debug_assert!(amount <= length); + debug_assert!(initial.len() <= amount as usize); + let mut indices: Vec = Vec::with_capacity(length as usize); + indices.extend(0..length); + let initial_len = initial.len() as u32; + for (i, j) in initial.into_iter().enumerate() { + debug_assert!(i <= (j as usize)); + debug_assert_eq!(j, indices[j as usize]); + indices.swap(i, j as usize); + } + for i in initial_len..amount { + let j: u32 = rng.gen_range(i..length); + indices.swap(i as usize, j as usize); + } + indices.truncate(amount as usize); + debug_assert_eq!(indices.len(), amount as usize); + indices +} + +/// Adaptation of `rand::seq::index::sample_rejection` for the case where there exists an +/// initial a priory sample to begin with +fn sample_rejection(rng: &mut R, length: u32, amount: u32, mut initial: HashSet) -> Vec +where + R: Rng + ?Sized, +{ + debug_assert!(amount < length); + debug_assert!(initial.len() <= amount as usize); + let distr = Uniform::new(0, length); + let mut indices = Vec::with_capacity(amount as usize); + indices.extend(initial.iter().copied()); + for _ in indices.len()..amount as usize { + let mut pos = distr.sample(rng); + while !initial.insert(pos) { + pos = distr.sample(rng); + } + indices.push(pos); + } + + assert_eq!(indices.len(), amount as usize); + indices +} diff --git a/mining/src/mempool/model/mod.rs b/mining/src/mempool/model/mod.rs index 88997e46f..11659bc42 100644 --- a/mining/src/mempool/model/mod.rs +++ b/mining/src/mempool/model/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod accepted_transactions; +pub(crate) mod feerate_key; +pub(crate) mod frontier; pub(crate) mod map; pub(crate) mod orphan_pool; pub(crate) mod pool; diff --git a/mining/src/mempool/model/transactions_pool.rs b/mining/src/mempool/model/transactions_pool.rs index 318b22925..97842b543 100644 --- a/mining/src/mempool/model/transactions_pool.rs +++ b/mining/src/mempool/model/transactions_pool.rs @@ -17,48 +17,13 @@ use kaspa_consensus_core::{ tx::{MutableTransaction, TransactionOutpoint}, }; use kaspa_core::{time::unix_now, trace, warn}; +use rand::thread_rng; use std::{ - collections::{hash_map::Keys, hash_set::Iter, BTreeSet}, + collections::{hash_map::Keys, hash_set::Iter}, sync::Arc, }; -#[derive(Clone, Debug, Eq, PartialEq)] -struct FeerateTxKey { - fee: u64, - mass: u64, - id: TransactionId, -} - -impl From<&MempoolTransaction> for FeerateTxKey { - fn from(tx: &MempoolTransaction) -> Self { - Self { fee: tx.mtx.calculated_fee.unwrap(), mass: tx.mtx.tx.mass(), id: tx.id() } - } -} - -impl PartialOrd for FeerateTxKey { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for FeerateTxKey { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - let (feerate, other_feerate) = (self.fee as f64 / self.mass as f64, other.fee as f64 / other.mass as f64); - match feerate.total_cmp(&other_feerate) { - core::cmp::Ordering::Equal => {} - ord => return ord, - } - match self.fee.cmp(&other.fee) { - core::cmp::Ordering::Equal => {} - ord => return ord, - } - // match self.mass.cmp(&other.mass) { - // core::cmp::Ordering::Equal => {} - // ord => return ord, - // } - self.id.cmp(&other.id) - } -} +use super::frontier::Frontier; /// Pool of transactions to be included in a block template /// @@ -92,7 +57,7 @@ pub(crate) struct TransactionsPool { /// Transactions dependencies formed by outputs present in pool - successor relations. chained_transactions: TransactionsEdges, /// Transactions with no parents in the mempool -- ready to be inserted into a block template - ready_transactions: BTreeSet, + ready_transactions: Frontier, last_expire_scan_daa_score: u64, /// last expire scan time in milliseconds @@ -201,16 +166,15 @@ impl TransactionsPool { self.ready_transactions.len() } - /// all_ready_transactions returns all fully populated mempool transactions having no parents in the mempool. + /// all_ready_transactions returns a representative sample of fully populated + /// mempool transactions having no parents in the mempool. /// These transactions are ready for being inserted in a block template. pub(crate) fn all_ready_transactions(&self) -> Vec { // The returned transactions are leaving the mempool so they are cloned - // self.ready_transactions.range(range) + let mut rng = thread_rng(); self.ready_transactions - .iter() - .rev() // Iterate in descending feerate order - .take(self.config.maximum_ready_transaction_count as usize) - .map(|key| CandidateTransaction::from_mutable(&self.all_transactions.get(&key.id).unwrap().mtx)) + .sample(&mut rng, self.config.maximum_ready_transaction_count) + .map(|key| CandidateTransaction::from_mutable(&self.all_transactions.get(&key).unwrap().mtx)) .collect() } @@ -271,8 +235,8 @@ impl TransactionsPool { // An error is returned if the mempool is filled with high priority and other unremovable transactions. let tx_count = self.len() + free_slots - transactions_to_remove.len(); - if tx_count as u64 > self.config.maximum_transaction_count { - let err = RuleError::RejectMempoolIsFull(tx_count - free_slots, self.config.maximum_transaction_count); + if tx_count as u64 > self.config.maximum_transaction_count as u64 { + let err = RuleError::RejectMempoolIsFull(tx_count - free_slots, self.config.maximum_transaction_count as u64); warn!("{}", err.to_string()); return Err(err); } diff --git a/mining/src/mempool/model/tx.rs b/mining/src/mempool/model/tx.rs index 48f24b9f6..9b65faeb2 100644 --- a/mining/src/mempool/model/tx.rs +++ b/mining/src/mempool/model/tx.rs @@ -2,7 +2,6 @@ use crate::mempool::tx::{Priority, RbfPolicy}; use kaspa_consensus_core::tx::{MutableTransaction, Transaction, TransactionId, TransactionOutpoint}; use kaspa_mining_errors::mempool::RuleError; use std::{ - cmp::Ordering, fmt::{Display, Formatter}, sync::Arc, }; @@ -35,26 +34,6 @@ impl MempoolTransaction { } } -impl Ord for MempoolTransaction { - fn cmp(&self, other: &Self) -> Ordering { - self.fee_rate().total_cmp(&other.fee_rate()).then(self.id().cmp(&other.id())) - } -} - -impl Eq for MempoolTransaction {} - -impl PartialOrd for MempoolTransaction { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl PartialEq for MempoolTransaction { - fn eq(&self, other: &Self) -> bool { - self.fee_rate() == other.fee_rate() - } -} - impl RbfPolicy { #[cfg(test)] /// Returns an alternate policy accepting a transaction insertion in case the policy requires a replacement From 02a2f7aedce08d472b42a8eee03fd6c281f089e1 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 1 Aug 2024 10:57:27 +0000 Subject: [PATCH 04/74] mempool sampling benchmark (wip) --- Cargo.lock | 1 + mining/benches/bench.rs | 79 ++++++++++++++++++++++++++- mining/src/block_template/policy.rs | 4 +- mining/src/block_template/selector.rs | 6 +- mining/src/lib.rs | 4 ++ mining/src/model/candidate_tx.rs | 2 +- mining/src/model/mod.rs | 2 +- 7 files changed, 88 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d4703b8a..358ce3893 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3031,6 +3031,7 @@ version = "0.14.1" dependencies = [ "criterion", "futures-util", + "indexmap 2.2.6", "itertools 0.11.0", "kaspa-addresses", "kaspa-consensus-core", diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index 59ff685dd..ee8fa7777 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -1,6 +1,19 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use kaspa_mining::model::topological_index::TopologicalIndex; -use std::collections::{hash_set::Iter, HashMap, HashSet}; +use itertools::Itertools; +use kaspa_consensus_core::{ + subnets::SUBNETWORK_ID_NATIVE, + tx::{Transaction, TransactionId, TransactionInput, TransactionOutpoint}, +}; +use kaspa_hashes::{HasherBase, TransactionID}; +use kaspa_mining::{ + model::{candidate_tx::CandidateTransaction, topological_index::TopologicalIndex}, + FeerateTransactionKey, Frontier, Policy, TransactionsSelector, +}; +use rand::{thread_rng, Rng}; +use std::{ + collections::{hash_set::Iter, HashMap, HashSet}, + sync::Arc, +}; #[derive(Default)] pub struct Dag @@ -68,5 +81,65 @@ pub fn bench_compare_topological_index_fns(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_compare_topological_index_fns); +fn generate_unique_tx(i: u64) -> Arc { + let mut hasher = TransactionID::new(); + let prev = hasher.update(i.to_le_bytes()).clone().finalize(); + let input = TransactionInput::new(TransactionOutpoint::new(prev, 0), vec![], 0, 0); + Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) +} + +fn stage_two_sampling( + container: impl IntoIterator, + _map: &HashMap, +) -> Vec { + let tx = generate_unique_tx(u64::MAX); + let set = container + .into_iter() + .map(|_h| { + // let k = map.get(&h).unwrap(); + CandidateTransaction { calculated_fee: 2500, calculated_mass: 1650, tx: tx.clone() } + }) + .collect_vec(); + let mut selector = TransactionsSelector::new(Policy::new(500_000), set); + selector.select_transactions() +} + +pub fn bench_two_stage_sampling(c: &mut Criterion) { + let mut rng = thread_rng(); + let mut group = c.benchmark_group("mempool sampling"); + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = rng.gen_range(1..100000); + let mass: u64 = 1650; + let tx = generate_unique_tx(i); + map.insert(tx.id(), FeerateTransactionKey { fee: fee.max(mass), mass, id: tx.id() }); + } + + let len = cap; + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + group.bench_function("mempool sample stage one", |b| { + b.iter(|| { + black_box({ + let stage_one = frontier.sample(&mut rng, 10_000); + stage_one.into_iter().map(|k| k.as_bytes()[0] as u64).sum::() + }) + }) + }); + group.bench_function("mempool sample stage one & two", |b| { + b.iter(|| { + black_box({ + let stage_one = frontier.sample(&mut rng, 10_000); + let stage_two = stage_two_sampling(stage_one, &map); + stage_two.into_iter().map(|k| k.gas).sum::() + }) + }) + }); + group.finish(); +} + +criterion_group!(benches, bench_two_stage_sampling, bench_compare_topological_index_fns); criterion_main!(benches); diff --git a/mining/src/block_template/policy.rs b/mining/src/block_template/policy.rs index ff5197255..ca6185142 100644 --- a/mining/src/block_template/policy.rs +++ b/mining/src/block_template/policy.rs @@ -2,13 +2,13 @@ /// the generation of block templates. See the documentation for /// NewBlockTemplate for more details on each of these parameters are used. #[derive(Clone)] -pub(crate) struct Policy { +pub struct Policy { /// max_block_mass is the maximum block mass to be used when generating a block template. pub(crate) max_block_mass: u64, } impl Policy { - pub(crate) fn new(max_block_mass: u64) -> Self { + pub fn new(max_block_mass: u64) -> Self { Self { max_block_mass } } } diff --git a/mining/src/block_template/selector.rs b/mining/src/block_template/selector.rs index 0059ad92e..8c5046935 100644 --- a/mining/src/block_template/selector.rs +++ b/mining/src/block_template/selector.rs @@ -28,7 +28,7 @@ pub(crate) const ALPHA: i32 = 3; /// if REBALANCE_THRESHOLD is 0.95, there's a 1-in-20 chance of collision. const REBALANCE_THRESHOLD: f64 = 0.95; -pub(crate) struct TransactionsSelector { +pub struct TransactionsSelector { policy: Policy, /// Transaction store transactions: Vec, @@ -53,7 +53,7 @@ pub(crate) struct TransactionsSelector { } impl TransactionsSelector { - pub(crate) fn new(policy: Policy, mut transactions: Vec) -> Self { + pub fn new(policy: Policy, mut transactions: Vec) -> Self { let _sw = Stopwatch::<100>::with_threshold("TransactionsSelector::new op"); // Sort the transactions by subnetwork_id. transactions.sort_by(|a, b| a.tx.subnetwork_id.cmp(&b.tx.subnetwork_id)); @@ -103,7 +103,7 @@ impl TransactionsSelector { /// select_transactions loops over the candidate transactions /// and appends the ones that will be included in the next block into /// selected_txs. - pub(crate) fn select_transactions(&mut self) -> Vec { + pub fn select_transactions(&mut self) -> Vec { let _sw = Stopwatch::<15>::with_threshold("select_transaction op"); let mut rng = rand::thread_rng(); diff --git a/mining/src/lib.rs b/mining/src/lib.rs index ba5f6c6a6..a2cbe8793 100644 --- a/mining/src/lib.rs +++ b/mining/src/lib.rs @@ -15,6 +15,10 @@ pub mod mempool; pub mod model; pub mod monitor; +// Exposed for benchmarks +pub use block_template::{policy::Policy, selector::TransactionsSelector}; +pub use mempool::model::{feerate_key::FeerateTransactionKey, frontier::Frontier}; + #[cfg(test)] pub mod testutils; diff --git a/mining/src/model/candidate_tx.rs b/mining/src/model/candidate_tx.rs index f1fdf7c71..70f7d4ba1 100644 --- a/mining/src/model/candidate_tx.rs +++ b/mining/src/model/candidate_tx.rs @@ -4,7 +4,7 @@ use std::sync::Arc; /// Transaction with additional metadata needed in order to be a candidate /// in the transaction selection algorithm #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct CandidateTransaction { +pub struct CandidateTransaction { /// The actual transaction pub tx: Arc, /// Populated fee diff --git a/mining/src/model/mod.rs b/mining/src/model/mod.rs index 66c91cae5..dcec6f17f 100644 --- a/mining/src/model/mod.rs +++ b/mining/src/model/mod.rs @@ -1,7 +1,7 @@ use kaspa_consensus_core::tx::TransactionId; use std::collections::HashSet; -pub(crate) mod candidate_tx; +pub mod candidate_tx; pub mod owner_txs; pub mod topological_index; pub mod topological_sort; From 488d0cb15d7ea4ffea65a873bfb9f203736de4de Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 1 Aug 2024 15:49:27 +0000 Subject: [PATCH 05/74] Use arc tx rather than tx id in order to save the indirect map access as well as reduce frontier sizes + filter all the top bucket and not only selected ones --- mining/benches/bench.rs | 26 +-- mining/src/mempool/model/feerate_key.rs | 14 +- mining/src/mempool/model/frontier.rs | 151 ++++++++++++------ mining/src/mempool/model/transactions_pool.rs | 2 +- mining/src/model/candidate_tx.rs | 15 +- utils/src/vec.rs | 9 ++ 6 files changed, 138 insertions(+), 79 deletions(-) diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index ee8fa7777..b8998e24d 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -2,7 +2,7 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use itertools::Itertools; use kaspa_consensus_core::{ subnets::SUBNETWORK_ID_NATIVE, - tx::{Transaction, TransactionId, TransactionInput, TransactionOutpoint}, + tx::{Transaction, TransactionInput, TransactionOutpoint}, }; use kaspa_hashes::{HasherBase, TransactionID}; use kaspa_mining::{ @@ -88,18 +88,8 @@ fn generate_unique_tx(i: u64) -> Arc { Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) } -fn stage_two_sampling( - container: impl IntoIterator, - _map: &HashMap, -) -> Vec { - let tx = generate_unique_tx(u64::MAX); - let set = container - .into_iter() - .map(|_h| { - // let k = map.get(&h).unwrap(); - CandidateTransaction { calculated_fee: 2500, calculated_mass: 1650, tx: tx.clone() } - }) - .collect_vec(); +fn stage_two_sampling(container: impl IntoIterator) -> Vec { + let set = container.into_iter().map(CandidateTransaction::from_key).collect_vec(); let mut selector = TransactionsSelector::new(Policy::new(500_000), set); selector.select_transactions() } @@ -113,10 +103,10 @@ pub fn bench_two_stage_sampling(c: &mut Criterion) { let fee: u64 = rng.gen_range(1..100000); let mass: u64 = 1650; let tx = generate_unique_tx(i); - map.insert(tx.id(), FeerateTransactionKey { fee: fee.max(mass), mass, id: tx.id() }); + map.insert(tx.id(), FeerateTransactionKey { fee: fee.max(mass), mass, tx }); } - let len = cap; + let len = cap; // / 10; let mut frontier = Frontier::default(); for item in map.values().take(len).cloned() { frontier.insert(item).then_some(()).unwrap(); @@ -124,8 +114,8 @@ pub fn bench_two_stage_sampling(c: &mut Criterion) { group.bench_function("mempool sample stage one", |b| { b.iter(|| { black_box({ - let stage_one = frontier.sample(&mut rng, 10_000); - stage_one.into_iter().map(|k| k.as_bytes()[0] as u64).sum::() + let stage_one = frontier.sample(&mut rng, 10_000).collect_vec(); + stage_one.into_iter().map(|k| k.mass).sum::() }) }) }); @@ -133,7 +123,7 @@ pub fn bench_two_stage_sampling(c: &mut Criterion) { b.iter(|| { black_box({ let stage_one = frontier.sample(&mut rng, 10_000); - let stage_two = stage_two_sampling(stage_one, &map); + let stage_two = stage_two_sampling(stage_one); stage_two.into_iter().map(|k| k.gas).sum::() }) }) diff --git a/mining/src/mempool/model/feerate_key.rs b/mining/src/mempool/model/feerate_key.rs index 833711f57..494443955 100644 --- a/mining/src/mempool/model/feerate_key.rs +++ b/mining/src/mempool/model/feerate_key.rs @@ -1,19 +1,19 @@ -use kaspa_consensus_core::tx::TransactionId; - use super::tx::MempoolTransaction; +use kaspa_consensus_core::tx::Transaction; +use std::sync::Arc; #[derive(Clone, Debug)] pub struct FeerateTransactionKey { pub fee: u64, pub mass: u64, - pub id: TransactionId, + pub tx: Arc, } impl Eq for FeerateTransactionKey {} impl PartialEq for FeerateTransactionKey { fn eq(&self, other: &Self) -> bool { - self.id == other.id + self.tx.id() == other.tx.id() } } @@ -26,7 +26,7 @@ impl FeerateTransactionKey { impl std::hash::Hash for FeerateTransactionKey { fn hash(&self, state: &mut H) { // Transaction id is a sufficient identifier for this key - self.id.hash(state); + self.tx.id().hash(state); } } @@ -57,12 +57,12 @@ impl Ord for FeerateTransactionKey { // Finally, we compare transaction ids in order to allow multiple transactions with // the same fee and mass to exist within the same sorted container - self.id.cmp(&other.id) + self.tx.id().cmp(&other.tx.id()) } } impl From<&MempoolTransaction> for FeerateTransactionKey { fn from(tx: &MempoolTransaction) -> Self { - Self { fee: tx.mtx.calculated_fee.unwrap(), mass: tx.mtx.tx.mass(), id: tx.id() } + Self { fee: tx.mtx.calculated_fee.unwrap(), mass: tx.mtx.tx.mass(), tx: tx.mtx.tx.clone() } } } diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 0b1b6f059..e7f0dc626 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -3,7 +3,7 @@ use crate::block_template::selector::ALPHA; use super::feerate_key::FeerateTransactionKey; use indexmap::IndexSet; use itertools::Either; -use kaspa_consensus_core::tx::TransactionId; +use kaspa_utils::vec::VecExtensions; use rand::{distributions::Uniform, prelude::Distribution, Rng}; use std::collections::{BTreeSet, HashSet}; @@ -16,7 +16,7 @@ pub struct Frontier { feerate_order: BTreeSet, /// Frontier transactions accessible via random access - index: IndexSet, + index: IndexSet, /// Total sampling weight: Σ_{tx in frontier}(tx.fee/tx.mass)^alpha total_weight: f64, @@ -32,7 +32,7 @@ impl Frontier { pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { let (weight, mass) = (key.feerate().powi(ALPHA), key.mass); - self.index.insert(key.id); + self.index.insert(key.clone()); if self.feerate_order.insert(key) { self.total_weight += weight; self.total_mass += mass; @@ -44,7 +44,7 @@ impl Frontier { pub fn remove(&mut self, key: &FeerateTransactionKey) -> bool { let (weight, mass) = (key.feerate().powi(ALPHA), key.mass); - self.index.swap_remove(&key.id); + self.index.swap_remove(key); if self.feerate_order.remove(key) { self.total_weight -= weight; self.total_mass -= mass; @@ -54,7 +54,7 @@ impl Frontier { } } - fn sample_top_bucket<'a, R>(&'a self, rng: &'a mut R, overall_amount: u32) -> impl Iterator + 'a + fn sample_top_bucket(&self, rng: &mut R, overall_amount: u32) -> (Vec, HashSet) where R: Rng + ?Sized, { @@ -62,51 +62,60 @@ impl Frontier { debug_assert!(overall_amount <= frontier_length); let sampling_ratio = overall_amount as f64 / frontier_length as f64; let distr = Uniform::new(0f64, 1f64); - self.feerate_order - .iter() - .rev() - .map_while(move |key| { - let weight = key.feerate().powi(ALPHA); - let exclusive_total_weight = self.total_weight - weight; - let sample_approx_weight = exclusive_total_weight * sampling_ratio; - if weight < exclusive_total_weight / 100.0 { - None // break out map_while - } else { - let p = weight / self.total_weight; - let p_s = weight / (sample_approx_weight + weight); - debug_assert!(p <= p_s); - // Flip a coin with the reversed probability - if distr.sample(rng) < (sample_approx_weight + weight) / self.total_weight { - Some(Some(self.index.get_index_of(&key.id).unwrap() as u32)) + let mut filter = HashSet::new(); + let filter_ref = &mut filter; + ( + self.feerate_order + .iter() + .rev() + .map_while(move |key| { + let weight = key.feerate().powi(ALPHA); + let exclusive_total_weight = self.total_weight - weight; + let sample_approx_weight = exclusive_total_weight * sampling_ratio; + if weight < exclusive_total_weight / 100.0 { + None // break out map_while } else { - Some(None) // signals a continue but not a break + let p = weight / self.total_weight; + let p_s = weight / (sample_approx_weight + weight); + debug_assert!(p <= p_s); + let idx = self.index.get_index_of(key).unwrap() as u32; + // Register this index as "already sampled" + filter_ref.insert(idx); + // Flip a coin with the reversed probability + if distr.sample(rng) < (sample_approx_weight + weight) / self.total_weight { + Some(Some(idx)) + } else { + Some(None) // signals a continue but not a break + } } - } - }) - .flatten() + }) + .flatten() + .collect(), + filter, + ) } - pub fn sample<'a, R>(&'a self, rng: &'a mut R, overall_amount: u32) -> impl Iterator + 'a + pub fn sample<'a, R>(&'a self, rng: &'a mut R, overall_amount: u32) -> impl Iterator + 'a where R: Rng + ?Sized, { let frontier_length = self.feerate_order.len() as u32; if frontier_length <= overall_amount { - return Either::Left(self.index.iter().copied()); + return Either::Left(self.index.iter().cloned()); } // Based on values taken from `rand::seq::index::sample` const C: [f32; 2] = [270.0, 330.0 / 9.0]; let j = if frontier_length < 500_000 { 0 } else { 1 }; let indices = if (frontier_length as f32) < C[j] * (overall_amount as f32) { - let initial = self.sample_top_bucket(rng, overall_amount).collect::>(); - sample_inplace(rng, frontier_length, overall_amount, initial) + let (top, filter) = self.sample_top_bucket(rng, overall_amount); + sample_inplace(rng, frontier_length, overall_amount - top.len() as u32, overall_amount, filter).chain(top) } else { - let initial = self.sample_top_bucket(rng, overall_amount).collect::>(); - sample_rejection(rng, frontier_length, overall_amount, initial) + let (top, filter) = self.sample_top_bucket(rng, overall_amount); + sample_rejection(rng, frontier_length, overall_amount - top.len() as u32, overall_amount, filter).chain(top) }; - Either::Right(indices.into_iter().map(|i| self.index.get_index(i as usize).copied().unwrap())) + Either::Right(indices.into_iter().map(|i| self.index.get_index(i as usize).cloned().unwrap())) } pub(crate) fn len(&self) -> usize { @@ -116,22 +125,19 @@ impl Frontier { /// Adaptation of `rand::seq::index::sample_inplace` for the case where there exists an /// initial a priory sample to begin with -fn sample_inplace(rng: &mut R, length: u32, amount: u32, initial: Vec) -> Vec +fn sample_inplace(rng: &mut R, length: u32, amount: u32, capacity: u32, filter: HashSet) -> Vec where R: Rng + ?Sized, { debug_assert!(amount <= length); - debug_assert!(initial.len() <= amount as usize); - let mut indices: Vec = Vec::with_capacity(length as usize); + debug_assert!(filter.len() <= amount as usize); + let mut indices: Vec = Vec::with_capacity(length.max(capacity) as usize); indices.extend(0..length); - let initial_len = initial.len() as u32; - for (i, j) in initial.into_iter().enumerate() { - debug_assert!(i <= (j as usize)); - debug_assert_eq!(j, indices[j as usize]); - indices.swap(i, j as usize); - } - for i in initial_len..amount { - let j: u32 = rng.gen_range(i..length); + for i in 0..amount { + let mut j: u32 = rng.gen_range(i..length); + while filter.contains(&j) { + j = rng.gen_range(i..length); + } indices.swap(i as usize, j as usize); } indices.truncate(amount as usize); @@ -141,18 +147,17 @@ where /// Adaptation of `rand::seq::index::sample_rejection` for the case where there exists an /// initial a priory sample to begin with -fn sample_rejection(rng: &mut R, length: u32, amount: u32, mut initial: HashSet) -> Vec +fn sample_rejection(rng: &mut R, length: u32, amount: u32, capacity: u32, mut filter: HashSet) -> Vec where R: Rng + ?Sized, { debug_assert!(amount < length); - debug_assert!(initial.len() <= amount as usize); + debug_assert!(filter.len() <= amount as usize); let distr = Uniform::new(0, length); - let mut indices = Vec::with_capacity(amount as usize); - indices.extend(initial.iter().copied()); + let mut indices = Vec::with_capacity(amount.max(capacity) as usize); for _ in indices.len()..amount as usize { let mut pos = distr.sample(rng); - while !initial.insert(pos) { + while !filter.insert(pos) { pos = distr.sample(rng); } indices.push(pos); @@ -161,3 +166,53 @@ where assert_eq!(indices.len(), amount as usize); indices } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{model::candidate_tx::CandidateTransaction, Policy, TransactionsSelector}; + use itertools::Itertools; + use kaspa_consensus_core::{ + subnets::SUBNETWORK_ID_NATIVE, + tx::{Transaction, TransactionInput, TransactionOutpoint}, + }; + use kaspa_hashes::{HasherBase, TransactionID}; + use rand::thread_rng; + use std::{collections::HashMap, sync::Arc}; + + fn generate_unique_tx(i: u64) -> Arc { + let mut hasher = TransactionID::new(); + let prev = hasher.update(i.to_le_bytes()).clone().finalize(); + let input = TransactionInput::new(TransactionOutpoint::new(prev, 0), vec![], 0, 0); + Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) + } + + fn stage_two_sampling(container: impl IntoIterator) -> Vec { + let set = container.into_iter().map(CandidateTransaction::from_key).collect_vec(); + let mut selector = TransactionsSelector::new(Policy::new(500_000), set); + selector.select_transactions() + } + + #[test] + pub fn test_two_stage_sampling() { + let mut rng = thread_rng(); + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = rng.gen_range(1..100000); + let mass: u64 = 1650; + let tx = generate_unique_tx(i); + map.insert(tx.id(), FeerateTransactionKey { fee: fee.max(mass), mass, tx }); + } + + let len = cap; // / 10; + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let stage_one = frontier.sample(&mut rng, 10_000); + let stage_two = stage_two_sampling(stage_one); + stage_two.into_iter().map(|k| k.gas).sum::(); + } +} diff --git a/mining/src/mempool/model/transactions_pool.rs b/mining/src/mempool/model/transactions_pool.rs index 97842b543..f16bc796f 100644 --- a/mining/src/mempool/model/transactions_pool.rs +++ b/mining/src/mempool/model/transactions_pool.rs @@ -174,7 +174,7 @@ impl TransactionsPool { let mut rng = thread_rng(); self.ready_transactions .sample(&mut rng, self.config.maximum_ready_transaction_count) - .map(|key| CandidateTransaction::from_mutable(&self.all_transactions.get(&key).unwrap().mtx)) + .map(CandidateTransaction::from_key) .collect() } diff --git a/mining/src/model/candidate_tx.rs b/mining/src/model/candidate_tx.rs index 70f7d4ba1..858758eaf 100644 --- a/mining/src/model/candidate_tx.rs +++ b/mining/src/model/candidate_tx.rs @@ -1,4 +1,5 @@ -use kaspa_consensus_core::tx::{MutableTransaction, Transaction}; +use crate::FeerateTransactionKey; +use kaspa_consensus_core::tx::Transaction; use std::sync::Arc; /// Transaction with additional metadata needed in order to be a candidate @@ -14,9 +15,13 @@ pub struct CandidateTransaction { } impl CandidateTransaction { - pub(crate) fn from_mutable(tx: &MutableTransaction) -> Self { - let mass = tx.tx.mass(); - assert_ne!(mass, 0, "mass field is expected to be set when inserting to the mempool"); - Self { tx: tx.tx.clone(), calculated_fee: tx.calculated_fee.expect("fee is expected to be populated"), calculated_mass: mass } + // pub(crate) fn from_mutable(tx: &MutableTransaction) -> Self { + // let mass = tx.tx.mass(); + // assert_ne!(mass, 0, "mass field is expected to be set when inserting to the mempool"); + // Self { tx: tx.tx.clone(), calculated_fee: tx.calculated_fee.expect("fee is expected to be populated"), calculated_mass: mass } + // } + + pub fn from_key(key: FeerateTransactionKey) -> Self { + Self { tx: key.tx, calculated_fee: key.fee, calculated_mass: key.mass } } } diff --git a/utils/src/vec.rs b/utils/src/vec.rs index 01bd59b9e..0ab30add2 100644 --- a/utils/src/vec.rs +++ b/utils/src/vec.rs @@ -4,6 +4,10 @@ pub trait VecExtensions { /// Inserts the provided `value` at `index` while swapping the item at index to the end of the container fn swap_insert(&mut self, index: usize, value: T); + + /// Chains two containers one after the other and returns the result. The method is identical + /// to [`Vec::append`] but can be used more ergonomically in a fluent calling fashion + fn chain(self, other: Self) -> Self; } impl VecExtensions for Vec { @@ -19,4 +23,9 @@ impl VecExtensions for Vec { let loc = self.len() - 1; self.swap(index, loc); } + + fn chain(mut self, mut other: Self) -> Self { + self.append(&mut other); + self + } } From f191ae381dfe8fe3a9a42b71eb281ae2ae2d169d Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 1 Aug 2024 15:50:20 +0000 Subject: [PATCH 06/74] Modify mempool bmk and simnet settings --- consensus/core/src/config/params.rs | 3 ++- testing/integration/src/mempool_benchmarks.rs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/consensus/core/src/config/params.rs b/consensus/core/src/config/params.rs index e2f2639a1..f512cf1e1 100644 --- a/consensus/core/src/config/params.rs +++ b/consensus/core/src/config/params.rs @@ -501,7 +501,8 @@ pub const SIMNET_PARAMS: Params = Params { target_time_per_block: Testnet11Bps::target_time_per_block(), past_median_time_sample_rate: Testnet11Bps::past_median_time_sample_rate(), difficulty_sample_rate: Testnet11Bps::difficulty_adjustment_sample_rate(), - max_block_parents: Testnet11Bps::max_block_parents(), + // For simnet, we deviate from TN11 configuration and allow at least 64 parents in order to support mempool benchmarks out of the box + max_block_parents: if Testnet11Bps::max_block_parents() > 64 { Testnet11Bps::max_block_parents() } else { 64 }, mergeset_size_limit: Testnet11Bps::mergeset_size_limit(), merge_depth: Testnet11Bps::merge_depth_bound(), finality_depth: Testnet11Bps::finality_depth(), diff --git a/testing/integration/src/mempool_benchmarks.rs b/testing/integration/src/mempool_benchmarks.rs index 3df716594..00d9b7803 100644 --- a/testing/integration/src/mempool_benchmarks.rs +++ b/testing/integration/src/mempool_benchmarks.rs @@ -295,8 +295,8 @@ async fn bench_bbt_latency_2() { const BLOCK_COUNT: usize = usize::MAX; const MEMPOOL_TARGET: u64 = 600_000; - const TX_COUNT: usize = 1_400_000; - const TX_LEVEL_WIDTH: usize = 20_000; + const TX_COUNT: usize = 1_000_000; + const TX_LEVEL_WIDTH: usize = 300_000; const TPS_PRESSURE: u64 = u64::MAX; const SUBMIT_BLOCK_CLIENTS: usize = 20; From 0eb70e8f3105e507fe18aef45d2cdbd25e572336 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 1 Aug 2024 15:50:37 +0000 Subject: [PATCH 07/74] Temp: rpc message initial --- rpc/core/src/model/message.rs | 54 +++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/rpc/core/src/model/message.rs b/rpc/core/src/model/message.rs index d19984b58..2dd7261dd 100644 --- a/rpc/core/src/model/message.rs +++ b/rpc/core/src/model/message.rs @@ -850,6 +850,60 @@ impl GetDaaScoreTimestampEstimateResponse { } } +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// Fee rate estimations + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateRequest {} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct FeerateBucket { + pub feerate: f64, + pub estimated_seconds: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateResponse { + pub low_bucket: FeerateBucket, + pub normal_bucket: FeerateBucket, + pub priority_bucket: FeerateBucket, +} + +// pub struct GetFeeEstimateExperimentalRequest { +// pub verbose: bool, +// } + +// pub struct GetFeeEstimateExperimentalResponse { +// // Same as the usual response +// pub low_bucket: FeerateBucket, +// pub normal_bucket: FeerateBucket, +// pub priority_bucket: FeerateBucket, + +// /// Experimental verbose data +// pub verbose: Option, +// } + +// pub struct FeeEstimateVerboseExperimentalData { +// // mempool load factor in relation to tx/s +// // processing capacity +// pub mempool_load_factor: f64, +// // temperature of the mempool water +// pub mempool_water_temperature_celsius: f64, +// // optional internal context data that +// // represents components used to calculate +// // fee estimates and time periods. +// // ... + +// // total_mass, total_fee, etc +// /// Built from Maxim's implementation +// pub next_block_template_feerate_min: f64, +// pub next_block_template_feerate_median: f64, +// pub next_block_template_feerate_max: f64, +// } + // ---------------------------------------------------------------------------- // Subscriptions & notifications // ---------------------------------------------------------------------------- From 71a8609812859d997a97a472830462ca7d0ef908 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 1 Aug 2024 17:17:27 +0000 Subject: [PATCH 08/74] Move sample to rand utils --- mining/src/mempool/model/frontier.rs | 71 +++------------------------- utils/Cargo.toml | 2 +- utils/src/lib.rs | 1 + utils/src/rand/mod.rs | 1 + utils/src/rand/seq.rs | 60 +++++++++++++++++++++++ 5 files changed, 70 insertions(+), 65 deletions(-) create mode 100644 utils/src/rand/mod.rs create mode 100644 utils/src/rand/seq.rs diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index e7f0dc626..6788c8254 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -1,9 +1,8 @@ -use crate::block_template::selector::ALPHA; - use super::feerate_key::FeerateTransactionKey; +use crate::block_template::selector::ALPHA; use indexmap::IndexSet; use itertools::Either; -use kaspa_utils::vec::VecExtensions; +use kaspa_utils::{rand::seq::index, vec::VecExtensions}; use rand::{distributions::Uniform, prelude::Distribution, Rng}; use std::collections::{BTreeSet, HashSet}; @@ -26,10 +25,6 @@ pub struct Frontier { } impl Frontier { - // pub fn new() -> Self { - // Self { ..Default::default() } - // } - pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { let (weight, mass) = (key.feerate().powi(ALPHA), key.mass); self.index.insert(key.clone()); @@ -104,17 +99,9 @@ impl Frontier { return Either::Left(self.index.iter().cloned()); } - // Based on values taken from `rand::seq::index::sample` - const C: [f32; 2] = [270.0, 330.0 / 9.0]; - let j = if frontier_length < 500_000 { 0 } else { 1 }; - let indices = if (frontier_length as f32) < C[j] * (overall_amount as f32) { - let (top, filter) = self.sample_top_bucket(rng, overall_amount); - sample_inplace(rng, frontier_length, overall_amount - top.len() as u32, overall_amount, filter).chain(top) - } else { - let (top, filter) = self.sample_top_bucket(rng, overall_amount); - sample_rejection(rng, frontier_length, overall_amount - top.len() as u32, overall_amount, filter).chain(top) - }; - + let (top, filter) = self.sample_top_bucket(rng, overall_amount); + // println!("|F|: {}, |P|: {}", filter.len(), top.len()); + let indices = index::sample(rng, frontier_length, overall_amount - top.len() as u32, overall_amount, filter).chain(top); Either::Right(indices.into_iter().map(|i| self.index.get_index(i as usize).cloned().unwrap())) } @@ -123,50 +110,6 @@ impl Frontier { } } -/// Adaptation of `rand::seq::index::sample_inplace` for the case where there exists an -/// initial a priory sample to begin with -fn sample_inplace(rng: &mut R, length: u32, amount: u32, capacity: u32, filter: HashSet) -> Vec -where - R: Rng + ?Sized, -{ - debug_assert!(amount <= length); - debug_assert!(filter.len() <= amount as usize); - let mut indices: Vec = Vec::with_capacity(length.max(capacity) as usize); - indices.extend(0..length); - for i in 0..amount { - let mut j: u32 = rng.gen_range(i..length); - while filter.contains(&j) { - j = rng.gen_range(i..length); - } - indices.swap(i as usize, j as usize); - } - indices.truncate(amount as usize); - debug_assert_eq!(indices.len(), amount as usize); - indices -} - -/// Adaptation of `rand::seq::index::sample_rejection` for the case where there exists an -/// initial a priory sample to begin with -fn sample_rejection(rng: &mut R, length: u32, amount: u32, capacity: u32, mut filter: HashSet) -> Vec -where - R: Rng + ?Sized, -{ - debug_assert!(amount < length); - debug_assert!(filter.len() <= amount as usize); - let distr = Uniform::new(0, length); - let mut indices = Vec::with_capacity(amount.max(capacity) as usize); - for _ in indices.len()..amount as usize { - let mut pos = distr.sample(rng); - while !filter.insert(pos) { - pos = distr.sample(rng); - } - indices.push(pos); - } - - assert_eq!(indices.len(), amount as usize); - indices -} - #[cfg(test)] mod tests { use super::*; @@ -196,10 +139,10 @@ mod tests { #[test] pub fn test_two_stage_sampling() { let mut rng = thread_rng(); - let cap = 1_000_000; + let cap = 100_000; let mut map = HashMap::with_capacity(cap); for i in 0..cap as u64 { - let fee: u64 = rng.gen_range(1..100000); + let fee: u64 = if i % (cap as u64 / 100) == 0 { 1000000 } else { rng.gen_range(1..10000) }; let mass: u64 = 1650; let tx = generate_unique_tx(i); map.insert(tx.id(), FeerateTransactionKey { fee: fee.max(mass), mass, tx }); diff --git a/utils/Cargo.toml b/utils/Cargo.toml index a3002afab..cecc69450 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -25,6 +25,7 @@ triggered.workspace = true uuid.workspace = true log.workspace = true wasm-bindgen.workspace = true +rand.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] rlimit.workspace = true @@ -36,7 +37,6 @@ async-trait.workspace = true futures-util.workspace = true tokio = { workspace = true, features = ["rt", "time", "macros"] } criterion.workspace = true -rand.workspace = true [[bench]] name = "bench" diff --git a/utils/src/lib.rs b/utils/src/lib.rs index bd3143719..7a54e4078 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -8,6 +8,7 @@ pub mod iter; pub mod mem_size; pub mod networking; pub mod option; +pub mod rand; pub mod refs; pub mod as_slice; diff --git a/utils/src/rand/mod.rs b/utils/src/rand/mod.rs new file mode 100644 index 000000000..ed6dcf110 --- /dev/null +++ b/utils/src/rand/mod.rs @@ -0,0 +1 @@ +pub mod seq; diff --git a/utils/src/rand/seq.rs b/utils/src/rand/seq.rs new file mode 100644 index 000000000..19948784c --- /dev/null +++ b/utils/src/rand/seq.rs @@ -0,0 +1,60 @@ +pub mod index { + use rand::{distributions::Uniform, prelude::Distribution, Rng}; + use std::collections::HashSet; + + /// Adaptation of `rand::seq::index::sample` for the case where there exists an a priory filter of indices + pub fn sample(rng: &mut R, length: u32, amount: u32, capacity: u32, filter: HashSet) -> Vec + where + R: Rng + ?Sized, + { + const C: [f32; 2] = [270.0, 330.0 / 9.0]; + let j = if length < 500_000 { 0 } else { 1 }; + if (length as f32) < C[j] * (amount as f32) { + sample_inplace(rng, length, amount, capacity, filter) + } else { + sample_rejection(rng, length, amount, capacity, filter) + } + } + + /// Adaptation of `rand::seq::index::sample_inplace` for the case where there exists an a priory filter of indices + fn sample_inplace(rng: &mut R, length: u32, amount: u32, capacity: u32, filter: HashSet) -> Vec + where + R: Rng + ?Sized, + { + debug_assert!(amount <= length); + debug_assert!(filter.len() <= amount as usize); + let mut indices: Vec = Vec::with_capacity(length.max(capacity) as usize); + indices.extend(0..length); + for i in 0..amount { + let mut j: u32 = rng.gen_range(i..length); + while filter.contains(&j) { + j = rng.gen_range(i..length); + } + indices.swap(i as usize, j as usize); + } + indices.truncate(amount as usize); + debug_assert_eq!(indices.len(), amount as usize); + indices + } + + /// Adaptation of `rand::seq::index::sample_rejection` for the case where there exists an a priory filter of indices + fn sample_rejection(rng: &mut R, length: u32, amount: u32, capacity: u32, mut filter: HashSet) -> Vec + where + R: Rng + ?Sized, + { + debug_assert!(amount < length); + debug_assert!(filter.len() <= amount as usize); + let distr = Uniform::new(0, length); + let mut indices = Vec::with_capacity(amount.max(capacity) as usize); + for _ in indices.len()..amount as usize { + let mut pos = distr.sample(rng); + while !filter.insert(pos) { + pos = distr.sample(rng); + } + indices.push(pos); + } + + assert_eq!(indices.len(), amount as usize); + indices + } +} From 415eef26b359c6154219732ed5b36ea27e9ea7b5 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 1 Aug 2024 17:36:43 +0000 Subject: [PATCH 09/74] Fix top bucket sampling to match analysis --- mining/src/mempool/model/frontier.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 6788c8254..27a036879 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -56,7 +56,8 @@ impl Frontier { let frontier_length = self.feerate_order.len() as u32; debug_assert!(overall_amount <= frontier_length); let sampling_ratio = overall_amount as f64 / frontier_length as f64; - let distr = Uniform::new(0f64, 1f64); + let eps = 0.0001; + let unit = Uniform::new(0f64, 1f64); let mut filter = HashSet::new(); let filter_ref = &mut filter; ( @@ -67,17 +68,14 @@ impl Frontier { let weight = key.feerate().powi(ALPHA); let exclusive_total_weight = self.total_weight - weight; let sample_approx_weight = exclusive_total_weight * sampling_ratio; - if weight < exclusive_total_weight / 100.0 { + if weight * (1.0 - sampling_ratio - eps) < eps * exclusive_total_weight { None // break out map_while } else { - let p = weight / self.total_weight; - let p_s = weight / (sample_approx_weight + weight); - debug_assert!(p <= p_s); let idx = self.index.get_index_of(key).unwrap() as u32; // Register this index as "already sampled" filter_ref.insert(idx); // Flip a coin with the reversed probability - if distr.sample(rng) < (sample_approx_weight + weight) / self.total_weight { + if unit.sample(rng) < (sample_approx_weight + weight) / self.total_weight { Some(Some(idx)) } else { Some(None) // signals a continue but not a break @@ -85,6 +83,7 @@ impl Frontier { } }) .flatten() + .take(overall_amount as usize) // Bound with overall amount just in case .collect(), filter, ) @@ -148,7 +147,7 @@ mod tests { map.insert(tx.id(), FeerateTransactionKey { fee: fee.max(mass), mass, tx }); } - let len = cap; // / 10; + let len = cap; let mut frontier = Frontier::default(); for item in map.values().take(len).cloned() { frontier.insert(item).then_some(()).unwrap(); From 54bf205a7499b091fcbac457bb128e7d12f900ae Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 1 Aug 2024 17:37:01 +0000 Subject: [PATCH 10/74] Add outliers to the bmk --- mining/benches/bench.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index b8998e24d..82f4dab66 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -100,7 +100,7 @@ pub fn bench_two_stage_sampling(c: &mut Criterion) { let cap = 1_000_000; let mut map = HashMap::with_capacity(cap); for i in 0..cap as u64 { - let fee: u64 = rng.gen_range(1..100000); + let fee: u64 = if i % (cap as u64 / 100000) == 0 { 1000000 } else { rng.gen_range(1..10000) }; let mass: u64 = 1650; let tx = generate_unique_tx(i); map.insert(tx.id(), FeerateTransactionKey { fee: fee.max(mass), mass, tx }); From df7b229f0d00a807bf85a84f03f2739637eebbe4 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 1 Aug 2024 17:48:36 +0000 Subject: [PATCH 11/74] sample comments and doc --- utils/src/rand/seq.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/utils/src/rand/seq.rs b/utils/src/rand/seq.rs index 19948784c..708c939d5 100644 --- a/utils/src/rand/seq.rs +++ b/utils/src/rand/seq.rs @@ -2,7 +2,12 @@ pub mod index { use rand::{distributions::Uniform, prelude::Distribution, Rng}; use std::collections::HashSet; - /// Adaptation of `rand::seq::index::sample` for the case where there exists an a priory filter of indices + /// Adaptation of [`rand::seq::index::sample`] for the case where there exists an a priory filter + /// of indices which should not be selected. + /// + /// Assumes `|filter| << length`. + /// + /// The argument `capacity` can be used to ensure a larger allocation within the returned vector. pub fn sample(rng: &mut R, length: u32, amount: u32, capacity: u32, filter: HashSet) -> Vec where R: Rng + ?Sized, @@ -16,7 +21,12 @@ pub mod index { } } - /// Adaptation of `rand::seq::index::sample_inplace` for the case where there exists an a priory filter of indices + /// Adaptation of [`rand::seq::index::sample_inplace`] for the case where there exists an a priory filter + /// of indices which should not be selected. + /// + /// Assumes `|filter| << length`. + /// + /// The argument `capacity` can be used to ensure a larger allocation within the returned vector. fn sample_inplace(rng: &mut R, length: u32, amount: u32, capacity: u32, filter: HashSet) -> Vec where R: Rng + ?Sized, @@ -27,6 +37,7 @@ pub mod index { indices.extend(0..length); for i in 0..amount { let mut j: u32 = rng.gen_range(i..length); + // Assumes |filter| << length while filter.contains(&j) { j = rng.gen_range(i..length); } @@ -37,7 +48,12 @@ pub mod index { indices } - /// Adaptation of `rand::seq::index::sample_rejection` for the case where there exists an a priory filter of indices + /// Adaptation of [`rand::seq::index::sample_rejection`] for the case where there exists an a priory filter + /// of indices which should not be selected. + /// + /// Assumes `|filter| << length`. + /// + /// The argument `capacity` can be used to ensure a larger allocation within the returned vector. fn sample_rejection(rng: &mut R, length: u32, amount: u32, capacity: u32, mut filter: HashSet) -> Vec where R: Rng + ?Sized, From f502c0ad89d00c65a33e5bf994096077fc954b18 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 1 Aug 2024 23:09:29 +0000 Subject: [PATCH 12/74] use b plus tree with argument customization in order to implement a highly-efficient O(k log n) one-shot mempool sampling --- Cargo.lock | 7 ++ mining/Cargo.toml | 3 +- mining/benches/bench.rs | 9 +- mining/src/mempool/model/frontier.rs | 168 +++++++++++++++++---------- 4 files changed, 122 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 358ce3893..81ab7ba80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3048,6 +3048,7 @@ dependencies = [ "secp256k1", "serde", "smallvec", + "sweep-bptree", "thiserror", "tokio", ] @@ -5780,6 +5781,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "sweep-bptree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea7b1b7c5eaabc40bab84ec98b2f12523d97e91c9bfc430fe5d2a1ea15c9960" + [[package]] name = "syn" version = "1.0.109" diff --git a/mining/Cargo.toml b/mining/Cargo.toml index 14b384a82..4fa786cda 100644 --- a/mining/Cargo.toml +++ b/mining/Cargo.toml @@ -29,7 +29,8 @@ serde.workspace = true smallvec.workspace = true thiserror.workspace = true indexmap.workspace = true -tokio = { workspace = true, features = [ "rt-multi-thread", "macros", "signal" ] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "signal"] } +sweep-bptree = "0.4.1" [dev-dependencies] kaspa-txscript.workspace = true diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index 82f4dab66..96c7a50e0 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -111,20 +111,19 @@ pub fn bench_two_stage_sampling(c: &mut Criterion) { for item in map.values().take(len).cloned() { frontier.insert(item).then_some(()).unwrap(); } - group.bench_function("mempool sample stage one", |b| { + group.bench_function("mempool sample 2 blocks", |b| { b.iter(|| { black_box({ - let stage_one = frontier.sample(&mut rng, 10_000).collect_vec(); + let stage_one = frontier.sample(&mut rng, 600).collect_vec(); stage_one.into_iter().map(|k| k.mass).sum::() }) }) }); - group.bench_function("mempool sample stage one & two", |b| { + group.bench_function("mempool sample 10k", |b| { b.iter(|| { black_box({ let stage_one = frontier.sample(&mut rng, 10_000); - let stage_two = stage_two_sampling(stage_one); - stage_two.into_iter().map(|k| k.gas).sum::() + stage_one.into_iter().map(|k| k.mass).sum::() }) }) }); diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 27a036879..2c8f45997 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -1,21 +1,73 @@ use super::feerate_key::FeerateTransactionKey; use crate::block_template::selector::ALPHA; +use arg::FeerateWeight; use indexmap::IndexSet; use itertools::Either; use kaspa_utils::{rand::seq::index, vec::VecExtensions}; use rand::{distributions::Uniform, prelude::Distribution, Rng}; use std::collections::{BTreeSet, HashSet}; +use sweep_bptree::BPlusTreeMap; + +pub mod arg { + use crate::block_template::selector::ALPHA; + use sweep_bptree::tree::{Argument, SearchArgument}; + + type FeerateKey = super::FeerateTransactionKey; + + #[derive(Clone, Copy, Debug, Default)] + pub struct FeerateWeight(f64); + + impl FeerateWeight { + /// Returns the weight value + pub fn weight(&self) -> f64 { + self.0 + } + } + + impl Argument for FeerateWeight { + fn from_leaf(keys: &[FeerateKey]) -> Self { + Self(keys.iter().map(|k| k.feerate().powi(ALPHA)).sum()) + } + + fn from_inner(_keys: &[FeerateKey], arguments: &[Self]) -> Self { + Self(arguments.iter().map(|a| a.0).sum()) + } + } + + impl SearchArgument for FeerateWeight { + type Query = f64; + + fn locate_in_leaf(query: Self::Query, keys: &[FeerateKey]) -> Option { + let mut sum = 0.0; + for (i, k) in keys.iter().enumerate() { + let w = k.feerate().powi(ALPHA); + sum += w; + if query <= sum { + return Some(i); + } + } + None + } + + fn locate_in_inner(mut query: Self::Query, _keys: &[FeerateKey], arguments: &[Self]) -> Option<(usize, Self::Query)> { + for (i, a) in arguments.iter().enumerate() { + if query >= a.0 { + query -= a.0; + } else { + return Some((i, query)); + } + } + None + } + } +} /// Management of the transaction pool frontier, that is, the set of transactions in /// the transaction pool which have no mempool ancestors and are essentially ready /// to enter the next block template. -#[derive(Default)] pub struct Frontier { /// Frontier transactions sorted by feerate order - feerate_order: BTreeSet, - - /// Frontier transactions accessible via random access - index: IndexSet, + feerate_order: BPlusTreeMap, /// Total sampling weight: Σ_{tx in frontier}(tx.fee/tx.mass)^alpha total_weight: f64, @@ -24,11 +76,16 @@ pub struct Frontier { total_mass: u64, } +impl Default for Frontier { + fn default() -> Self { + Self { feerate_order: BPlusTreeMap::new(), total_weight: Default::default(), total_mass: Default::default() } + } +} + impl Frontier { pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { let (weight, mass) = (key.feerate().powi(ALPHA), key.mass); - self.index.insert(key.clone()); - if self.feerate_order.insert(key) { + if self.feerate_order.insert(key, ()).is_none() { self.total_weight += weight; self.total_mass += mass; true @@ -39,8 +96,7 @@ impl Frontier { pub fn remove(&mut self, key: &FeerateTransactionKey) -> bool { let (weight, mass) = (key.feerate().powi(ALPHA), key.mass); - self.index.swap_remove(key); - if self.feerate_order.remove(key) { + if self.feerate_order.remove(&key).is_some() { self.total_weight -= weight; self.total_mass -= mass; true @@ -49,63 +105,29 @@ impl Frontier { } } - fn sample_top_bucket(&self, rng: &mut R, overall_amount: u32) -> (Vec, HashSet) + pub fn sample<'a, R>(&'a self, rng: &'a mut R, amount: u32) -> impl Iterator + 'a where R: Rng + ?Sized, { - let frontier_length = self.feerate_order.len() as u32; - debug_assert!(overall_amount <= frontier_length); - let sampling_ratio = overall_amount as f64 / frontier_length as f64; - let eps = 0.0001; - let unit = Uniform::new(0f64, 1f64); - let mut filter = HashSet::new(); - let filter_ref = &mut filter; - ( - self.feerate_order - .iter() - .rev() - .map_while(move |key| { - let weight = key.feerate().powi(ALPHA); - let exclusive_total_weight = self.total_weight - weight; - let sample_approx_weight = exclusive_total_weight * sampling_ratio; - if weight * (1.0 - sampling_ratio - eps) < eps * exclusive_total_weight { - None // break out map_while - } else { - let idx = self.index.get_index_of(key).unwrap() as u32; - // Register this index as "already sampled" - filter_ref.insert(idx); - // Flip a coin with the reversed probability - if unit.sample(rng) < (sample_approx_weight + weight) / self.total_weight { - Some(Some(idx)) - } else { - Some(None) // signals a continue but not a break - } - } - }) - .flatten() - .take(overall_amount as usize) // Bound with overall amount just in case - .collect(), - filter, - ) - } - - pub fn sample<'a, R>(&'a self, rng: &'a mut R, overall_amount: u32) -> impl Iterator + 'a - where - R: Rng + ?Sized, - { - let frontier_length = self.feerate_order.len() as u32; - if frontier_length <= overall_amount { - return Either::Left(self.index.iter().cloned()); + let length = self.feerate_order.len() as u32; + if length <= amount { + return Either::Left(self.feerate_order.iter().map(|(k, _)| k.clone())); } - - let (top, filter) = self.sample_top_bucket(rng, overall_amount); - // println!("|F|: {}, |P|: {}", filter.len(), top.len()); - let indices = index::sample(rng, frontier_length, overall_amount - top.len() as u32, overall_amount, filter).chain(top); - Either::Right(indices.into_iter().map(|i| self.index.get_index(i as usize).cloned().unwrap())) + let distr = Uniform::new(0f64, self.total_weight); + let mut cache = HashSet::new(); + Either::Right((0..amount).map(move |_| { + let query = distr.sample(rng); + let mut item = self.feerate_order.get_by_argument(query).unwrap().0; + while !cache.insert(item.tx.id()) { + let query = distr.sample(rng); + item = self.feerate_order.get_by_argument(query).unwrap().0; + } + item.clone() + })) } pub(crate) fn len(&self) -> usize { - self.index.len() + self.feerate_order.len() } } @@ -157,4 +179,32 @@ mod tests { let stage_two = stage_two_sampling(stage_one); stage_two.into_iter().map(|k| k.gas).sum::(); } + + #[test] + fn test_sweep_btree() { + use sweep_bptree::argument::count::Count; + use sweep_bptree::BPlusTreeMap; + + // use Count as Argument to create a order statistic tree + let mut map = BPlusTreeMap::::new(); + map.insert(1, 2); + map.insert(2, 3); + map.insert(3, 4); + + // get by order, time complexity is log(n) + assert_eq!(map.get_by_argument(0), Some((&1, &2))); + assert_eq!(map.get_by_argument(1), Some((&2, &3))); + + // get the offset for key + + // 0 does not exists + assert_eq!(map.rank_by_argument(&0), Err(0)); + + assert_eq!(map.rank_by_argument(&1), Ok(0)); + assert_eq!(map.rank_by_argument(&2), Ok(1)); + assert_eq!(map.rank_by_argument(&3), Ok(2)); + + // 4 does not exists + assert_eq!(map.rank_by_argument(&4), Err(3)); + } } From e34f04737daf639a34de22c6081f69425e967a0f Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 1 Aug 2024 23:12:22 +0000 Subject: [PATCH 13/74] todo --- mining/benches/bench.rs | 20 ++++++++++--------- mining/src/mempool/model/transactions_pool.rs | 3 ++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index 96c7a50e0..c4430e7fa 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -94,6 +94,8 @@ fn stage_two_sampling(container: impl IntoIterator selector.select_transactions() } +// TODO: bench frontier insertions and removals + pub fn bench_two_stage_sampling(c: &mut Criterion) { let mut rng = thread_rng(); let mut group = c.benchmark_group("mempool sampling"); @@ -114,19 +116,19 @@ pub fn bench_two_stage_sampling(c: &mut Criterion) { group.bench_function("mempool sample 2 blocks", |b| { b.iter(|| { black_box({ - let stage_one = frontier.sample(&mut rng, 600).collect_vec(); - stage_one.into_iter().map(|k| k.mass).sum::() - }) - }) - }); - group.bench_function("mempool sample 10k", |b| { - b.iter(|| { - black_box({ - let stage_one = frontier.sample(&mut rng, 10_000); + let stage_one = frontier.sample(&mut rng, 300).collect_vec(); stage_one.into_iter().map(|k| k.mass).sum::() }) }) }); + // group.bench_function("mempool sample 10k", |b| { + // b.iter(|| { + // black_box({ + // let stage_one = frontier.sample(&mut rng, 10_000); + // stage_one.into_iter().map(|k| k.mass).sum::() + // }) + // }) + // }); group.finish(); } diff --git a/mining/src/mempool/model/transactions_pool.rs b/mining/src/mempool/model/transactions_pool.rs index f16bc796f..99fae6927 100644 --- a/mining/src/mempool/model/transactions_pool.rs +++ b/mining/src/mempool/model/transactions_pool.rs @@ -172,8 +172,9 @@ impl TransactionsPool { pub(crate) fn all_ready_transactions(&self) -> Vec { // The returned transactions are leaving the mempool so they are cloned let mut rng = thread_rng(); + // TODO: adapt to one-shot sampling self.ready_transactions - .sample(&mut rng, self.config.maximum_ready_transaction_count) + .sample(&mut rng, 600) // self.config.maximum_ready_transaction_count) .map(CandidateTransaction::from_key) .collect() } From 78cfd9690ad5b15f29ab63c02b0d5d59020e7ed4 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 1 Aug 2024 23:27:40 +0000 Subject: [PATCH 14/74] keep a computed weight field --- mining/benches/bench.rs | 4 ++-- mining/src/mempool/model/feerate_key.rs | 13 ++++++++++++- mining/src/mempool/model/frontier.rs | 18 +++++++----------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index c4430e7fa..edb1fe4fd 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -105,7 +105,7 @@ pub fn bench_two_stage_sampling(c: &mut Criterion) { let fee: u64 = if i % (cap as u64 / 100000) == 0 { 1000000 } else { rng.gen_range(1..10000) }; let mass: u64 = 1650; let tx = generate_unique_tx(i); - map.insert(tx.id(), FeerateTransactionKey { fee: fee.max(mass), mass, tx }); + map.insert(tx.id(), FeerateTransactionKey::new(fee.max(mass), mass, tx)); } let len = cap; // / 10; @@ -116,7 +116,7 @@ pub fn bench_two_stage_sampling(c: &mut Criterion) { group.bench_function("mempool sample 2 blocks", |b| { b.iter(|| { black_box({ - let stage_one = frontier.sample(&mut rng, 300).collect_vec(); + let stage_one = frontier.sample(&mut rng, 400).collect_vec(); stage_one.into_iter().map(|k| k.mass).sum::() }) }) diff --git a/mining/src/mempool/model/feerate_key.rs b/mining/src/mempool/model/feerate_key.rs index 494443955..b655d86c7 100644 --- a/mining/src/mempool/model/feerate_key.rs +++ b/mining/src/mempool/model/feerate_key.rs @@ -1,3 +1,5 @@ +use crate::block_template::selector::ALPHA; + use super::tx::MempoolTransaction; use kaspa_consensus_core::tx::Transaction; use std::sync::Arc; @@ -6,6 +8,7 @@ use std::sync::Arc; pub struct FeerateTransactionKey { pub fee: u64, pub mass: u64, + weight: f64, pub tx: Arc, } @@ -18,9 +21,17 @@ impl PartialEq for FeerateTransactionKey { } impl FeerateTransactionKey { + pub fn new(fee: u64, mass: u64, tx: Arc) -> Self { + Self { fee, mass, weight: (fee as f64 / mass as f64).powi(ALPHA), tx } + } + pub fn feerate(&self) -> f64 { self.fee as f64 / self.mass as f64 } + + pub fn weight(&self) -> f64 { + self.weight + } } impl std::hash::Hash for FeerateTransactionKey { @@ -63,6 +74,6 @@ impl Ord for FeerateTransactionKey { impl From<&MempoolTransaction> for FeerateTransactionKey { fn from(tx: &MempoolTransaction) -> Self { - Self { fee: tx.mtx.calculated_fee.unwrap(), mass: tx.mtx.tx.mass(), tx: tx.mtx.tx.clone() } + Self::new(tx.mtx.calculated_fee.unwrap(), tx.mtx.tx.mass(), tx.mtx.tx.clone()) } } diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 2c8f45997..f27c266a3 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -1,15 +1,11 @@ use super::feerate_key::FeerateTransactionKey; -use crate::block_template::selector::ALPHA; use arg::FeerateWeight; -use indexmap::IndexSet; use itertools::Either; -use kaspa_utils::{rand::seq::index, vec::VecExtensions}; use rand::{distributions::Uniform, prelude::Distribution, Rng}; -use std::collections::{BTreeSet, HashSet}; +use std::collections::HashSet; use sweep_bptree::BPlusTreeMap; pub mod arg { - use crate::block_template::selector::ALPHA; use sweep_bptree::tree::{Argument, SearchArgument}; type FeerateKey = super::FeerateTransactionKey; @@ -26,7 +22,7 @@ pub mod arg { impl Argument for FeerateWeight { fn from_leaf(keys: &[FeerateKey]) -> Self { - Self(keys.iter().map(|k| k.feerate().powi(ALPHA)).sum()) + Self(keys.iter().map(|k| k.weight()).sum()) } fn from_inner(_keys: &[FeerateKey], arguments: &[Self]) -> Self { @@ -40,7 +36,7 @@ pub mod arg { fn locate_in_leaf(query: Self::Query, keys: &[FeerateKey]) -> Option { let mut sum = 0.0; for (i, k) in keys.iter().enumerate() { - let w = k.feerate().powi(ALPHA); + let w = k.weight(); sum += w; if query <= sum { return Some(i); @@ -84,7 +80,7 @@ impl Default for Frontier { impl Frontier { pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { - let (weight, mass) = (key.feerate().powi(ALPHA), key.mass); + let (weight, mass) = (key.weight(), key.mass); if self.feerate_order.insert(key, ()).is_none() { self.total_weight += weight; self.total_mass += mass; @@ -95,8 +91,8 @@ impl Frontier { } pub fn remove(&mut self, key: &FeerateTransactionKey) -> bool { - let (weight, mass) = (key.feerate().powi(ALPHA), key.mass); - if self.feerate_order.remove(&key).is_some() { + let (weight, mass) = (key.weight(), key.mass); + if self.feerate_order.remove(key).is_some() { self.total_weight -= weight; self.total_mass -= mass; true @@ -166,7 +162,7 @@ mod tests { let fee: u64 = if i % (cap as u64 / 100) == 0 { 1000000 } else { rng.gen_range(1..10000) }; let mass: u64 = 1650; let tx = generate_unique_tx(i); - map.insert(tx.id(), FeerateTransactionKey { fee: fee.max(mass), mass, tx }); + map.insert(tx.id(), FeerateTransactionKey::new(fee.max(mass), mass, tx)); } let len = cap; From cc825eff3a52fde5625cf226efdf64395bd708d4 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Fri, 2 Aug 2024 12:56:20 +0000 Subject: [PATCH 15/74] Test feerate weight queries + an implied fix (change <= to <) --- mining/src/mempool/model/frontier.rs | 38 +++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index f27c266a3..430987fe3 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -38,7 +38,7 @@ pub mod arg { for (i, k) in keys.iter().enumerate() { let w = k.weight(); sum += w; - if query <= sum { + if query < sum { return Some(i); } } @@ -122,9 +122,13 @@ impl Frontier { })) } - pub(crate) fn len(&self) -> usize { + pub fn len(&self) -> usize { self.feerate_order.len() } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } } #[cfg(test)] @@ -147,6 +151,10 @@ mod tests { Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) } + fn build_feerate_key(fee: u64, mass: u64, id: u64) -> FeerateTransactionKey { + FeerateTransactionKey::new(fee, mass, generate_unique_tx(id)) + } + fn stage_two_sampling(container: impl IntoIterator) -> Vec { let set = container.into_iter().map(CandidateTransaction::from_key).collect_vec(); let mut selector = TransactionsSelector::new(Policy::new(500_000), set); @@ -156,7 +164,7 @@ mod tests { #[test] pub fn test_two_stage_sampling() { let mut rng = thread_rng(); - let cap = 100_000; + let cap = 1000; let mut map = HashMap::with_capacity(cap); for i in 0..cap as u64 { let fee: u64 = if i % (cap as u64 / 100) == 0 { 1000000 } else { rng.gen_range(1..10000) }; @@ -176,6 +184,30 @@ mod tests { stage_two.into_iter().map(|k| k.gas).sum::(); } + #[test] + fn test_feerate_weight_queries() { + let mut btree: BPlusTreeMap = BPlusTreeMap::new(); + let mass = 2000; + let fees = [123, 113, 10_000, 1000, 2050, 2048]; + let mut weights = Vec::with_capacity(fees.len()); + for (i, fee) in fees.iter().copied().enumerate() { + let key = build_feerate_key(fee, mass, i as u64); + weights.push(key.weight()); + btree.insert(key, ()); + } + let fees_weights = fees.into_iter().zip(weights).sorted_by(|(_, w1), (_, w2)| w1.total_cmp(w2)).collect_vec(); + let eps = 0.000000001; + let mut sum = 0.0; + for (fee, weight) in fees_weights { + let samples = [sum, sum + weight / 2.0, sum + weight - eps]; + for sample in samples { + let key = btree.get_by_argument(sample).unwrap().0; + assert_eq!(fee, key.fee); + } + sum += weight; + } + } + #[test] fn test_sweep_btree() { use sweep_bptree::argument::count::Count; From 44033d8b72eee1c58dcff3c7c7b99988c0759e0a Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Fri, 2 Aug 2024 12:57:23 +0000 Subject: [PATCH 16/74] temp remove warns --- mining/benches/bench.rs | 1 + mining/src/feerate/mod.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index edb1fe4fd..54d9709cc 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -88,6 +88,7 @@ fn generate_unique_tx(i: u64) -> Arc { Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) } +#[allow(dead_code)] fn stage_two_sampling(container: impl IntoIterator) -> Vec { let set = container.into_iter().map(CandidateTransaction::from_key).collect_vec(); let mut selector = TransactionsSelector::new(Policy::new(500_000), set); diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index 569db12a4..a08c74b9e 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -13,6 +13,7 @@ pub struct FeerateEstimations { pub priority_bucket: FeerateBucket, } +#[allow(dead_code)] // TEMP (PR) pub struct FeerateEstimator { /// The total probability weight of all current mempool ready transactions, i.e., Σ_{tx in mempool}(tx.fee/tx.mass)^alpha total_weight: f64, From 5d596d3f02f5563ed5aa9ecc048919f070168ec7 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Fri, 2 Aug 2024 13:33:03 +0000 Subject: [PATCH 17/74] 1. use inner BPlusTree in order to allow access to iterator as double ended 2. reduce collisions by removing the top element from the range if it was hit --- mining/src/mempool/model/frontier.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 430987fe3..1e2ac2248 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -3,7 +3,7 @@ use arg::FeerateWeight; use itertools::Either; use rand::{distributions::Uniform, prelude::Distribution, Rng}; use std::collections::HashSet; -use sweep_bptree::BPlusTreeMap; +use sweep_bptree::{BPlusTree, NodeStoreVec}; pub mod arg { use sweep_bptree::tree::{Argument, SearchArgument}; @@ -58,12 +58,14 @@ pub mod arg { } } +pub type FrontierTree = BPlusTree>; + /// Management of the transaction pool frontier, that is, the set of transactions in /// the transaction pool which have no mempool ancestors and are essentially ready /// to enter the next block template. pub struct Frontier { /// Frontier transactions sorted by feerate order - feerate_order: BPlusTreeMap, + feerate_order: FrontierTree, /// Total sampling weight: Σ_{tx in frontier}(tx.fee/tx.mass)^alpha total_weight: f64, @@ -74,7 +76,7 @@ pub struct Frontier { impl Default for Frontier { fn default() -> Self { - Self { feerate_order: BPlusTreeMap::new(), total_weight: Default::default(), total_mass: Default::default() } + Self { feerate_order: FrontierTree::new(Default::default()), total_weight: Default::default(), total_mass: Default::default() } } } @@ -109,12 +111,21 @@ impl Frontier { if length <= amount { return Either::Left(self.feerate_order.iter().map(|(k, _)| k.clone())); } - let distr = Uniform::new(0f64, self.total_weight); + let mut total_weight = self.total_weight; + let mut distr = Uniform::new(0f64, total_weight); + let mut down_iter = self.feerate_order.iter().rev(); + let mut top = down_iter.next().expect("amount < length").0; let mut cache = HashSet::new(); Either::Right((0..amount).map(move |_| { let query = distr.sample(rng); let mut item = self.feerate_order.get_by_argument(query).unwrap().0; while !cache.insert(item.tx.id()) { + if top == item { + // Narrow the search to reduce further sampling collisions + total_weight -= top.weight(); + distr = Uniform::new(0f64, total_weight); + top = down_iter.next().expect("amount < length").0; + } let query = distr.sample(rng); item = self.feerate_order.get_by_argument(query).unwrap().0; } @@ -186,7 +197,7 @@ mod tests { #[test] fn test_feerate_weight_queries() { - let mut btree: BPlusTreeMap = BPlusTreeMap::new(); + let mut btree = FrontierTree::new(Default::default()); let mass = 2000; let fees = [123, 113, 10_000, 1000, 2050, 2048]; let mut weights = Vec::with_capacity(fees.len()); From 542f8bac27606fbdd6b03ac7801be6b68ff66943 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Fri, 2 Aug 2024 13:35:01 +0000 Subject: [PATCH 18/74] rename --- mining/src/mempool/model/frontier.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 1e2ac2248..c324a58c3 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -64,8 +64,8 @@ pub type FrontierTree = BPlusTree Self { - Self { feerate_order: FrontierTree::new(Default::default()), total_weight: Default::default(), total_mass: Default::default() } + Self { search_tree: FrontierTree::new(Default::default()), total_weight: Default::default(), total_mass: Default::default() } } } impl Frontier { pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { let (weight, mass) = (key.weight(), key.mass); - if self.feerate_order.insert(key, ()).is_none() { + if self.search_tree.insert(key, ()).is_none() { self.total_weight += weight; self.total_mass += mass; true @@ -94,7 +94,7 @@ impl Frontier { pub fn remove(&mut self, key: &FeerateTransactionKey) -> bool { let (weight, mass) = (key.weight(), key.mass); - if self.feerate_order.remove(key).is_some() { + if self.search_tree.remove(key).is_some() { self.total_weight -= weight; self.total_mass -= mass; true @@ -107,18 +107,18 @@ impl Frontier { where R: Rng + ?Sized, { - let length = self.feerate_order.len() as u32; + let length = self.search_tree.len() as u32; if length <= amount { - return Either::Left(self.feerate_order.iter().map(|(k, _)| k.clone())); + return Either::Left(self.search_tree.iter().map(|(k, _)| k.clone())); } let mut total_weight = self.total_weight; let mut distr = Uniform::new(0f64, total_weight); - let mut down_iter = self.feerate_order.iter().rev(); + let mut down_iter = self.search_tree.iter().rev(); let mut top = down_iter.next().expect("amount < length").0; let mut cache = HashSet::new(); Either::Right((0..amount).map(move |_| { let query = distr.sample(rng); - let mut item = self.feerate_order.get_by_argument(query).unwrap().0; + let mut item = self.search_tree.get_by_argument(query).unwrap().0; while !cache.insert(item.tx.id()) { if top == item { // Narrow the search to reduce further sampling collisions @@ -127,14 +127,14 @@ impl Frontier { top = down_iter.next().expect("amount < length").0; } let query = distr.sample(rng); - item = self.feerate_order.get_by_argument(query).unwrap().0; + item = self.search_tree.get_by_argument(query).unwrap().0; } item.clone() })) } pub fn len(&self) -> usize { - self.feerate_order.len() + self.search_tree.len() } pub fn is_empty(&self) -> bool { From 6b0fee8cf7a9b2f365496b6cc13cfa2d0cfeb9a4 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Fri, 2 Aug 2024 13:38:00 +0000 Subject: [PATCH 19/74] test btree rev iter --- mining/src/mempool/model/frontier.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index c324a58c3..8b5c98d25 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -219,6 +219,24 @@ mod tests { } } + #[test] + fn test_btree_rev_iter() { + let mut btree = FrontierTree::new(Default::default()); + let mass = 2000; + let fees = [123, 113, 10_000, 1000, 2050, 2048]; + let mut weights = Vec::with_capacity(fees.len()); + for (i, fee) in fees.iter().copied().enumerate() { + let key = build_feerate_key(fee, mass, i as u64); + weights.push(key.weight()); + btree.insert(key, ()); + } + let fees_weights = fees.into_iter().zip(weights).sorted_by(|(_, w1), (_, w2)| w1.total_cmp(w2)).collect_vec(); + for ((fee, weight), item) in fees_weights.into_iter().rev().zip(btree.iter().rev()) { + assert_eq!(fee, item.0.fee); + assert_eq!(weight, item.0.weight()); + } + } + #[test] fn test_sweep_btree() { use sweep_bptree::argument::count::Count; From c3a8a380b84cbf66145c496b59a043ed3a7e8665 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Fri, 2 Aug 2024 16:27:08 +0000 Subject: [PATCH 20/74] clamp the query to the bounds (logically) --- mining/src/mempool/model/frontier.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 8b5c98d25..cb75004c0 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -42,7 +42,13 @@ pub mod arg { return Some(i); } } - None + // In order to avoid sensitivity to floating number arithmetics, + // we logically "clamp" the search, returning the last leaf if the query + // value is out of bounds + match keys.len() { + 0 => None, + n => Some(n - 1), + } } fn locate_in_inner(mut query: Self::Query, _keys: &[FeerateKey], arguments: &[Self]) -> Option<(usize, Self::Query)> { @@ -53,7 +59,14 @@ pub mod arg { return Some((i, query)); } } - None + // In order to avoid sensitivity to floating number arithmetics, + // we logically "clamp" the search, returning the last subtree if the query + // value is out of bounds. Eventually this will lead to the return of the + // last leaf (see locate_in_leaf as well) + match arguments.len() { + 0 => None, + n => Some((n - 1, arguments[n - 1].0)), + } } } } @@ -118,7 +131,7 @@ impl Frontier { let mut cache = HashSet::new(); Either::Right((0..amount).map(move |_| { let query = distr.sample(rng); - let mut item = self.search_tree.get_by_argument(query).unwrap().0; + let mut item = self.search_tree.get_by_argument(query).expect("clamped").0; while !cache.insert(item.tx.id()) { if top == item { // Narrow the search to reduce further sampling collisions @@ -127,7 +140,7 @@ impl Frontier { top = down_iter.next().expect("amount < length").0; } let query = distr.sample(rng); - item = self.search_tree.get_by_argument(query).unwrap().0; + item = self.search_tree.get_by_argument(query).expect("clamped").0; } item.clone() })) From 632008fae5e18ace42e6e13d4f2b159e38e26e6a Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 07:44:14 +0000 Subject: [PATCH 21/74] use a larger tree for tests, add checks for clamped search bounds --- mining/src/mempool/model/frontier.rs | 76 ++++++++++++---------------- 1 file changed, 33 insertions(+), 43 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index cb75004c0..3b8092c51 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -212,69 +212,59 @@ mod tests { fn test_feerate_weight_queries() { let mut btree = FrontierTree::new(Default::default()); let mass = 2000; - let fees = [123, 113, 10_000, 1000, 2050, 2048]; - let mut weights = Vec::with_capacity(fees.len()); + // The btree stores N=64 keys at each node/leaf, so we make sure the tree has more than + // 64^2 keys in order to trigger at least a few intermediate tree nodes + let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); + let mut v = Vec::with_capacity(fees.len()); for (i, fee) in fees.iter().copied().enumerate() { let key = build_feerate_key(fee, mass, i as u64); - weights.push(key.weight()); + v.push(key.clone()); btree.insert(key, ()); } - let fees_weights = fees.into_iter().zip(weights).sorted_by(|(_, w1), (_, w2)| w1.total_cmp(w2)).collect_vec(); - let eps = 0.000000001; + v.sort(); + let eps: f64 = 0.001; let mut sum = 0.0; - for (fee, weight) in fees_weights { - let samples = [sum, sum + weight / 2.0, sum + weight - eps]; + for expected in v { + let weight = expected.weight(); + let eps = eps.min(weight / 3.0); + let samples = [sum + eps, sum + weight / 2.0, sum + weight - eps]; for sample in samples { let key = btree.get_by_argument(sample).unwrap().0; - assert_eq!(fee, key.fee); + assert_eq!(&expected, key); + assert!(expected.cmp(key).is_eq()); // Assert Ord equality as well } sum += weight; } + + // Test clamped search bounds + assert_eq!(btree.first(), btree.get_by_argument(f64::NEG_INFINITY)); + assert_eq!(btree.first(), btree.get_by_argument(-1.0)); + assert_eq!(btree.first(), btree.get_by_argument(-eps)); + assert_eq!(btree.first(), btree.get_by_argument(0.0)); + assert_eq!(btree.last(), btree.get_by_argument(sum)); + assert_eq!(btree.last(), btree.get_by_argument(sum + eps)); + assert_eq!(btree.last(), btree.get_by_argument(sum + 1.0)); + assert_eq!(btree.last(), btree.get_by_argument(1.0 / 0.0)); + assert_eq!(btree.last(), btree.get_by_argument(f64::INFINITY)); + assert!(btree.get_by_argument(f64::NAN).is_some()); } #[test] fn test_btree_rev_iter() { let mut btree = FrontierTree::new(Default::default()); let mass = 2000; - let fees = [123, 113, 10_000, 1000, 2050, 2048]; - let mut weights = Vec::with_capacity(fees.len()); + let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); + let mut v = Vec::with_capacity(fees.len()); for (i, fee) in fees.iter().copied().enumerate() { let key = build_feerate_key(fee, mass, i as u64); - weights.push(key.weight()); + v.push(key.clone()); btree.insert(key, ()); } - let fees_weights = fees.into_iter().zip(weights).sorted_by(|(_, w1), (_, w2)| w1.total_cmp(w2)).collect_vec(); - for ((fee, weight), item) in fees_weights.into_iter().rev().zip(btree.iter().rev()) { - assert_eq!(fee, item.0.fee); - assert_eq!(weight, item.0.weight()); - } - } - - #[test] - fn test_sweep_btree() { - use sweep_bptree::argument::count::Count; - use sweep_bptree::BPlusTreeMap; - - // use Count as Argument to create a order statistic tree - let mut map = BPlusTreeMap::::new(); - map.insert(1, 2); - map.insert(2, 3); - map.insert(3, 4); + v.sort(); - // get by order, time complexity is log(n) - assert_eq!(map.get_by_argument(0), Some((&1, &2))); - assert_eq!(map.get_by_argument(1), Some((&2, &3))); - - // get the offset for key - - // 0 does not exists - assert_eq!(map.rank_by_argument(&0), Err(0)); - - assert_eq!(map.rank_by_argument(&1), Ok(0)); - assert_eq!(map.rank_by_argument(&2), Ok(1)); - assert_eq!(map.rank_by_argument(&3), Ok(2)); - - // 4 does not exists - assert_eq!(map.rank_by_argument(&4), Err(3)); + for (expected, item) in v.into_iter().rev().zip(btree.iter().rev()) { + assert_eq!(&expected, item.0); + assert!(expected.cmp(item.0).is_eq()); // Assert Ord equality as well + } } } From 67eed1c4e5baf702c66804c90668069cfd1d88ae Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 09:05:16 +0000 Subject: [PATCH 22/74] Add benchmarks for frontier insertions and removals --- mining/benches/bench.rs | 83 +++++++++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 23 deletions(-) diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index 54d9709cc..a01040ff4 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -5,10 +5,7 @@ use kaspa_consensus_core::{ tx::{Transaction, TransactionInput, TransactionOutpoint}, }; use kaspa_hashes::{HasherBase, TransactionID}; -use kaspa_mining::{ - model::{candidate_tx::CandidateTransaction, topological_index::TopologicalIndex}, - FeerateTransactionKey, Frontier, Policy, TransactionsSelector, -}; +use kaspa_mining::{model::topological_index::TopologicalIndex, FeerateTransactionKey, Frontier}; use rand::{thread_rng, Rng}; use std::{ collections::{hash_set::Iter, HashMap, HashSet}, @@ -88,15 +85,10 @@ fn generate_unique_tx(i: u64) -> Arc { Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) } -#[allow(dead_code)] -fn stage_two_sampling(container: impl IntoIterator) -> Vec { - let set = container.into_iter().map(CandidateTransaction::from_key).collect_vec(); - let mut selector = TransactionsSelector::new(Policy::new(500_000), set); - selector.select_transactions() +fn build_feerate_key(fee: u64, mass: u64, id: u64) -> FeerateTransactionKey { + FeerateTransactionKey::new(fee, mass, generate_unique_tx(id)) } -// TODO: bench frontier insertions and removals - pub fn bench_two_stage_sampling(c: &mut Criterion) { let mut rng = thread_rng(); let mut group = c.benchmark_group("mempool sampling"); @@ -105,16 +97,16 @@ pub fn bench_two_stage_sampling(c: &mut Criterion) { for i in 0..cap as u64 { let fee: u64 = if i % (cap as u64 / 100000) == 0 { 1000000 } else { rng.gen_range(1..10000) }; let mass: u64 = 1650; - let tx = generate_unique_tx(i); - map.insert(tx.id(), FeerateTransactionKey::new(fee.max(mass), mass, tx)); + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); } - let len = cap; // / 10; + let len = cap; let mut frontier = Frontier::default(); for item in map.values().take(len).cloned() { frontier.insert(item).then_some(()).unwrap(); } - group.bench_function("mempool sample 2 blocks", |b| { + group.bench_function("mempool one-shot sample", |b| { b.iter(|| { black_box({ let stage_one = frontier.sample(&mut rng, 400).collect_vec(); @@ -122,14 +114,59 @@ pub fn bench_two_stage_sampling(c: &mut Criterion) { }) }) }); - // group.bench_function("mempool sample 10k", |b| { - // b.iter(|| { - // black_box({ - // let stage_one = frontier.sample(&mut rng, 10_000); - // stage_one.into_iter().map(|k| k.mass).sum::() - // }) - // }) - // }); + + // Benchmark frontier insertions and removals (see comparisons below) + let remove = map.values().take(map.len() / 10).cloned().collect_vec(); + group.bench_function("frontier remove/add", |b| { + b.iter(|| { + black_box({ + for r in remove.iter() { + frontier.remove(r).then_some(()).unwrap(); + } + for r in remove.iter().cloned() { + frontier.insert(r).then_some(()).unwrap(); + } + 0 + }) + }) + }); + + // Benchmark hashmap insertions and removals for comparison + let remove = map.iter().take(map.len() / 10).map(|(&k, v)| (k, v.clone())).collect_vec(); + group.bench_function("map remove/add", |b| { + b.iter(|| { + black_box({ + for r in remove.iter() { + map.remove(&r.0).unwrap(); + } + for r in remove.iter().cloned() { + map.insert(r.0, r.1.clone()); + } + 0 + }) + }) + }); + + // Benchmark std btree set insertions and removals for comparison + // Results show that frontier (sweep bptree) and std btree set are roughly the same. + // The slightly higher cost for sweep bptree should be attributed to subtree weight + // maintenance (see FeerateWeight) + #[allow(clippy::mutable_key_type)] + let mut std_btree = std::collections::BTreeSet::from_iter(map.values().cloned()); + let remove = map.iter().take(map.len() / 10).map(|(&k, v)| (k, v.clone())).collect_vec(); + group.bench_function("std btree remove/add", |b| { + b.iter(|| { + black_box({ + for (_, key) in remove.iter() { + std_btree.remove(key).then_some(()).unwrap(); + } + for (_, key) in remove.iter() { + std_btree.insert(key.clone()); + } + 0 + }) + }) + }); group.finish(); } From c66d67b5074661d23b082eb27888acca83798477 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 09:14:28 +0000 Subject: [PATCH 23/74] add item removal to the queries test --- mining/src/mempool/model/frontier.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 3b8092c51..43f559eed 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -215,13 +215,33 @@ mod tests { // The btree stores N=64 keys at each node/leaf, so we make sure the tree has more than // 64^2 keys in order to trigger at least a few intermediate tree nodes let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); - let mut v = Vec::with_capacity(fees.len()); + + #[allow(clippy::mutable_key_type)] + let mut s = HashSet::with_capacity(fees.len()); for (i, fee) in fees.iter().copied().enumerate() { let key = build_feerate_key(fee, mass, i as u64); - v.push(key.clone()); + s.insert(key.clone()); btree.insert(key, ()); } + + // Randomly remove 1/6 of the items + let remove = s.iter().take(fees.len() / 6).cloned().collect_vec(); + for r in remove { + s.remove(&r); + btree.remove(&r); + } + + // Collect to vec and sort for reference + let mut v = s.into_iter().collect_vec(); v.sort(); + + // Test reverse iteration + for (expected, item) in v.iter().rev().zip(btree.iter().rev()) { + assert_eq!(&expected, &item.0); + assert!(expected.cmp(item.0).is_eq()); // Assert Ord equality as well + } + + // Sweep through the tree and verify that weight search queries are handled correctly let eps: f64 = 0.001; let mut sum = 0.0; for expected in v { From 102043a3cf2ed20ec71a3746a32e791928c145a8 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 14:28:24 +0000 Subject: [PATCH 24/74] Important numeric stability improvement: use the visitor api to implement a prefix weight counter to be used for search narrowing --- mining/src/mempool/model/frontier.rs | 72 +++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 13 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 43f559eed..0a6d97df7 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -1,11 +1,12 @@ use super::feerate_key::FeerateTransactionKey; -use arg::FeerateWeight; +use arg::{FeerateWeight, PrefixWeightVisitor}; use itertools::Either; use rand::{distributions::Uniform, prelude::Distribution, Rng}; use std::collections::HashSet; use sweep_bptree::{BPlusTree, NodeStoreVec}; pub mod arg { + use sweep_bptree::tree::visit::{DescendVisit, DescendVisitResult}; use sweep_bptree::tree::{Argument, SearchArgument}; type FeerateKey = super::FeerateTransactionKey; @@ -69,6 +70,53 @@ pub mod arg { } } } + + pub struct PrefixWeightVisitor<'a> { + key: &'a FeerateKey, + accumulated_weight: f64, + } + + impl<'a> PrefixWeightVisitor<'a> { + pub fn new(key: &'a FeerateKey) -> Self { + Self { key, accumulated_weight: Default::default() } + } + + fn search_in_keys(&self, keys: &[FeerateKey]) -> usize { + match keys.binary_search(self.key) { + Err(idx) => { + // The idx is the place where a matching element could be inserted while maintaining + // sorted order, go to left child + idx + } + Ok(idx) => { + // Exact match, go to right child. + idx + 1 + } + } + } + } + + impl<'a> DescendVisit for PrefixWeightVisitor<'a> { + type Result = f64; + + fn visit_inner(&mut self, keys: &[FeerateKey], arguments: &[FeerateWeight]) -> DescendVisitResult { + let idx = self.search_in_keys(keys); + // trace!("[visit_inner] {}, {}, {}", keys.len(), arguments.len(), idx); + for argument in arguments.iter().take(idx) { + self.accumulated_weight += argument.weight(); + } + DescendVisitResult::GoDown(idx) + } + + fn visit_leaf(&mut self, keys: &[FeerateKey], _values: &[()]) -> Option { + let idx = self.search_in_keys(keys); + // trace!("[visit_leaf] {}, {}", keys.len(), idx); + for key in keys.iter().take(idx) { + self.accumulated_weight += key.weight(); + } + Some(self.accumulated_weight) + } + } } pub type FrontierTree = BPlusTree>; @@ -80,24 +128,24 @@ pub struct Frontier { /// Frontier transactions sorted by feerate order and searchable for weight sampling search_tree: FrontierTree, - /// Total sampling weight: Σ_{tx in frontier}(tx.fee/tx.mass)^alpha - total_weight: f64, - /// Total masses: Σ_{tx in frontier} tx.mass total_mass: u64, } impl Default for Frontier { fn default() -> Self { - Self { search_tree: FrontierTree::new(Default::default()), total_weight: Default::default(), total_mass: Default::default() } + Self { search_tree: FrontierTree::new(Default::default()), total_mass: Default::default() } } } impl Frontier { + pub fn total_weight(&self) -> f64 { + self.search_tree.root_argument().weight() + } + pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { - let (weight, mass) = (key.weight(), key.mass); + let mass = key.mass; if self.search_tree.insert(key, ()).is_none() { - self.total_weight += weight; self.total_mass += mass; true } else { @@ -106,9 +154,8 @@ impl Frontier { } pub fn remove(&mut self, key: &FeerateTransactionKey) -> bool { - let (weight, mass) = (key.weight(), key.mass); + let mass = key.mass; if self.search_tree.remove(key).is_some() { - self.total_weight -= weight; self.total_mass -= mass; true } else { @@ -124,8 +171,7 @@ impl Frontier { if length <= amount { return Either::Left(self.search_tree.iter().map(|(k, _)| k.clone())); } - let mut total_weight = self.total_weight; - let mut distr = Uniform::new(0f64, total_weight); + let mut distr = Uniform::new(0f64, self.total_weight()); let mut down_iter = self.search_tree.iter().rev(); let mut top = down_iter.next().expect("amount < length").0; let mut cache = HashSet::new(); @@ -135,9 +181,9 @@ impl Frontier { while !cache.insert(item.tx.id()) { if top == item { // Narrow the search to reduce further sampling collisions - total_weight -= top.weight(); - distr = Uniform::new(0f64, total_weight); top = down_iter.next().expect("amount < length").0; + let remaining_weight = self.search_tree.descend_visit(PrefixWeightVisitor::new(top)).unwrap(); + distr = Uniform::new(0f64, remaining_weight); } let query = distr.sample(rng); item = self.search_tree.get_by_argument(query).expect("clamped").0; From cc0877d39d864dea48d671d49eac7f76963162e9 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 14:36:33 +0000 Subject: [PATCH 25/74] test highly irregular sampling --- mining/src/mempool/model/frontier.rs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 0a6d97df7..33a933829 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -204,7 +204,6 @@ impl Frontier { #[cfg(test)] mod tests { use super::*; - use crate::{model::candidate_tx::CandidateTransaction, Policy, TransactionsSelector}; use itertools::Itertools; use kaspa_consensus_core::{ subnets::SUBNETWORK_ID_NATIVE, @@ -225,22 +224,20 @@ mod tests { FeerateTransactionKey::new(fee, mass, generate_unique_tx(id)) } - fn stage_two_sampling(container: impl IntoIterator) -> Vec { - let set = container.into_iter().map(CandidateTransaction::from_key).collect_vec(); - let mut selector = TransactionsSelector::new(Policy::new(500_000), set); - selector.select_transactions() - } - #[test] - pub fn test_two_stage_sampling() { + pub fn test_highly_irregular_sampling() { let mut rng = thread_rng(); let cap = 1000; let mut map = HashMap::with_capacity(cap); for i in 0..cap as u64 { - let fee: u64 = if i % (cap as u64 / 100) == 0 { 1000000 } else { rng.gen_range(1..10000) }; + let mut fee: u64 = if i % (cap as u64 / 100) == 0 { 1000000 } else { rng.gen_range(1..10000) }; + if i == 0 { + // Add an extremely large fee in order to create extremely high variance + fee = 100_000_000 * 1_000_000; // 1M KAS + } let mass: u64 = 1650; - let tx = generate_unique_tx(i); - map.insert(tx.id(), FeerateTransactionKey::new(fee.max(mass), mass, tx)); + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); } let len = cap; @@ -249,9 +246,8 @@ mod tests { frontier.insert(item).then_some(()).unwrap(); } - let stage_one = frontier.sample(&mut rng, 10_000); - let stage_two = stage_two_sampling(stage_one); - stage_two.into_iter().map(|k| k.gas).sum::(); + let sample = frontier.sample(&mut rng, 100).collect_vec(); + assert_eq!(100, sample.len()); } #[test] @@ -302,6 +298,8 @@ mod tests { sum += weight; } + println!("{}, {}", sum, btree.root_argument().weight()); + // Test clamped search bounds assert_eq!(btree.first(), btree.get_by_argument(f64::NEG_INFINITY)); assert_eq!(btree.first(), btree.get_by_argument(-1.0)); From 1beb861ab159c0c17fd517592f114134ddc821ce Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 18:35:49 +0000 Subject: [PATCH 26/74] Implement initial selectors + order the code a bit --- mining/benches/bench.rs | 79 +++++- mining/src/block_template/policy.rs | 2 +- mining/src/lib.rs | 2 +- mining/src/mempool/model/frontier.rs | 262 +++++++++--------- .../model/{ => frontier}/feerate_key.rs | 4 +- .../mempool/model/frontier/feerate_weight.rs | 112 ++++++++ .../src/mempool/model/frontier/selectors.rs | 140 ++++++++++ mining/src/mempool/model/mod.rs | 1 - mining/src/mempool/model/transactions_pool.rs | 6 +- 9 files changed, 460 insertions(+), 148 deletions(-) rename mining/src/mempool/model/{ => frontier}/feerate_key.rs (96%) create mode 100644 mining/src/mempool/model/frontier/feerate_weight.rs create mode 100644 mining/src/mempool/model/frontier/selectors.rs diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index a01040ff4..911432a7f 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -5,7 +5,7 @@ use kaspa_consensus_core::{ tx::{Transaction, TransactionInput, TransactionOutpoint}, }; use kaspa_hashes::{HasherBase, TransactionID}; -use kaspa_mining::{model::topological_index::TopologicalIndex, FeerateTransactionKey, Frontier}; +use kaspa_mining::{model::topological_index::TopologicalIndex, FeerateTransactionKey, Frontier, Policy}; use rand::{thread_rng, Rng}; use std::{ collections::{hash_set::Iter, HashMap, HashSet}, @@ -89,7 +89,7 @@ fn build_feerate_key(fee: u64, mass: u64, id: u64) -> FeerateTransactionKey { FeerateTransactionKey::new(fee, mass, generate_unique_tx(id)) } -pub fn bench_two_stage_sampling(c: &mut Criterion) { +pub fn bench_mempool_sampling(c: &mut Criterion) { let mut rng = thread_rng(); let mut group = c.benchmark_group("mempool sampling"); let cap = 1_000_000; @@ -109,8 +109,8 @@ pub fn bench_two_stage_sampling(c: &mut Criterion) { group.bench_function("mempool one-shot sample", |b| { b.iter(|| { black_box({ - let stage_one = frontier.sample(&mut rng, 400).collect_vec(); - stage_one.into_iter().map(|k| k.mass).sum::() + let selected = frontier.sample_inplace(&mut rng, &Policy::new(500_000)); + selected.into_iter().map(|(_, k)| k.mass).sum::() }) }) }); @@ -170,5 +170,74 @@ pub fn bench_two_stage_sampling(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_two_stage_sampling, bench_compare_topological_index_fns); +pub fn bench_mempool_sampling_small(c: &mut Criterion) { + let mut rng = thread_rng(); + let mut group = c.benchmark_group("mempool sampling small"); + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = rng.gen_range(1..1000000); + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + for len in [100, 300, 350, 500, 1000, 2000, 5000, 10_000, 100_000, 500_000, 1_000_000] { + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + group.bench_function(format!("legacy selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_legacy(); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + group.bench_function(format!("mutable tree selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_mutable_tree(); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + group.bench_function(format!("sample inplace selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_sample_inplace(); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + if frontier.total_mass() <= 500_000 { + group.bench_function(format!("take all selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_take_all(); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + } + + group.bench_function(format!("dynamic selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector(&Policy::new(500_000)); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + } + + group.finish(); +} + +criterion_group!(benches, bench_mempool_sampling, bench_mempool_sampling_small, bench_compare_topological_index_fns); criterion_main!(benches); diff --git a/mining/src/block_template/policy.rs b/mining/src/block_template/policy.rs index ca6185142..12ee98e28 100644 --- a/mining/src/block_template/policy.rs +++ b/mining/src/block_template/policy.rs @@ -1,6 +1,6 @@ /// Policy houses the policy (configuration parameters) which is used to control /// the generation of block templates. See the documentation for -/// NewBlockTemplate for more details on each of these parameters are used. +/// NewBlockTemplate for more details on how each of these parameters are used. #[derive(Clone)] pub struct Policy { /// max_block_mass is the maximum block mass to be used when generating a block template. diff --git a/mining/src/lib.rs b/mining/src/lib.rs index a2cbe8793..6ae3e1420 100644 --- a/mining/src/lib.rs +++ b/mining/src/lib.rs @@ -17,7 +17,7 @@ pub mod monitor; // Exposed for benchmarks pub use block_template::{policy::Policy, selector::TransactionsSelector}; -pub use mempool::model::{feerate_key::FeerateTransactionKey, frontier::Frontier}; +pub use mempool::model::frontier::{feerate_key::FeerateTransactionKey, Frontier}; #[cfg(test)] pub mod testutils; diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 33a933829..d2c96a709 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -1,123 +1,17 @@ -use super::feerate_key::FeerateTransactionKey; -use arg::{FeerateWeight, PrefixWeightVisitor}; -use itertools::Either; +use crate::Policy; + +use feerate_key::FeerateTransactionKey; +use feerate_weight::{FeerateWeight, PrefixWeightVisitor}; +use indexmap::IndexMap; +use kaspa_consensus_core::{block::TemplateTransactionSelector, tx::TransactionId}; use rand::{distributions::Uniform, prelude::Distribution, Rng}; +use selectors::{SequenceSelector, TakeAllSelector, WeightTreeSelector}; use std::collections::HashSet; use sweep_bptree::{BPlusTree, NodeStoreVec}; -pub mod arg { - use sweep_bptree::tree::visit::{DescendVisit, DescendVisitResult}; - use sweep_bptree::tree::{Argument, SearchArgument}; - - type FeerateKey = super::FeerateTransactionKey; - - #[derive(Clone, Copy, Debug, Default)] - pub struct FeerateWeight(f64); - - impl FeerateWeight { - /// Returns the weight value - pub fn weight(&self) -> f64 { - self.0 - } - } - - impl Argument for FeerateWeight { - fn from_leaf(keys: &[FeerateKey]) -> Self { - Self(keys.iter().map(|k| k.weight()).sum()) - } - - fn from_inner(_keys: &[FeerateKey], arguments: &[Self]) -> Self { - Self(arguments.iter().map(|a| a.0).sum()) - } - } - - impl SearchArgument for FeerateWeight { - type Query = f64; - - fn locate_in_leaf(query: Self::Query, keys: &[FeerateKey]) -> Option { - let mut sum = 0.0; - for (i, k) in keys.iter().enumerate() { - let w = k.weight(); - sum += w; - if query < sum { - return Some(i); - } - } - // In order to avoid sensitivity to floating number arithmetics, - // we logically "clamp" the search, returning the last leaf if the query - // value is out of bounds - match keys.len() { - 0 => None, - n => Some(n - 1), - } - } - - fn locate_in_inner(mut query: Self::Query, _keys: &[FeerateKey], arguments: &[Self]) -> Option<(usize, Self::Query)> { - for (i, a) in arguments.iter().enumerate() { - if query >= a.0 { - query -= a.0; - } else { - return Some((i, query)); - } - } - // In order to avoid sensitivity to floating number arithmetics, - // we logically "clamp" the search, returning the last subtree if the query - // value is out of bounds. Eventually this will lead to the return of the - // last leaf (see locate_in_leaf as well) - match arguments.len() { - 0 => None, - n => Some((n - 1, arguments[n - 1].0)), - } - } - } - - pub struct PrefixWeightVisitor<'a> { - key: &'a FeerateKey, - accumulated_weight: f64, - } - - impl<'a> PrefixWeightVisitor<'a> { - pub fn new(key: &'a FeerateKey) -> Self { - Self { key, accumulated_weight: Default::default() } - } - - fn search_in_keys(&self, keys: &[FeerateKey]) -> usize { - match keys.binary_search(self.key) { - Err(idx) => { - // The idx is the place where a matching element could be inserted while maintaining - // sorted order, go to left child - idx - } - Ok(idx) => { - // Exact match, go to right child. - idx + 1 - } - } - } - } - - impl<'a> DescendVisit for PrefixWeightVisitor<'a> { - type Result = f64; - - fn visit_inner(&mut self, keys: &[FeerateKey], arguments: &[FeerateWeight]) -> DescendVisitResult { - let idx = self.search_in_keys(keys); - // trace!("[visit_inner] {}, {}, {}", keys.len(), arguments.len(), idx); - for argument in arguments.iter().take(idx) { - self.accumulated_weight += argument.weight(); - } - DescendVisitResult::GoDown(idx) - } - - fn visit_leaf(&mut self, keys: &[FeerateKey], _values: &[()]) -> Option { - let idx = self.search_in_keys(keys); - // trace!("[visit_leaf] {}, {}", keys.len(), idx); - for key in keys.iter().take(idx) { - self.accumulated_weight += key.weight(); - } - Some(self.accumulated_weight) - } - } -} +pub mod feerate_key; +pub mod feerate_weight; +mod selectors; pub type FrontierTree = BPlusTree>; @@ -143,6 +37,10 @@ impl Frontier { self.search_tree.root_argument().weight() } + pub fn total_mass(&self) -> u64 { + self.total_mass + } + pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { let mass = key.mass; if self.search_tree.insert(key, ()).is_none() { @@ -163,33 +61,93 @@ impl Frontier { } } - pub fn sample<'a, R>(&'a self, rng: &'a mut R, amount: u32) -> impl Iterator + 'a + pub fn sample_inplace(&self, rng: &mut R, policy: &Policy) -> IndexMap where R: Rng + ?Sized, { - let length = self.search_tree.len() as u32; - if length <= amount { - return Either::Left(self.search_tree.iter().map(|(k, _)| k.clone())); + // TEMP + if self.search_tree.is_empty() { + return Default::default(); } + let mut distr = Uniform::new(0f64, self.total_weight()); let mut down_iter = self.search_tree.iter().rev(); - let mut top = down_iter.next().expect("amount < length").0; + let mut top = down_iter.next().unwrap().0; let mut cache = HashSet::new(); - Either::Right((0..amount).map(move |_| { + let mut res = IndexMap::new(); + let mut total_mass: u64 = 0; + let mut _collisions = 0; + while cache.len() < self.search_tree.len() { let query = distr.sample(rng); - let mut item = self.search_tree.get_by_argument(query).expect("clamped").0; - while !cache.insert(item.tx.id()) { - if top == item { - // Narrow the search to reduce further sampling collisions - top = down_iter.next().expect("amount < length").0; - let remaining_weight = self.search_tree.descend_visit(PrefixWeightVisitor::new(top)).unwrap(); - distr = Uniform::new(0f64, remaining_weight); + let item = { + let mut item = self.search_tree.get_by_argument(query).expect("clamped").0; + while !cache.insert(item.tx.id()) { + _collisions += 1; + if top == item { + // Narrow the search to reduce further sampling collisions + top = down_iter.next().unwrap().0; + let remaining_weight = self.search_tree.descend_visit(PrefixWeightVisitor::new(top)).unwrap(); + distr = Uniform::new(0f64, remaining_weight); + } + let query = distr.sample(rng); + item = self.search_tree.get_by_argument(query).expect("clamped").0; } - let query = distr.sample(rng); - item = self.search_tree.get_by_argument(query).expect("clamped").0; + item + }; + if total_mass.saturating_add(item.mass) > policy.max_block_mass { + break; // TODO } - item.clone() - })) + res.insert(item.tx.id(), item.clone()); + total_mass += item.mass; + } + // println!("Collisions: {collisions}, cache: {}", cache.len()); + res + } + + pub fn build_selector(&self, policy: &Policy) -> Box { + if self.total_mass <= policy.max_block_mass { + // println!("take all"); + self.build_selector_take_all() + } else if self.total_mass > policy.max_block_mass * 4 { + // println!("sample inplace"); + let mut rng = rand::thread_rng(); + Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, policy), policy.clone())) + } else { + // println!("legacy"); + Box::new(crate::TransactionsSelector::new( + policy.clone(), + self.search_tree + .iter() + .map(|(k, _)| k.clone()) + .map(crate::model::candidate_tx::CandidateTransaction::from_key) + .collect(), + )) + } + } + + pub fn build_selector_mutable_tree(&self) -> Box { + let mut tree = FrontierTree::new(Default::default()); + for (key, ()) in self.search_tree.iter() { + tree.insert(key.clone(), ()); + } + Box::new(WeightTreeSelector::new(tree, Policy::new(500_000))) + } + + pub fn build_selector_sample_inplace(&self) -> Box { + let mut rng = rand::thread_rng(); + let policy = Policy::new(500_000); + Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, &policy), policy)) + } + + pub fn build_selector_take_all(&self) -> Box { + Box::new(TakeAllSelector::new(self.search_tree.iter().map(|(k, _)| k.tx.clone()).collect())) + } + + pub fn build_selector_legacy(&self) -> Box { + Box::new(crate::TransactionsSelector::new( + Policy::new(500_000), + self.search_tree.iter().map(|(k, _)| k.clone()).map(crate::model::candidate_tx::CandidateTransaction::from_key).collect(), + )) } pub fn len(&self) -> usize { @@ -246,8 +204,8 @@ mod tests { frontier.insert(item).then_some(()).unwrap(); } - let sample = frontier.sample(&mut rng, 100).collect_vec(); - assert_eq!(100, sample.len()); + let _sample = frontier.sample_inplace(&mut rng, &Policy::new(500_000)); + // assert_eq!(100, sample.len()); } #[test] @@ -331,4 +289,38 @@ mod tests { assert!(expected.cmp(item.0).is_eq()); // Assert Ord equality as well } } + + #[test] + pub fn test_mempool_sampling_small() { + let mut rng = thread_rng(); + let cap = 2000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = rng.gen_range(1..1000000); + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let len = cap; + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let mut selector = frontier.build_selector(&Policy::new(500_000)); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector_legacy(); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector_mutable_tree(); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector_sample_inplace(); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector_take_all(); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + } } diff --git a/mining/src/mempool/model/feerate_key.rs b/mining/src/mempool/model/frontier/feerate_key.rs similarity index 96% rename from mining/src/mempool/model/feerate_key.rs rename to mining/src/mempool/model/frontier/feerate_key.rs index b655d86c7..03572d282 100644 --- a/mining/src/mempool/model/feerate_key.rs +++ b/mining/src/mempool/model/frontier/feerate_key.rs @@ -1,6 +1,4 @@ -use crate::block_template::selector::ALPHA; - -use super::tx::MempoolTransaction; +use crate::{block_template::selector::ALPHA, mempool::model::tx::MempoolTransaction}; use kaspa_consensus_core::tx::Transaction; use std::sync::Arc; diff --git a/mining/src/mempool/model/frontier/feerate_weight.rs b/mining/src/mempool/model/frontier/feerate_weight.rs new file mode 100644 index 000000000..85ff6998f --- /dev/null +++ b/mining/src/mempool/model/frontier/feerate_weight.rs @@ -0,0 +1,112 @@ +use super::feerate_key::FeerateTransactionKey; +use sweep_bptree::tree::visit::{DescendVisit, DescendVisitResult}; +use sweep_bptree::tree::{Argument, SearchArgument}; + +type FeerateKey = FeerateTransactionKey; + +#[derive(Clone, Copy, Debug, Default)] +pub struct FeerateWeight(f64); + +impl FeerateWeight { + /// Returns the weight value + pub fn weight(&self) -> f64 { + self.0 + } +} + +impl Argument for FeerateWeight { + fn from_leaf(keys: &[FeerateKey]) -> Self { + Self(keys.iter().map(|k| k.weight()).sum()) + } + + fn from_inner(_keys: &[FeerateKey], arguments: &[Self]) -> Self { + Self(arguments.iter().map(|a| a.0).sum()) + } +} + +impl SearchArgument for FeerateWeight { + type Query = f64; + + fn locate_in_leaf(query: Self::Query, keys: &[FeerateKey]) -> Option { + let mut sum = 0.0; + for (i, k) in keys.iter().enumerate() { + let w = k.weight(); + sum += w; + if query < sum { + return Some(i); + } + } + // In order to avoid sensitivity to floating number arithmetics, + // we logically "clamp" the search, returning the last leaf if the query + // value is out of bounds + match keys.len() { + 0 => None, + n => Some(n - 1), + } + } + + fn locate_in_inner(mut query: Self::Query, _keys: &[FeerateKey], arguments: &[Self]) -> Option<(usize, Self::Query)> { + for (i, a) in arguments.iter().enumerate() { + if query >= a.0 { + query -= a.0; + } else { + return Some((i, query)); + } + } + // In order to avoid sensitivity to floating number arithmetics, + // we logically "clamp" the search, returning the last subtree if the query + // value is out of bounds. Eventually this will lead to the return of the + // last leaf (see locate_in_leaf as well) + match arguments.len() { + 0 => None, + n => Some((n - 1, arguments[n - 1].0)), + } + } +} + +pub struct PrefixWeightVisitor<'a> { + key: &'a FeerateKey, + accumulated_weight: f64, +} + +impl<'a> PrefixWeightVisitor<'a> { + pub fn new(key: &'a FeerateKey) -> Self { + Self { key, accumulated_weight: Default::default() } + } + + fn search_in_keys(&self, keys: &[FeerateKey]) -> usize { + match keys.binary_search(self.key) { + Err(idx) => { + // The idx is the place where a matching element could be inserted while maintaining + // sorted order, go to left child + idx + } + Ok(idx) => { + // Exact match, go to right child. + idx + 1 + } + } + } +} + +impl<'a> DescendVisit for PrefixWeightVisitor<'a> { + type Result = f64; + + fn visit_inner(&mut self, keys: &[FeerateKey], arguments: &[FeerateWeight]) -> DescendVisitResult { + let idx = self.search_in_keys(keys); + // trace!("[visit_inner] {}, {}, {}", keys.len(), arguments.len(), idx); + for argument in arguments.iter().take(idx) { + self.accumulated_weight += argument.weight(); + } + DescendVisitResult::GoDown(idx) + } + + fn visit_leaf(&mut self, keys: &[FeerateKey], _values: &[()]) -> Option { + let idx = self.search_in_keys(keys); + // trace!("[visit_leaf] {}, {}", keys.len(), idx); + for key in keys.iter().take(idx) { + self.accumulated_weight += key.weight(); + } + Some(self.accumulated_weight) + } +} diff --git a/mining/src/mempool/model/frontier/selectors.rs b/mining/src/mempool/model/frontier/selectors.rs new file mode 100644 index 000000000..5633151ac --- /dev/null +++ b/mining/src/mempool/model/frontier/selectors.rs @@ -0,0 +1,140 @@ +use super::FrontierTree; +use crate::{FeerateTransactionKey, Policy}; +use indexmap::IndexMap; +use kaspa_consensus_core::{ + block::TemplateTransactionSelector, + tx::{Transaction, TransactionId}, +}; +use rand::Rng; +use std::{collections::HashMap, sync::Arc}; + +pub struct WeightTreeSelector { + search_tree: FrontierTree, + mass_map: HashMap, + total_mass: u64, + overall_candidates: usize, + overall_rejections: usize, + policy: Policy, +} + +impl WeightTreeSelector { + pub fn new(search_tree: FrontierTree, policy: Policy) -> Self { + Self { + search_tree, + mass_map: Default::default(), + total_mass: Default::default(), + overall_candidates: Default::default(), + overall_rejections: Default::default(), + policy, + } + } + + pub fn total_weight(&self) -> f64 { + self.search_tree.root_argument().weight() + } +} + +impl TemplateTransactionSelector for WeightTreeSelector { + fn select_transactions(&mut self) -> Vec { + let mut rng = rand::thread_rng(); + let mut transactions = Vec::new(); + self.mass_map.clear(); + + while self.total_mass <= self.policy.max_block_mass { + if self.search_tree.is_empty() { + break; + } + let query = rng.gen_range(0.0..self.total_weight()); + let (key, _) = self.search_tree.remove_by_argument(query).unwrap(); + if self.total_mass.saturating_add(key.mass) > self.policy.max_block_mass { + break; // TODO: or break + } + self.mass_map.insert(key.tx.id(), key.mass); + self.total_mass += key.mass; + transactions.push(key.tx.as_ref().clone()) + } + + transactions + } + + fn reject_selection(&mut self, tx_id: TransactionId) { + let mass = self.mass_map.get(&tx_id).unwrap(); + self.total_mass -= mass; + self.overall_rejections += 1; + } + + fn is_successful(&self) -> bool { + const SUFFICIENT_MASS_THRESHOLD: f64 = 0.79; + const LOW_REJECTION_FRACTION: f64 = 0.2; + + // We consider the operation successful if either mass occupation is above 79% or rejection rate is below 20% + self.overall_rejections == 0 + || (self.total_mass as f64) > self.policy.max_block_mass as f64 * SUFFICIENT_MASS_THRESHOLD + || (self.overall_rejections as f64) < self.overall_candidates as f64 * LOW_REJECTION_FRACTION + } +} + +pub struct SequenceSelector { + map: IndexMap, + total_mass: u64, + // overall_candidates: usize, + // overall_rejections: usize, + policy: Policy, +} + +impl SequenceSelector { + pub fn new(map: IndexMap, policy: Policy) -> Self { + Self { + map, + total_mass: Default::default(), + // overall_candidates: Default::default(), + // overall_rejections: Default::default(), + policy, + } + } +} + +impl TemplateTransactionSelector for SequenceSelector { + fn select_transactions(&mut self) -> Vec { + let mut transactions = Vec::new(); + for (_, key) in self.map.iter() { + if self.total_mass.saturating_add(key.mass) > self.policy.max_block_mass { + break; // TODO + } + // self.mass_map.insert(key.tx.id(), key.mass); + self.total_mass += key.mass; + transactions.push(key.tx.as_ref().clone()) + } + transactions + } + + fn reject_selection(&mut self, _tx_id: TransactionId) { + todo!() + } + + fn is_successful(&self) -> bool { + todo!() + } +} + +pub struct TakeAllSelector { + txs: Vec>, +} + +impl TakeAllSelector { + pub fn new(txs: Vec>) -> Self { + Self { txs } + } +} + +impl TemplateTransactionSelector for TakeAllSelector { + fn select_transactions(&mut self) -> Vec { + self.txs.drain(..).map(|tx| tx.as_ref().clone()).collect() + } + + fn reject_selection(&mut self, _tx_id: TransactionId) {} + + fn is_successful(&self) -> bool { + true + } +} diff --git a/mining/src/mempool/model/mod.rs b/mining/src/mempool/model/mod.rs index 11659bc42..bfe622293 100644 --- a/mining/src/mempool/model/mod.rs +++ b/mining/src/mempool/model/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod accepted_transactions; -pub(crate) mod feerate_key; pub(crate) mod frontier; pub(crate) mod map; pub(crate) mod orphan_pool; diff --git a/mining/src/mempool/model/transactions_pool.rs b/mining/src/mempool/model/transactions_pool.rs index 99fae6927..00415a637 100644 --- a/mining/src/mempool/model/transactions_pool.rs +++ b/mining/src/mempool/model/transactions_pool.rs @@ -11,6 +11,7 @@ use crate::{ tx::Priority, }, model::{candidate_tx::CandidateTransaction, topological_index::TopologicalIndex}, + Policy, }; use kaspa_consensus_core::{ tx::TransactionId, @@ -174,8 +175,9 @@ impl TransactionsPool { let mut rng = thread_rng(); // TODO: adapt to one-shot sampling self.ready_transactions - .sample(&mut rng, 600) // self.config.maximum_ready_transaction_count) - .map(CandidateTransaction::from_key) + .sample_inplace(&mut rng, &Policy::new(self.config.maximum_mass_per_block)) + .into_iter() + .map(|(_, k)| CandidateTransaction::from_key(k)) .collect() } From 0d0b51304dccafa35062f307717e96276784622c Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 22:07:05 +0000 Subject: [PATCH 27/74] Enhance and use the new selectors --- mining/benches/bench.rs | 2 +- mining/src/block_template/builder.rs | 23 ++--- mining/src/manager.rs | 16 ++-- mining/src/manager_tests.rs | 19 ++-- mining/src/mempool/mod.rs | 13 +-- mining/src/mempool/model/frontier.rs | 47 ++++++---- .../src/mempool/model/frontier/selectors.rs | 88 ++++++++++++++----- mining/src/mempool/model/transactions_pool.rs | 22 ++--- 8 files changed, 139 insertions(+), 91 deletions(-) diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index 911432a7f..e0325641c 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -110,7 +110,7 @@ pub fn bench_mempool_sampling(c: &mut Criterion) { b.iter(|| { black_box({ let selected = frontier.sample_inplace(&mut rng, &Policy::new(500_000)); - selected.into_iter().map(|(_, k)| k.mass).sum::() + selected.into_values().map(|k| k.mass).sum::() }) }) }); diff --git a/mining/src/block_template/builder.rs b/mining/src/block_template/builder.rs index de3428a74..e111b2562 100644 --- a/mining/src/block_template/builder.rs +++ b/mining/src/block_template/builder.rs @@ -1,25 +1,18 @@ -use super::{errors::BuilderResult, policy::Policy}; -use crate::{block_template::selector::TransactionsSelector, model::candidate_tx::CandidateTransaction}; +use super::errors::BuilderResult; use kaspa_consensus_core::{ api::ConsensusApi, - block::{BlockTemplate, TemplateBuildMode}, + block::{BlockTemplate, TemplateBuildMode, TemplateTransactionSelector}, coinbase::MinerData, merkle::calc_hash_merkle_root, tx::COINBASE_TRANSACTION_INDEX, }; -use kaspa_core::{ - debug, - time::{unix_now, Stopwatch}, -}; +use kaspa_core::time::{unix_now, Stopwatch}; -pub(crate) struct BlockTemplateBuilder { - policy: Policy, -} +pub(crate) struct BlockTemplateBuilder {} impl BlockTemplateBuilder { - pub(crate) fn new(max_block_mass: u64) -> Self { - let policy = Policy::new(max_block_mass); - Self { policy } + pub(crate) fn new() -> Self { + Self {} } /// BuildBlockTemplate creates a block template for a miner to consume @@ -89,12 +82,10 @@ impl BlockTemplateBuilder { &self, consensus: &dyn ConsensusApi, miner_data: &MinerData, - transactions: Vec, + selector: Box, build_mode: TemplateBuildMode, ) -> BuilderResult { let _sw = Stopwatch::<20>::with_threshold("build_block_template op"); - debug!("Considering {} transactions for a new block template", transactions.len()); - let selector = Box::new(TransactionsSelector::new(self.policy.clone(), transactions)); Ok(consensus.build_block_template(miner_data.clone(), selector, build_mode)?) } diff --git a/mining/src/manager.rs b/mining/src/manager.rs index c3bf61261..c8f0798a1 100644 --- a/mining/src/manager.rs +++ b/mining/src/manager.rs @@ -12,7 +12,6 @@ use crate::{ Mempool, }, model::{ - candidate_tx::CandidateTransaction, owner_txs::{GroupedOwnerTransactions, ScriptPublicKeySet}, topological_sort::IntoIterTopologically, tx_insert::TransactionInsertion, @@ -26,7 +25,7 @@ use kaspa_consensus_core::{ args::{TransactionValidationArgs, TransactionValidationBatchArgs}, ConsensusApi, }, - block::{BlockTemplate, TemplateBuildMode}, + block::{BlockTemplate, TemplateBuildMode, TemplateTransactionSelector}, coinbase::MinerData, errors::{block::RuleError as BlockRuleError, tx::TxRuleError}, tx::{MutableTransaction, Transaction, TransactionId, TransactionOutput}, @@ -107,14 +106,14 @@ impl MiningManager { loop { attempts += 1; - let transactions = self.block_candidate_transactions(); - let block_template_builder = BlockTemplateBuilder::new(self.config.maximum_mass_per_block); + let selector = self.build_selector(); + let block_template_builder = BlockTemplateBuilder::new(); let build_mode = if attempts < self.config.maximum_build_block_template_attempts { TemplateBuildMode::Standard } else { TemplateBuildMode::Infallible }; - match block_template_builder.build_block_template(consensus, miner_data, transactions, build_mode) { + match block_template_builder.build_block_template(consensus, miner_data, selector, build_mode) { Ok(block_template) => { let block_template = cache_lock.set_immutable_cached_template(block_template); match attempts { @@ -197,8 +196,9 @@ impl MiningManager { } } - pub(crate) fn block_candidate_transactions(&self) -> Vec { - self.mempool.read().block_candidate_transactions() + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier + pub(crate) fn build_selector(&self) -> Box { + self.mempool.read().build_selector() } /// Clears the block template cache, forcing the next call to get_block_template to build a new block template. @@ -209,7 +209,7 @@ impl MiningManager { #[cfg(test)] pub(crate) fn block_template_builder(&self) -> BlockTemplateBuilder { - BlockTemplateBuilder::new(self.config.maximum_mass_per_block) + BlockTemplateBuilder::new() } /// validate_and_insert_transaction validates the given transaction, and diff --git a/mining/src/manager_tests.rs b/mining/src/manager_tests.rs index a308e5657..0a2f04faf 100644 --- a/mining/src/manager_tests.rs +++ b/mining/src/manager_tests.rs @@ -7,9 +7,10 @@ mod tests { mempool::{ config::{Config, DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE}, errors::RuleError, + model::frontier::selectors::TakeAllSelector, tx::{Orphan, Priority, RbfPolicy}, }, - model::{candidate_tx::CandidateTransaction, tx_query::TransactionQuery}, + model::tx_query::TransactionQuery, testutils::consensus_mock::ConsensusMock, MiningCounters, }; @@ -1095,7 +1096,7 @@ mod tests { // Collect all parent transactions for the next block template. // They are ready since they have no parents in the mempool. - let transactions = mining_manager.block_candidate_transactions(); + let transactions = mining_manager.build_selector().select_transactions(); assert_eq!( TX_PAIRS_COUNT, transactions.len(), @@ -1103,7 +1104,7 @@ mod tests { ); parent_txs.iter().for_each(|x| { assert!( - transactions.iter().any(|tx| tx.tx.id() == x.id()), + transactions.iter().any(|tx| tx.id() == x.id()), "the parent transaction {} should be candidate for the next block template", x.id() ); @@ -1119,8 +1120,9 @@ mod tests { consensus: &dyn ConsensusApi, address_prefix: Prefix, mining_manager: &MiningManager, - transactions: Vec, + transactions: Vec, ) { + let transactions = transactions.into_iter().map(Arc::new).collect::>(); for _ in 0..4 { // Run a few times to get more randomness compare_modified_template_to_built( @@ -1187,7 +1189,7 @@ mod tests { consensus: &dyn ConsensusApi, address_prefix: Prefix, mining_manager: &MiningManager, - transactions: Vec, + transactions: Vec>, first_op: OpType, second_op: OpType, ) { @@ -1196,7 +1198,12 @@ mod tests { // Build a fresh template for coinbase2 as a reference let builder = mining_manager.block_template_builder(); - let result = builder.build_block_template(consensus, &miner_data_2, transactions, TemplateBuildMode::Standard); + let result = builder.build_block_template( + consensus, + &miner_data_2, + Box::new(TakeAllSelector::new(transactions)), + TemplateBuildMode::Standard, + ); assert!(result.is_ok(), "build block template failed for miner data 2"); let expected_template = result.unwrap(); diff --git a/mining/src/mempool/mod.rs b/mining/src/mempool/mod.rs index 1f63a3f44..30e1dfc69 100644 --- a/mining/src/mempool/mod.rs +++ b/mining/src/mempool/mod.rs @@ -1,6 +1,5 @@ use crate::{ model::{ - candidate_tx::CandidateTransaction, owner_txs::{GroupedOwnerTransactions, ScriptPublicKeySet}, tx_query::TransactionQuery, }, @@ -12,7 +11,10 @@ use self::{ model::{accepted_transactions::AcceptedTransactions, orphan_pool::OrphanPool, pool::Pool, transactions_pool::TransactionsPool}, tx::Priority, }; -use kaspa_consensus_core::tx::{MutableTransaction, TransactionId}; +use kaspa_consensus_core::{ + block::TemplateTransactionSelector, + tx::{MutableTransaction, TransactionId}, +}; use kaspa_core::time::Stopwatch; use std::sync::Arc; @@ -112,9 +114,10 @@ impl Mempool { count } - pub(crate) fn block_candidate_transactions(&self) -> Vec { - let _sw = Stopwatch::<10>::with_threshold("block_candidate_transactions op"); - self.transaction_pool.all_ready_transactions() + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier + pub(crate) fn build_selector(&self) -> Box { + let _sw = Stopwatch::<10>::with_threshold("build_selector op"); + self.transaction_pool.build_selector() } pub(crate) fn all_transaction_ids_with_priority(&self, priority: Priority) -> Vec { diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index d2c96a709..7aa1fc4a2 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -2,16 +2,19 @@ use crate::Policy; use feerate_key::FeerateTransactionKey; use feerate_weight::{FeerateWeight, PrefixWeightVisitor}; -use indexmap::IndexMap; -use kaspa_consensus_core::{block::TemplateTransactionSelector, tx::TransactionId}; +use kaspa_consensus_core::block::TemplateTransactionSelector; +use kaspa_core::trace; use rand::{distributions::Uniform, prelude::Distribution, Rng}; -use selectors::{SequenceSelector, TakeAllSelector, WeightTreeSelector}; +use selectors::{ + SequenceSelector, SequenceSelectorPriorityIndex, SequenceSelectorPriorityMap, SequenceSelectorTransaction, TakeAllSelector, + WeightTreeSelector, +}; use std::collections::HashSet; use sweep_bptree::{BPlusTree, NodeStoreVec}; -pub mod feerate_key; -pub mod feerate_weight; -mod selectors; +pub(crate) mod feerate_key; +pub(crate) mod feerate_weight; +pub(crate) mod selectors; pub type FrontierTree = BPlusTree>; @@ -61,7 +64,7 @@ impl Frontier { } } - pub fn sample_inplace(&self, rng: &mut R, policy: &Policy) -> IndexMap + pub fn sample_inplace(&self, rng: &mut R, policy: &Policy) -> SequenceSelectorPriorityMap where R: Rng + ?Sized, { @@ -70,14 +73,23 @@ impl Frontier { return Default::default(); } + // Sample 20% more than the hard limit in order to allow the SequenceSelector to + // compensate for consensus rejections. + // Note: this is a soft limit which is why the loop below might pass it if the + // next sampled transaction happens to cross the bound + let extended_mass_limit = (policy.max_block_mass as f64 * 1.2) as u64; + let mut distr = Uniform::new(0f64, self.total_weight()); let mut down_iter = self.search_tree.iter().rev(); let mut top = down_iter.next().unwrap().0; let mut cache = HashSet::new(); - let mut res = IndexMap::new(); - let mut total_mass: u64 = 0; + let mut res = SequenceSelectorPriorityMap::new(); + let mut total_selected_mass: u64 = 0; + let mut priority_index: SequenceSelectorPriorityIndex = 0; let mut _collisions = 0; - while cache.len() < self.search_tree.len() { + + // The sampling process is converging thus the cache will hold all entries eventually, which guarantees loop exit + 'outer: while cache.len() < self.search_tree.len() && total_selected_mass <= extended_mass_limit { let query = distr.sample(rng); let item = { let mut item = self.search_tree.get_by_argument(query).expect("clamped").0; @@ -85,7 +97,10 @@ impl Frontier { _collisions += 1; if top == item { // Narrow the search to reduce further sampling collisions - top = down_iter.next().unwrap().0; + match down_iter.next() { + Some(next) => top = next.0, + None => break 'outer, + } let remaining_weight = self.search_tree.descend_visit(PrefixWeightVisitor::new(top)).unwrap(); distr = Uniform::new(0f64, remaining_weight); } @@ -94,13 +109,11 @@ impl Frontier { } item }; - if total_mass.saturating_add(item.mass) > policy.max_block_mass { - break; // TODO - } - res.insert(item.tx.id(), item.clone()); - total_mass += item.mass; + res.insert(priority_index, SequenceSelectorTransaction::new(item.tx.clone(), item.mass)); + priority_index += 1; + total_selected_mass += item.mass; // Max standard mass + Mempool capacity imply this will not overflow } - // println!("Collisions: {collisions}, cache: {}", cache.len()); + trace!("[mempool frontier sample inplace] collisions: {_collisions}, cache: {}", cache.len()); res } diff --git a/mining/src/mempool/model/frontier/selectors.rs b/mining/src/mempool/model/frontier/selectors.rs index 5633151ac..dddd5c6fb 100644 --- a/mining/src/mempool/model/frontier/selectors.rs +++ b/mining/src/mempool/model/frontier/selectors.rs @@ -1,12 +1,14 @@ use super::FrontierTree; -use crate::{FeerateTransactionKey, Policy}; -use indexmap::IndexMap; +use crate::Policy; use kaspa_consensus_core::{ block::TemplateTransactionSelector, tx::{Transaction, TransactionId}, }; use rand::Rng; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; pub struct WeightTreeSelector { search_tree: FrontierTree, @@ -74,21 +76,41 @@ impl TemplateTransactionSelector for WeightTreeSelector { } } +pub struct SequenceSelectorTransaction { + pub tx: Arc, + pub mass: u64, +} + +impl SequenceSelectorTransaction { + pub fn new(tx: Arc, mass: u64) -> Self { + Self { tx, mass } + } +} + +pub type SequenceSelectorPriorityIndex = u32; + +pub type SequenceSelectorPriorityMap = BTreeMap; + +/// A selector which selects transactions in the order they are provided. The selector assumes +/// that the transactions were already selected via weighted sampling and simply tries them one +/// after the other until the block mass limit is reached. pub struct SequenceSelector { - map: IndexMap, - total_mass: u64, - // overall_candidates: usize, - // overall_rejections: usize, + priority_map: SequenceSelectorPriorityMap, + selected: HashMap, + total_selected_mass: u64, + overall_candidates: usize, + overall_rejections: usize, policy: Policy, } impl SequenceSelector { - pub fn new(map: IndexMap, policy: Policy) -> Self { + pub fn new(priority_map: SequenceSelectorPriorityMap, policy: Policy) -> Self { Self { - map, - total_mass: Default::default(), - // overall_candidates: Default::default(), - // overall_rejections: Default::default(), + overall_candidates: priority_map.len(), + priority_map, + selected: Default::default(), + total_selected_mass: Default::default(), + overall_rejections: Default::default(), policy, } } @@ -96,27 +118,44 @@ impl SequenceSelector { impl TemplateTransactionSelector for SequenceSelector { fn select_transactions(&mut self) -> Vec { + self.selected.clear(); let mut transactions = Vec::new(); - for (_, key) in self.map.iter() { - if self.total_mass.saturating_add(key.mass) > self.policy.max_block_mass { - break; // TODO + for (&priority, tx) in self.priority_map.iter() { + if self.total_selected_mass.saturating_add(tx.mass) > self.policy.max_block_mass { + // We assume the sequence is relatively small, hence we keep on searching + // for transactions with lower mass which might fit into the remaining gap + continue; } - // self.mass_map.insert(key.tx.id(), key.mass); - self.total_mass += key.mass; - transactions.push(key.tx.as_ref().clone()) + self.total_selected_mass += tx.mass; + self.selected.insert(tx.tx.id(), (tx.mass, priority)); + transactions.push(tx.tx.as_ref().clone()) + } + for (_, priority) in self.selected.values() { + self.priority_map.remove(priority); } transactions } - fn reject_selection(&mut self, _tx_id: TransactionId) { - todo!() + fn reject_selection(&mut self, tx_id: TransactionId) { + let &(mass, _) = self.selected.get(&tx_id).expect("only previously selected txs can be rejected (and only once)"); + self.total_selected_mass -= mass; + self.overall_rejections += 1; } fn is_successful(&self) -> bool { - todo!() + const SUFFICIENT_MASS_THRESHOLD: f64 = 0.8; + const LOW_REJECTION_FRACTION: f64 = 0.2; + + // We consider the operation successful if either mass occupation is above 80% or rejection rate is below 20% + self.overall_rejections == 0 + || (self.total_selected_mass as f64) > self.policy.max_block_mass as f64 * SUFFICIENT_MASS_THRESHOLD + || (self.overall_rejections as f64) < self.overall_candidates as f64 * LOW_REJECTION_FRACTION } } +/// A selector that selects all the transactions it holds and is always considered successful. +/// If all mempool transactions have combined mass which is <= block mass limit, this selector +/// should be called and provided with all the transactions. pub struct TakeAllSelector { txs: Vec>, } @@ -129,12 +168,17 @@ impl TakeAllSelector { impl TemplateTransactionSelector for TakeAllSelector { fn select_transactions(&mut self) -> Vec { + // Drain on the first call so that subsequent calls return nothing self.txs.drain(..).map(|tx| tx.as_ref().clone()).collect() } - fn reject_selection(&mut self, _tx_id: TransactionId) {} + fn reject_selection(&mut self, _tx_id: TransactionId) { + // No need to track rejections (for reduced mass), since there's nothing else to select + } fn is_successful(&self) -> bool { + // Considered successful because we provided all mempool transactions to this + // selector, so there's point in retries true } } diff --git a/mining/src/mempool/model/transactions_pool.rs b/mining/src/mempool/model/transactions_pool.rs index 00415a637..0d487c22a 100644 --- a/mining/src/mempool/model/transactions_pool.rs +++ b/mining/src/mempool/model/transactions_pool.rs @@ -10,15 +10,14 @@ use crate::{ }, tx::Priority, }, - model::{candidate_tx::CandidateTransaction, topological_index::TopologicalIndex}, + model::topological_index::TopologicalIndex, Policy, }; use kaspa_consensus_core::{ - tx::TransactionId, - tx::{MutableTransaction, TransactionOutpoint}, + block::TemplateTransactionSelector, + tx::{MutableTransaction, TransactionId, TransactionOutpoint}, }; use kaspa_core::{time::unix_now, trace, warn}; -use rand::thread_rng; use std::{ collections::{hash_map::Keys, hash_set::Iter}, sync::Arc, @@ -167,18 +166,9 @@ impl TransactionsPool { self.ready_transactions.len() } - /// all_ready_transactions returns a representative sample of fully populated - /// mempool transactions having no parents in the mempool. - /// These transactions are ready for being inserted in a block template. - pub(crate) fn all_ready_transactions(&self) -> Vec { - // The returned transactions are leaving the mempool so they are cloned - let mut rng = thread_rng(); - // TODO: adapt to one-shot sampling - self.ready_transactions - .sample_inplace(&mut rng, &Policy::new(self.config.maximum_mass_per_block)) - .into_iter() - .map(|(_, k)| CandidateTransaction::from_key(k)) - .collect() + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier + pub(crate) fn build_selector(&self) -> Box { + self.ready_transactions.build_selector(&Policy::new(self.config.maximum_mass_per_block)) } /// Is the mempool transaction identified by `transaction_id` unchained, thus having no successor? From 398889f640205e0e838bef1ab8a07c1bc31c5332 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 22:17:51 +0000 Subject: [PATCH 28/74] rename --- mining/benches/bench.rs | 4 +-- mining/src/block_template/selector.rs | 8 +++--- mining/src/lib.rs | 2 +- mining/src/mempool/model/frontier.rs | 35 ++++++++++++--------------- 4 files changed, 22 insertions(+), 27 deletions(-) diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index e0325641c..0240ab5c9 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -188,10 +188,10 @@ pub fn bench_mempool_sampling_small(c: &mut Criterion) { frontier.insert(item).then_some(()).unwrap(); } - group.bench_function(format!("legacy selector ({})", len), |b| { + group.bench_function(format!("rebalancing selector ({})", len), |b| { b.iter(|| { black_box({ - let mut selector = frontier.build_selector_legacy(); + let mut selector = frontier.build_rebalancing_selector(); selector.select_transactions().iter().map(|k| k.gas).sum::() }) }) diff --git a/mining/src/block_template/selector.rs b/mining/src/block_template/selector.rs index 8c5046935..bdf2eb1ed 100644 --- a/mining/src/block_template/selector.rs +++ b/mining/src/block_template/selector.rs @@ -28,7 +28,7 @@ pub(crate) const ALPHA: i32 = 3; /// if REBALANCE_THRESHOLD is 0.95, there's a 1-in-20 chance of collision. const REBALANCE_THRESHOLD: f64 = 0.95; -pub struct TransactionsSelector { +pub struct RebalancingWeightedTransactionSelector { policy: Policy, /// Transaction store transactions: Vec, @@ -52,7 +52,7 @@ pub struct TransactionsSelector { gas_usage_map: HashMap, } -impl TransactionsSelector { +impl RebalancingWeightedTransactionSelector { pub fn new(policy: Policy, mut transactions: Vec) -> Self { let _sw = Stopwatch::<100>::with_threshold("TransactionsSelector::new op"); // Sort the transactions by subnetwork_id. @@ -225,7 +225,7 @@ impl TransactionsSelector { } } -impl TemplateTransactionSelector for TransactionsSelector { +impl TemplateTransactionSelector for RebalancingWeightedTransactionSelector { fn select_transactions(&mut self) -> Vec { self.select_transactions() } @@ -278,7 +278,7 @@ mod tests { // Create a vector of transactions differing by output value so they have unique ids let transactions = (0..TX_INITIAL_COUNT).map(|i| create_transaction(SOMPI_PER_KASPA * (i + 1) as u64)).collect_vec(); let policy = Policy::new(100_000); - let mut selector = TransactionsSelector::new(policy, transactions); + let mut selector = RebalancingWeightedTransactionSelector::new(policy, transactions); let (mut kept, mut rejected) = (HashSet::new(), HashSet::new()); let mut reject_count = 32; for i in 0..10 { diff --git a/mining/src/lib.rs b/mining/src/lib.rs index 6ae3e1420..d1e2cf69a 100644 --- a/mining/src/lib.rs +++ b/mining/src/lib.rs @@ -16,7 +16,7 @@ pub mod model; pub mod monitor; // Exposed for benchmarks -pub use block_template::{policy::Policy, selector::TransactionsSelector}; +pub use block_template::{policy::Policy, selector::RebalancingWeightedTransactionSelector}; pub use mempool::model::frontier::{feerate_key::FeerateTransactionKey, Frontier}; #[cfg(test)] diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 7aa1fc4a2..f4c92cf78 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -1,4 +1,4 @@ -use crate::Policy; +use crate::{model::candidate_tx::CandidateTransaction, Policy, RebalancingWeightedTransactionSelector}; use feerate_key::FeerateTransactionKey; use feerate_weight::{FeerateWeight, PrefixWeightVisitor}; @@ -16,6 +16,11 @@ pub(crate) mod feerate_key; pub(crate) mod feerate_weight; pub(crate) mod selectors; +/// If the frontier contains less than 4x the block mass limit, we consider +/// inplace sampling to be less efficient (due to collisions) and thus use +/// the rebalancing selector +const COLLISION_FACTOR: u64 = 4; + pub type FrontierTree = BPlusTree>; /// Management of the transaction pool frontier, that is, the set of transactions in @@ -68,10 +73,7 @@ impl Frontier { where R: Rng + ?Sized, { - // TEMP - if self.search_tree.is_empty() { - return Default::default(); - } + debug_assert!(!self.search_tree.is_empty(), "expected to be called only if not empty"); // Sample 20% more than the hard limit in order to allow the SequenceSelector to // compensate for consensus rejections. @@ -119,21 +121,14 @@ impl Frontier { pub fn build_selector(&self, policy: &Policy) -> Box { if self.total_mass <= policy.max_block_mass { - // println!("take all"); - self.build_selector_take_all() - } else if self.total_mass > policy.max_block_mass * 4 { - // println!("sample inplace"); + Box::new(TakeAllSelector::new(self.search_tree.iter().map(|(k, _)| k.tx.clone()).collect())) + } else if self.total_mass > policy.max_block_mass * COLLISION_FACTOR { let mut rng = rand::thread_rng(); Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, policy), policy.clone())) } else { - // println!("legacy"); - Box::new(crate::TransactionsSelector::new( + Box::new(RebalancingWeightedTransactionSelector::new( policy.clone(), - self.search_tree - .iter() - .map(|(k, _)| k.clone()) - .map(crate::model::candidate_tx::CandidateTransaction::from_key) - .collect(), + self.search_tree.iter().map(|(k, _)| k.clone()).map(CandidateTransaction::from_key).collect(), )) } } @@ -156,10 +151,10 @@ impl Frontier { Box::new(TakeAllSelector::new(self.search_tree.iter().map(|(k, _)| k.tx.clone()).collect())) } - pub fn build_selector_legacy(&self) -> Box { - Box::new(crate::TransactionsSelector::new( + pub fn build_rebalancing_selector(&self) -> Box { + Box::new(RebalancingWeightedTransactionSelector::new( Policy::new(500_000), - self.search_tree.iter().map(|(k, _)| k.clone()).map(crate::model::candidate_tx::CandidateTransaction::from_key).collect(), + self.search_tree.iter().map(|(k, _)| k.clone()).map(CandidateTransaction::from_key).collect(), )) } @@ -324,7 +319,7 @@ mod tests { let mut selector = frontier.build_selector(&Policy::new(500_000)); selector.select_transactions().iter().map(|k| k.gas).sum::(); - let mut selector = frontier.build_selector_legacy(); + let mut selector = frontier.build_rebalancing_selector(); selector.select_transactions().iter().map(|k| k.gas).sum::(); let mut selector = frontier.build_selector_mutable_tree(); From 26832862cbdb829d82cc966c34e9b64eacf18fc6 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 22:33:17 +0000 Subject: [PATCH 29/74] minor refactor --- mining/benches/bench.rs | 2 +- mining/src/mempool/model/frontier.rs | 11 +++------ .../src/mempool/model/frontier/selectors.rs | 24 +++++++++++++++---- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index 0240ab5c9..f1c25f1d7 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -110,7 +110,7 @@ pub fn bench_mempool_sampling(c: &mut Criterion) { b.iter(|| { black_box({ let selected = frontier.sample_inplace(&mut rng, &Policy::new(500_000)); - selected.into_values().map(|k| k.mass).sum::() + selected.iter().map(|k| k.mass).sum::() }) }) }); diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index f4c92cf78..d2b6c56e1 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -5,10 +5,7 @@ use feerate_weight::{FeerateWeight, PrefixWeightVisitor}; use kaspa_consensus_core::block::TemplateTransactionSelector; use kaspa_core::trace; use rand::{distributions::Uniform, prelude::Distribution, Rng}; -use selectors::{ - SequenceSelector, SequenceSelectorPriorityIndex, SequenceSelectorPriorityMap, SequenceSelectorTransaction, TakeAllSelector, - WeightTreeSelector, -}; +use selectors::{SequenceSelector, SequenceSelectorPriorityMap, TakeAllSelector, WeightTreeSelector}; use std::collections::HashSet; use sweep_bptree::{BPlusTree, NodeStoreVec}; @@ -85,9 +82,8 @@ impl Frontier { let mut down_iter = self.search_tree.iter().rev(); let mut top = down_iter.next().unwrap().0; let mut cache = HashSet::new(); - let mut res = SequenceSelectorPriorityMap::new(); + let mut res = SequenceSelectorPriorityMap::default(); let mut total_selected_mass: u64 = 0; - let mut priority_index: SequenceSelectorPriorityIndex = 0; let mut _collisions = 0; // The sampling process is converging thus the cache will hold all entries eventually, which guarantees loop exit @@ -111,8 +107,7 @@ impl Frontier { } item }; - res.insert(priority_index, SequenceSelectorTransaction::new(item.tx.clone(), item.mass)); - priority_index += 1; + res.push(item.tx.clone(), item.mass); total_selected_mass += item.mass; // Max standard mass + Mempool capacity imply this will not overflow } trace!("[mempool frontier sample inplace] collisions: {_collisions}, cache: {}", cache.len()); diff --git a/mining/src/mempool/model/frontier/selectors.rs b/mining/src/mempool/model/frontier/selectors.rs index dddd5c6fb..dc9c70ac3 100644 --- a/mining/src/mempool/model/frontier/selectors.rs +++ b/mining/src/mempool/model/frontier/selectors.rs @@ -87,9 +87,23 @@ impl SequenceSelectorTransaction { } } -pub type SequenceSelectorPriorityIndex = u32; +type SequenceSelectorPriorityIndex = u32; -pub type SequenceSelectorPriorityMap = BTreeMap; +#[derive(Default)] +pub struct SequenceSelectorPriorityMap { + inner: BTreeMap, +} + +impl SequenceSelectorPriorityMap { + pub fn push(&mut self, tx: Arc, mass: u64) { + let idx = self.inner.len() as SequenceSelectorPriorityIndex; + self.inner.insert(idx, SequenceSelectorTransaction::new(tx, mass)); + } + + pub fn iter(&self) -> impl Iterator { + self.inner.values() + } +} /// A selector which selects transactions in the order they are provided. The selector assumes /// that the transactions were already selected via weighted sampling and simply tries them one @@ -106,7 +120,7 @@ pub struct SequenceSelector { impl SequenceSelector { pub fn new(priority_map: SequenceSelectorPriorityMap, policy: Policy) -> Self { Self { - overall_candidates: priority_map.len(), + overall_candidates: priority_map.inner.len(), priority_map, selected: Default::default(), total_selected_mass: Default::default(), @@ -120,7 +134,7 @@ impl TemplateTransactionSelector for SequenceSelector { fn select_transactions(&mut self) -> Vec { self.selected.clear(); let mut transactions = Vec::new(); - for (&priority, tx) in self.priority_map.iter() { + for (&priority, tx) in self.priority_map.inner.iter() { if self.total_selected_mass.saturating_add(tx.mass) > self.policy.max_block_mass { // We assume the sequence is relatively small, hence we keep on searching // for transactions with lower mass which might fit into the remaining gap @@ -131,7 +145,7 @@ impl TemplateTransactionSelector for SequenceSelector { transactions.push(tx.tx.as_ref().clone()) } for (_, priority) in self.selected.values() { - self.priority_map.remove(priority); + self.priority_map.inner.remove(priority); } transactions } From 524e1a385ca287c89643f36f5dbc73dd4d31d342 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 23:37:47 +0000 Subject: [PATCH 30/74] minor optimizations etc --- mining/src/mempool/model/frontier.rs | 18 ++++--- .../src/mempool/model/frontier/selectors.rs | 52 ++++++++++++++----- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index d2b6c56e1..402addabc 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -5,7 +5,7 @@ use feerate_weight::{FeerateWeight, PrefixWeightVisitor}; use kaspa_consensus_core::block::TemplateTransactionSelector; use kaspa_core::trace; use rand::{distributions::Uniform, prelude::Distribution, Rng}; -use selectors::{SequenceSelector, SequenceSelectorPriorityMap, TakeAllSelector, WeightTreeSelector}; +use selectors::{SequenceSelector, SequenceSelectorInput, TakeAllSelector, WeightTreeSelector}; use std::collections::HashSet; use sweep_bptree::{BPlusTree, NodeStoreVec}; @@ -18,6 +18,10 @@ pub(crate) mod selectors; /// the rebalancing selector const COLLISION_FACTOR: u64 = 4; +/// Multiplication factor for in-place sampling. We sample 20% more than the +/// hard limit in order to allow the SequenceSelector to compensate for consensus rejections. +const MASS_LIMIT_FACTOR: f64 = 1.2; + pub type FrontierTree = BPlusTree>; /// Management of the transaction pool frontier, that is, the set of transactions in @@ -66,7 +70,7 @@ impl Frontier { } } - pub fn sample_inplace(&self, rng: &mut R, policy: &Policy) -> SequenceSelectorPriorityMap + pub fn sample_inplace(&self, rng: &mut R, policy: &Policy) -> SequenceSelectorInput where R: Rng + ?Sized, { @@ -76,13 +80,13 @@ impl Frontier { // compensate for consensus rejections. // Note: this is a soft limit which is why the loop below might pass it if the // next sampled transaction happens to cross the bound - let extended_mass_limit = (policy.max_block_mass as f64 * 1.2) as u64; + let extended_mass_limit = (policy.max_block_mass as f64 * MASS_LIMIT_FACTOR) as u64; let mut distr = Uniform::new(0f64, self.total_weight()); let mut down_iter = self.search_tree.iter().rev(); let mut top = down_iter.next().unwrap().0; let mut cache = HashSet::new(); - let mut res = SequenceSelectorPriorityMap::default(); + let mut sequence = SequenceSelectorInput::default(); let mut total_selected_mass: u64 = 0; let mut _collisions = 0; @@ -107,11 +111,11 @@ impl Frontier { } item }; - res.push(item.tx.clone(), item.mass); - total_selected_mass += item.mass; // Max standard mass + Mempool capacity imply this will not overflow + sequence.push(item.tx.clone(), item.mass); + total_selected_mass += item.mass; // Max standard mass + Mempool capacity bound imply this will not overflow } trace!("[mempool frontier sample inplace] collisions: {_collisions}, cache: {}", cache.len()); - res + sequence } pub fn build_selector(&self, policy: &Policy) -> Box { diff --git a/mining/src/mempool/model/frontier/selectors.rs b/mining/src/mempool/model/frontier/selectors.rs index dc9c70ac3..11db67ed7 100644 --- a/mining/src/mempool/model/frontier/selectors.rs +++ b/mining/src/mempool/model/frontier/selectors.rs @@ -89,12 +89,15 @@ impl SequenceSelectorTransaction { type SequenceSelectorPriorityIndex = u32; +/// The input sequence for the [`SequenceSelector`] transaction selector #[derive(Default)] -pub struct SequenceSelectorPriorityMap { +pub struct SequenceSelectorInput { + /// We use the btree map ordered by insertion order in order to follow + /// the initial sequence order while allowing for efficient removal of previous selections inner: BTreeMap, } -impl SequenceSelectorPriorityMap { +impl SequenceSelectorInput { pub fn push(&mut self, tx: Arc, mass: u64) { let idx = self.inner.len() as SequenceSelectorPriorityIndex; self.inner.insert(idx, SequenceSelectorTransaction::new(tx, mass)); @@ -105,12 +108,20 @@ impl SequenceSelectorPriorityMap { } } +/// Helper struct for storing data related to previous selections +struct SequenceSelectorSelection { + tx_id: TransactionId, + mass: u64, + priority_index: SequenceSelectorPriorityIndex, +} + /// A selector which selects transactions in the order they are provided. The selector assumes /// that the transactions were already selected via weighted sampling and simply tries them one /// after the other until the block mass limit is reached. pub struct SequenceSelector { - priority_map: SequenceSelectorPriorityMap, - selected: HashMap, + priority_map: SequenceSelectorInput, + selected_vec: Vec, + selected_map: Option>, total_selected_mass: u64, overall_candidates: usize, overall_rejections: usize, @@ -118,40 +129,53 @@ pub struct SequenceSelector { } impl SequenceSelector { - pub fn new(priority_map: SequenceSelectorPriorityMap, policy: Policy) -> Self { + pub fn new(priority_map: SequenceSelectorInput, policy: Policy) -> Self { Self { overall_candidates: priority_map.inner.len(), + selected_vec: Vec::with_capacity(priority_map.inner.len()), priority_map, - selected: Default::default(), + selected_map: Default::default(), total_selected_mass: Default::default(), overall_rejections: Default::default(), policy, } } + + #[inline] + fn reset_selection(&mut self) { + self.selected_vec.clear(); + self.selected_map = None; + } } impl TemplateTransactionSelector for SequenceSelector { fn select_transactions(&mut self) -> Vec { - self.selected.clear(); - let mut transactions = Vec::new(); - for (&priority, tx) in self.priority_map.inner.iter() { + // Remove selections from the previous round if any + for selection in self.selected_vec.drain(..) { + self.priority_map.inner.remove(&selection.priority_index); + } + // Reset selection data structures + self.reset_selection(); + let mut transactions = Vec::with_capacity(self.priority_map.inner.len()); + + // Iterate the input sequence in order + for (&priority_index, tx) in self.priority_map.inner.iter() { if self.total_selected_mass.saturating_add(tx.mass) > self.policy.max_block_mass { // We assume the sequence is relatively small, hence we keep on searching // for transactions with lower mass which might fit into the remaining gap continue; } self.total_selected_mass += tx.mass; - self.selected.insert(tx.tx.id(), (tx.mass, priority)); + self.selected_vec.push(SequenceSelectorSelection { tx_id: tx.tx.id(), mass: tx.mass, priority_index }); transactions.push(tx.tx.as_ref().clone()) } - for (_, priority) in self.selected.values() { - self.priority_map.inner.remove(priority); - } transactions } fn reject_selection(&mut self, tx_id: TransactionId) { - let &(mass, _) = self.selected.get(&tx_id).expect("only previously selected txs can be rejected (and only once)"); + // Lazy-create the map only when there are actual rejections + let selected_map = self.selected_map.get_or_insert_with(|| self.selected_vec.iter().map(|tx| (tx.tx_id, tx.mass)).collect()); + let mass = selected_map.remove(&tx_id).expect("only previously selected txs can be rejected (and only once)"); self.total_selected_mass -= mass; self.overall_rejections += 1; } From ebda7d28095912fb1345ef5697cdfbdd6fc65262 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 23:49:38 +0000 Subject: [PATCH 31/74] increase default devnet prealloc amount to 100 tkas --- kaspad/src/args.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kaspad/src/args.rs b/kaspad/src/args.rs index 2774269d3..56dd7c1de 100644 --- a/kaspad/src/args.rs +++ b/kaspad/src/args.rs @@ -134,7 +134,7 @@ impl Default for Args { #[cfg(feature = "devnet-prealloc")] prealloc_address: None, #[cfg(feature = "devnet-prealloc")] - prealloc_amount: 1_000_000, + prealloc_amount: 10_000_000_000, disable_upnp: false, disable_dns_seeding: false, From 05be8bb6d94e94a4c78ba06f98b04908efb9fff9 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 23:53:58 +0000 Subject: [PATCH 32/74] cleanup --- mining/benches/bench.rs | 9 --- mining/src/mempool/model/frontier.rs | 16 ++--- .../src/mempool/model/frontier/selectors.rs | 68 ------------------- 3 files changed, 4 insertions(+), 89 deletions(-) diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index f1c25f1d7..4a605b8b7 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -197,15 +197,6 @@ pub fn bench_mempool_sampling_small(c: &mut Criterion) { }) }); - group.bench_function(format!("mutable tree selector ({})", len), |b| { - b.iter(|| { - black_box({ - let mut selector = frontier.build_selector_mutable_tree(); - selector.select_transactions().iter().map(|k| k.gas).sum::() - }) - }) - }); - group.bench_function(format!("sample inplace selector ({})", len), |b| { b.iter(|| { black_box({ diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 402addabc..084949580 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -5,7 +5,7 @@ use feerate_weight::{FeerateWeight, PrefixWeightVisitor}; use kaspa_consensus_core::block::TemplateTransactionSelector; use kaspa_core::trace; use rand::{distributions::Uniform, prelude::Distribution, Rng}; -use selectors::{SequenceSelector, SequenceSelectorInput, TakeAllSelector, WeightTreeSelector}; +use selectors::{SequenceSelector, SequenceSelectorInput, TakeAllSelector}; use std::collections::HashSet; use sweep_bptree::{BPlusTree, NodeStoreVec}; @@ -132,14 +132,6 @@ impl Frontier { } } - pub fn build_selector_mutable_tree(&self) -> Box { - let mut tree = FrontierTree::new(Default::default()); - for (key, ()) in self.search_tree.iter() { - tree.insert(key.clone(), ()); - } - Box::new(WeightTreeSelector::new(tree, Policy::new(500_000))) - } - pub fn build_selector_sample_inplace(&self) -> Box { let mut rng = rand::thread_rng(); let policy = Policy::new(500_000); @@ -321,13 +313,13 @@ mod tests { let mut selector = frontier.build_rebalancing_selector(); selector.select_transactions().iter().map(|k| k.gas).sum::(); - let mut selector = frontier.build_selector_mutable_tree(); - selector.select_transactions().iter().map(|k| k.gas).sum::(); - let mut selector = frontier.build_selector_sample_inplace(); selector.select_transactions().iter().map(|k| k.gas).sum::(); let mut selector = frontier.build_selector_take_all(); selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector(&Policy::new(500_000)); + selector.select_transactions().iter().map(|k| k.gas).sum::(); } } diff --git a/mining/src/mempool/model/frontier/selectors.rs b/mining/src/mempool/model/frontier/selectors.rs index 11db67ed7..a7d89805b 100644 --- a/mining/src/mempool/model/frontier/selectors.rs +++ b/mining/src/mempool/model/frontier/selectors.rs @@ -1,81 +1,13 @@ -use super::FrontierTree; use crate::Policy; use kaspa_consensus_core::{ block::TemplateTransactionSelector, tx::{Transaction, TransactionId}, }; -use rand::Rng; use std::{ collections::{BTreeMap, HashMap}, sync::Arc, }; -pub struct WeightTreeSelector { - search_tree: FrontierTree, - mass_map: HashMap, - total_mass: u64, - overall_candidates: usize, - overall_rejections: usize, - policy: Policy, -} - -impl WeightTreeSelector { - pub fn new(search_tree: FrontierTree, policy: Policy) -> Self { - Self { - search_tree, - mass_map: Default::default(), - total_mass: Default::default(), - overall_candidates: Default::default(), - overall_rejections: Default::default(), - policy, - } - } - - pub fn total_weight(&self) -> f64 { - self.search_tree.root_argument().weight() - } -} - -impl TemplateTransactionSelector for WeightTreeSelector { - fn select_transactions(&mut self) -> Vec { - let mut rng = rand::thread_rng(); - let mut transactions = Vec::new(); - self.mass_map.clear(); - - while self.total_mass <= self.policy.max_block_mass { - if self.search_tree.is_empty() { - break; - } - let query = rng.gen_range(0.0..self.total_weight()); - let (key, _) = self.search_tree.remove_by_argument(query).unwrap(); - if self.total_mass.saturating_add(key.mass) > self.policy.max_block_mass { - break; // TODO: or break - } - self.mass_map.insert(key.tx.id(), key.mass); - self.total_mass += key.mass; - transactions.push(key.tx.as_ref().clone()) - } - - transactions - } - - fn reject_selection(&mut self, tx_id: TransactionId) { - let mass = self.mass_map.get(&tx_id).unwrap(); - self.total_mass -= mass; - self.overall_rejections += 1; - } - - fn is_successful(&self) -> bool { - const SUFFICIENT_MASS_THRESHOLD: f64 = 0.79; - const LOW_REJECTION_FRACTION: f64 = 0.2; - - // We consider the operation successful if either mass occupation is above 79% or rejection rate is below 20% - self.overall_rejections == 0 - || (self.total_mass as f64) > self.policy.max_block_mass as f64 * SUFFICIENT_MASS_THRESHOLD - || (self.overall_rejections as f64) < self.overall_candidates as f64 * LOW_REJECTION_FRACTION - } -} - pub struct SequenceSelectorTransaction { pub tx: Arc, pub mass: u64, From 7c6325e0a3396b0410c5350b5b1eac3b608004ed Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 4 Aug 2024 23:55:18 +0000 Subject: [PATCH 33/74] cleanup --- utils/src/lib.rs | 1 - utils/src/rand/mod.rs | 1 - utils/src/rand/seq.rs | 76 ------------------------------------------- 3 files changed, 78 deletions(-) delete mode 100644 utils/src/rand/mod.rs delete mode 100644 utils/src/rand/seq.rs diff --git a/utils/src/lib.rs b/utils/src/lib.rs index 7a54e4078..bd3143719 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -8,7 +8,6 @@ pub mod iter; pub mod mem_size; pub mod networking; pub mod option; -pub mod rand; pub mod refs; pub mod as_slice; diff --git a/utils/src/rand/mod.rs b/utils/src/rand/mod.rs deleted file mode 100644 index ed6dcf110..000000000 --- a/utils/src/rand/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod seq; diff --git a/utils/src/rand/seq.rs b/utils/src/rand/seq.rs deleted file mode 100644 index 708c939d5..000000000 --- a/utils/src/rand/seq.rs +++ /dev/null @@ -1,76 +0,0 @@ -pub mod index { - use rand::{distributions::Uniform, prelude::Distribution, Rng}; - use std::collections::HashSet; - - /// Adaptation of [`rand::seq::index::sample`] for the case where there exists an a priory filter - /// of indices which should not be selected. - /// - /// Assumes `|filter| << length`. - /// - /// The argument `capacity` can be used to ensure a larger allocation within the returned vector. - pub fn sample(rng: &mut R, length: u32, amount: u32, capacity: u32, filter: HashSet) -> Vec - where - R: Rng + ?Sized, - { - const C: [f32; 2] = [270.0, 330.0 / 9.0]; - let j = if length < 500_000 { 0 } else { 1 }; - if (length as f32) < C[j] * (amount as f32) { - sample_inplace(rng, length, amount, capacity, filter) - } else { - sample_rejection(rng, length, amount, capacity, filter) - } - } - - /// Adaptation of [`rand::seq::index::sample_inplace`] for the case where there exists an a priory filter - /// of indices which should not be selected. - /// - /// Assumes `|filter| << length`. - /// - /// The argument `capacity` can be used to ensure a larger allocation within the returned vector. - fn sample_inplace(rng: &mut R, length: u32, amount: u32, capacity: u32, filter: HashSet) -> Vec - where - R: Rng + ?Sized, - { - debug_assert!(amount <= length); - debug_assert!(filter.len() <= amount as usize); - let mut indices: Vec = Vec::with_capacity(length.max(capacity) as usize); - indices.extend(0..length); - for i in 0..amount { - let mut j: u32 = rng.gen_range(i..length); - // Assumes |filter| << length - while filter.contains(&j) { - j = rng.gen_range(i..length); - } - indices.swap(i as usize, j as usize); - } - indices.truncate(amount as usize); - debug_assert_eq!(indices.len(), amount as usize); - indices - } - - /// Adaptation of [`rand::seq::index::sample_rejection`] for the case where there exists an a priory filter - /// of indices which should not be selected. - /// - /// Assumes `|filter| << length`. - /// - /// The argument `capacity` can be used to ensure a larger allocation within the returned vector. - fn sample_rejection(rng: &mut R, length: u32, amount: u32, capacity: u32, mut filter: HashSet) -> Vec - where - R: Rng + ?Sized, - { - debug_assert!(amount < length); - debug_assert!(filter.len() <= amount as usize); - let distr = Uniform::new(0, length); - let mut indices = Vec::with_capacity(amount.max(capacity) as usize); - for _ in indices.len()..amount as usize { - let mut pos = distr.sample(rng); - while !filter.insert(pos) { - pos = distr.sample(rng); - } - indices.push(pos); - } - - assert_eq!(indices.len(), amount as usize); - indices - } -} From fc08f26ef4d10dd919738b7519c2be7571b1ef42 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 00:38:26 +0000 Subject: [PATCH 34/74] initial build_feerate_estimator --- mining/src/feerate/mod.rs | 27 ++++++++----- mining/src/mempool/model/frontier.rs | 58 +++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 19 deletions(-) diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index a08c74b9e..e2524f203 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -13,37 +13,44 @@ pub struct FeerateEstimations { pub priority_bucket: FeerateBucket, } -#[allow(dead_code)] // TEMP (PR) +pub struct FeerateEstimatorArgs { + pub network_mass_per_second: u64, +} + pub struct FeerateEstimator { /// The total probability weight of all current mempool ready transactions, i.e., Σ_{tx in mempool}(tx.fee/tx.mass)^alpha total_weight: f64, - /// The amortized time between transactions given the current transaction masses present in the mempool, i.e., - /// the inverse of the transaction inclusion rate. For instance, if the average transaction mass is 2500 grams, + /// The amortized time between transactions given the current transaction masses present in the mempool. Or in + /// other words, the inverse of the transaction inclusion rate. For instance, if the average transaction mass is 2500 grams, /// the block mass limit is 500,000 and the network has 10 BPS, then this number would be 1/2000 seconds. - inclusion_time: f64, + inclusion_interval: f64, } impl FeerateEstimator { - fn feerate_to_time(&self, feerate: f64) -> f64 { - let (c1, c2) = (self.inclusion_time, self.total_weight); + pub fn new(total_weight: f64, inclusion_interval: f64) -> Self { + Self { total_weight, inclusion_interval } + } + + pub(crate) fn feerate_to_time(&self, feerate: f64) -> f64 { + let (c1, c2) = (self.inclusion_interval, self.total_weight); c1 * c2 / feerate.powi(ALPHA) + c1 } fn time_to_feerate(&self, time: f64) -> f64 { - let (c1, c2) = (self.inclusion_time, self.total_weight); + let (c1, c2) = (self.inclusion_interval, self.total_weight); ((c1 * c2 / time) / (1f64 - c1 / time)).powf(1f64 / ALPHA as f64) } /// The antiderivative function of [`feerate_to_time`] excluding the constant shift `+ c1` fn feerate_to_time_antiderivative(&self, feerate: f64) -> f64 { - let (c1, c2) = (self.inclusion_time, self.total_weight); + let (c1, c2) = (self.inclusion_interval, self.total_weight); c1 * c2 / (-2f64 * feerate.powi(ALPHA - 1)) } fn quantile(&self, lower: f64, upper: f64, frac: f64) -> f64 { assert!((0f64..=1f64).contains(&frac)); - let (c1, c2) = (self.inclusion_time, self.total_weight); + let (c1, c2) = (self.inclusion_interval, self.total_weight); let z1 = self.feerate_to_time_antiderivative(lower); let z2 = self.feerate_to_time_antiderivative(upper); let z = frac * z2 + (1f64 - frac) * z1; @@ -68,7 +75,7 @@ mod tests { #[test] fn test_feerate_estimations() { - let estimator = FeerateEstimator { total_weight: 1002283.659, inclusion_time: 0.004f64 }; + let estimator = FeerateEstimator { total_weight: 1002283.659, inclusion_interval: 0.004f64 }; let estimations = estimator.calc_estimations(); assert!(estimations.low_bucket.feerate <= estimations.normal_bucket.feerate); assert!(estimations.normal_bucket.feerate <= estimations.priority_bucket.feerate); diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 084949580..319ee8a8f 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -1,4 +1,8 @@ -use crate::{model::candidate_tx::CandidateTransaction, Policy, RebalancingWeightedTransactionSelector}; +use crate::{ + feerate::{FeerateEstimator, FeerateEstimatorArgs}, + model::candidate_tx::CandidateTransaction, + Policy, RebalancingWeightedTransactionSelector, +}; use feerate_key::FeerateTransactionKey; use feerate_weight::{FeerateWeight, PrefixWeightVisitor}; @@ -50,6 +54,14 @@ impl Frontier { self.total_mass } + pub fn len(&self) -> usize { + self.search_tree.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { let mass = key.mass; if self.search_tree.insert(key, ()).is_none() { @@ -83,8 +95,8 @@ impl Frontier { let extended_mass_limit = (policy.max_block_mass as f64 * MASS_LIMIT_FACTOR) as u64; let mut distr = Uniform::new(0f64, self.total_weight()); - let mut down_iter = self.search_tree.iter().rev(); - let mut top = down_iter.next().unwrap().0; + let mut down_iter = self.search_tree.iter().rev().map(|(key, ())| key); + let mut top = down_iter.next().unwrap(); let mut cache = HashSet::new(); let mut sequence = SequenceSelectorInput::default(); let mut total_selected_mass: u64 = 0; @@ -100,7 +112,7 @@ impl Frontier { if top == item { // Narrow the search to reduce further sampling collisions match down_iter.next() { - Some(next) => top = next.0, + Some(next) => top = next, None => break 'outer, } let remaining_weight = self.search_tree.descend_visit(PrefixWeightVisitor::new(top)).unwrap(); @@ -132,16 +144,19 @@ impl Frontier { } } + /// Exposed for benchmarking purposes pub fn build_selector_sample_inplace(&self) -> Box { let mut rng = rand::thread_rng(); let policy = Policy::new(500_000); Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, &policy), policy)) } + /// Exposed for benchmarking purposes pub fn build_selector_take_all(&self) -> Box { Box::new(TakeAllSelector::new(self.search_tree.iter().map(|(k, _)| k.tx.clone()).collect())) } + /// Exposed for benchmarking purposes pub fn build_rebalancing_selector(&self) -> Box { Box::new(RebalancingWeightedTransactionSelector::new( Policy::new(500_000), @@ -149,12 +164,37 @@ impl Frontier { )) } - pub fn len(&self) -> usize { - self.search_tree.len() - } + pub fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { + let mut total_mass = self.total_mass(); + let mut mass_per_second = args.network_mass_per_second; + let mut count = self.len(); + let mut average_transaction_mass = match self.len() { + // TODO (PR): remove consts + 0 => 500_000.0 / 300.0, + n => total_mass as f64 / n as f64, + }; + let mut inclusion_interval = average_transaction_mass / mass_per_second as f64; + let mut estimator = FeerateEstimator::new(self.total_weight(), inclusion_interval); + + // Search for better estimators by possibly removing extremely high outliers + for key in self.search_tree.iter().rev().map(|(key, ())| key) { + // TODO (PR): explain the importance of this visitor for numerical stability + let prefix_weight = self.search_tree.descend_visit(PrefixWeightVisitor::new(key)).unwrap(); + let pending_estimator = FeerateEstimator::new(prefix_weight, inclusion_interval); + + // Test the pending estimator vs the current one + if pending_estimator.feerate_to_time(1.0) < estimator.feerate_to_time(1.0) { + estimator = pending_estimator; + } - pub fn is_empty(&self) -> bool { - self.len() == 0 + // Update values for the next iteration + count -= 1; + total_mass -= key.mass; + mass_per_second -= key.mass; // TODO (PR): remove per block? lower bound? + average_transaction_mass = total_mass as f64 / count as f64; + inclusion_interval = average_transaction_mass / mass_per_second as f64 + } + estimator } } From 1729cbbd11ab9b613bb6b687265c59d008574ccf Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 00:42:00 +0000 Subject: [PATCH 35/74] todos --- mining/src/feerate/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index e2524f203..34390d3a1 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -8,9 +8,9 @@ pub struct FeerateBucket { #[derive(Clone, Debug)] pub struct FeerateEstimations { - pub low_bucket: FeerateBucket, - pub normal_bucket: FeerateBucket, - pub priority_bucket: FeerateBucket, + pub priority_bucket: FeerateBucket, // TODO (PR): priority = sub-second and at least 1.0 feerate (standard) + pub normal_bucket: FeerateBucket, // TODO (PR): change to a vec with sub-minute guarantee for first value + pub low_bucket: FeerateBucket, // TODO (PR): change to a vec with sub-hour guarantee for first value } pub struct FeerateEstimatorArgs { @@ -62,9 +62,9 @@ impl FeerateEstimator { let low = self.time_to_feerate(3600f64).max(self.quantile(1f64, high, 0.25)); let mid = self.time_to_feerate(60f64).max(self.quantile(low, high, 0.5)); FeerateEstimations { - low_bucket: FeerateBucket { feerate: low, estimated_seconds: self.feerate_to_time(low) }, - normal_bucket: FeerateBucket { feerate: mid, estimated_seconds: self.feerate_to_time(mid) }, priority_bucket: FeerateBucket { feerate: high, estimated_seconds: self.feerate_to_time(high) }, + normal_bucket: FeerateBucket { feerate: mid, estimated_seconds: self.feerate_to_time(mid) }, + low_bucket: FeerateBucket { feerate: low, estimated_seconds: self.feerate_to_time(low) }, } } } From 3122e16593eb995df3242771dc163ca1e3fa331f Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 00:56:36 +0000 Subject: [PATCH 36/74] minor --- mining/src/mempool/model/frontier/feerate_key.rs | 4 +++- mining/src/model/candidate_tx.rs | 6 ------ utils/Cargo.toml | 2 +- utils/src/vec.rs | 6 +++--- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/mining/src/mempool/model/frontier/feerate_key.rs b/mining/src/mempool/model/frontier/feerate_key.rs index 03572d282..1bee96f1d 100644 --- a/mining/src/mempool/model/frontier/feerate_key.rs +++ b/mining/src/mempool/model/frontier/feerate_key.rs @@ -72,6 +72,8 @@ impl Ord for FeerateTransactionKey { impl From<&MempoolTransaction> for FeerateTransactionKey { fn from(tx: &MempoolTransaction) -> Self { - Self::new(tx.mtx.calculated_fee.unwrap(), tx.mtx.tx.mass(), tx.mtx.tx.clone()) + let mass = tx.mtx.tx.mass(); + assert_ne!(mass, 0, "mass field is expected to be set when inserting to the mempool"); + Self::new(tx.mtx.calculated_fee.expect("fee is expected to be populated"), mass, tx.mtx.tx.clone()) } } diff --git a/mining/src/model/candidate_tx.rs b/mining/src/model/candidate_tx.rs index 858758eaf..b8cc34cc4 100644 --- a/mining/src/model/candidate_tx.rs +++ b/mining/src/model/candidate_tx.rs @@ -15,12 +15,6 @@ pub struct CandidateTransaction { } impl CandidateTransaction { - // pub(crate) fn from_mutable(tx: &MutableTransaction) -> Self { - // let mass = tx.tx.mass(); - // assert_ne!(mass, 0, "mass field is expected to be set when inserting to the mempool"); - // Self { tx: tx.tx.clone(), calculated_fee: tx.calculated_fee.expect("fee is expected to be populated"), calculated_mass: mass } - // } - pub fn from_key(key: FeerateTransactionKey) -> Self { Self { tx: key.tx, calculated_fee: key.fee, calculated_mass: key.mass } } diff --git a/utils/Cargo.toml b/utils/Cargo.toml index cecc69450..a3002afab 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -25,7 +25,6 @@ triggered.workspace = true uuid.workspace = true log.workspace = true wasm-bindgen.workspace = true -rand.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] rlimit.workspace = true @@ -37,6 +36,7 @@ async-trait.workspace = true futures-util.workspace = true tokio = { workspace = true, features = ["rt", "time", "macros"] } criterion.workspace = true +rand.workspace = true [[bench]] name = "bench" diff --git a/utils/src/vec.rs b/utils/src/vec.rs index 0ab30add2..fa1d67a27 100644 --- a/utils/src/vec.rs +++ b/utils/src/vec.rs @@ -5,9 +5,9 @@ pub trait VecExtensions { /// Inserts the provided `value` at `index` while swapping the item at index to the end of the container fn swap_insert(&mut self, index: usize, value: T); - /// Chains two containers one after the other and returns the result. The method is identical + /// Merges two containers one into the other and returns the result. The method is identical /// to [`Vec::append`] but can be used more ergonomically in a fluent calling fashion - fn chain(self, other: Self) -> Self; + fn merge(self, other: Self) -> Self; } impl VecExtensions for Vec { @@ -24,7 +24,7 @@ impl VecExtensions for Vec { self.swap(index, loc); } - fn chain(mut self, mut other: Self) -> Self { + fn merge(mut self, mut other: Self) -> Self { self.append(&mut other); self } From e8238b5309c68ae8a68c0e97f696ce9407b766d9 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 09:12:06 +0000 Subject: [PATCH 37/74] Remove obsolete constant --- mining/src/mempool/config.rs | 5 ----- mining/src/mempool/model/frontier/selectors.rs | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/mining/src/mempool/config.rs b/mining/src/mempool/config.rs index 41b677c49..06945b35a 100644 --- a/mining/src/mempool/config.rs +++ b/mining/src/mempool/config.rs @@ -1,7 +1,6 @@ use kaspa_consensus_core::constants::TX_VERSION; pub(crate) const DEFAULT_MAXIMUM_TRANSACTION_COUNT: u32 = 1_000_000; -pub(crate) const DEFAULT_MAXIMUM_READY_TRANSACTION_COUNT: u32 = 10_000; pub(crate) const DEFAULT_MAXIMUM_BUILD_BLOCK_TEMPLATE_ATTEMPTS: u64 = 5; pub(crate) const DEFAULT_TRANSACTION_EXPIRE_INTERVAL_SECONDS: u64 = 60; @@ -30,7 +29,6 @@ pub(crate) const DEFAULT_MAXIMUM_STANDARD_TRANSACTION_VERSION: u16 = TX_VERSION; #[derive(Clone, Debug)] pub struct Config { pub maximum_transaction_count: u32, - pub maximum_ready_transaction_count: u32, pub maximum_build_block_template_attempts: u64, pub transaction_expire_interval_daa_score: u64, pub transaction_expire_scan_interval_daa_score: u64, @@ -53,7 +51,6 @@ impl Config { #[allow(clippy::too_many_arguments)] pub fn new( maximum_transaction_count: u32, - maximum_ready_transaction_count: u32, maximum_build_block_template_attempts: u64, transaction_expire_interval_daa_score: u64, transaction_expire_scan_interval_daa_score: u64, @@ -73,7 +70,6 @@ impl Config { ) -> Self { Self { maximum_transaction_count, - maximum_ready_transaction_count, maximum_build_block_template_attempts, transaction_expire_interval_daa_score, transaction_expire_scan_interval_daa_score, @@ -98,7 +94,6 @@ impl Config { pub const fn build_default(target_milliseconds_per_block: u64, relay_non_std_transactions: bool, max_block_mass: u64) -> Self { Self { maximum_transaction_count: DEFAULT_MAXIMUM_TRANSACTION_COUNT, - maximum_ready_transaction_count: DEFAULT_MAXIMUM_READY_TRANSACTION_COUNT, maximum_build_block_template_attempts: DEFAULT_MAXIMUM_BUILD_BLOCK_TEMPLATE_ATTEMPTS, transaction_expire_interval_daa_score: DEFAULT_TRANSACTION_EXPIRE_INTERVAL_SECONDS * 1000 / target_milliseconds_per_block, transaction_expire_scan_interval_daa_score: DEFAULT_TRANSACTION_EXPIRE_SCAN_INTERVAL_SECONDS * 1000 diff --git a/mining/src/mempool/model/frontier/selectors.rs b/mining/src/mempool/model/frontier/selectors.rs index a7d89805b..9183d7696 100644 --- a/mining/src/mempool/model/frontier/selectors.rs +++ b/mining/src/mempool/model/frontier/selectors.rs @@ -148,7 +148,7 @@ impl TemplateTransactionSelector for TakeAllSelector { fn is_successful(&self) -> bool { // Considered successful because we provided all mempool transactions to this - // selector, so there's point in retries + // selector, so there's no point in retries true } } From 6a28c580a41074f9681eefabdf6499d6e7970bda Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 10:01:18 +0000 Subject: [PATCH 38/74] Restructure search tree methods into an encapsulated struct --- mining/src/mempool/model/frontier.rs | 142 ++-------------- .../src/mempool/model/frontier/feerate_key.rs | 23 +++ .../mempool/model/frontier/feerate_weight.rs | 157 +++++++++++++++++- 3 files changed, 196 insertions(+), 126 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 319ee8a8f..3c0b8e45a 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -5,13 +5,12 @@ use crate::{ }; use feerate_key::FeerateTransactionKey; -use feerate_weight::{FeerateWeight, PrefixWeightVisitor}; +use feerate_weight::SearchTree; use kaspa_consensus_core::block::TemplateTransactionSelector; use kaspa_core::trace; use rand::{distributions::Uniform, prelude::Distribution, Rng}; use selectors::{SequenceSelector, SequenceSelectorInput, TakeAllSelector}; use std::collections::HashSet; -use sweep_bptree::{BPlusTree, NodeStoreVec}; pub(crate) mod feerate_key; pub(crate) mod feerate_weight; @@ -26,28 +25,21 @@ const COLLISION_FACTOR: u64 = 4; /// hard limit in order to allow the SequenceSelector to compensate for consensus rejections. const MASS_LIMIT_FACTOR: f64 = 1.2; -pub type FrontierTree = BPlusTree>; - /// Management of the transaction pool frontier, that is, the set of transactions in /// the transaction pool which have no mempool ancestors and are essentially ready /// to enter the next block template. +#[derive(Default)] pub struct Frontier { /// Frontier transactions sorted by feerate order and searchable for weight sampling - search_tree: FrontierTree, + search_tree: SearchTree, /// Total masses: Σ_{tx in frontier} tx.mass total_mass: u64, } -impl Default for Frontier { - fn default() -> Self { - Self { search_tree: FrontierTree::new(Default::default()), total_mass: Default::default() } - } -} - impl Frontier { pub fn total_weight(&self) -> f64 { - self.search_tree.root_argument().weight() + self.search_tree.total_weight() } pub fn total_mass(&self) -> u64 { @@ -64,7 +56,7 @@ impl Frontier { pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { let mass = key.mass; - if self.search_tree.insert(key, ()).is_none() { + if self.search_tree.insert(key) { self.total_mass += mass; true } else { @@ -74,7 +66,7 @@ impl Frontier { pub fn remove(&mut self, key: &FeerateTransactionKey) -> bool { let mass = key.mass; - if self.search_tree.remove(key).is_some() { + if self.search_tree.remove(key) { self.total_mass -= mass; true } else { @@ -95,7 +87,7 @@ impl Frontier { let extended_mass_limit = (policy.max_block_mass as f64 * MASS_LIMIT_FACTOR) as u64; let mut distr = Uniform::new(0f64, self.total_weight()); - let mut down_iter = self.search_tree.iter().rev().map(|(key, ())| key); + let mut down_iter = self.search_tree.descending_iter(); let mut top = down_iter.next().unwrap(); let mut cache = HashSet::new(); let mut sequence = SequenceSelectorInput::default(); @@ -106,7 +98,7 @@ impl Frontier { 'outer: while cache.len() < self.search_tree.len() && total_selected_mass <= extended_mass_limit { let query = distr.sample(rng); let item = { - let mut item = self.search_tree.get_by_argument(query).expect("clamped").0; + let mut item = self.search_tree.search(query); while !cache.insert(item.tx.id()) { _collisions += 1; if top == item { @@ -115,11 +107,11 @@ impl Frontier { Some(next) => top = next, None => break 'outer, } - let remaining_weight = self.search_tree.descend_visit(PrefixWeightVisitor::new(top)).unwrap(); + let remaining_weight = self.search_tree.prefix_weight(top); distr = Uniform::new(0f64, remaining_weight); } let query = distr.sample(rng); - item = self.search_tree.get_by_argument(query).expect("clamped").0; + item = self.search_tree.search(query); } item }; @@ -132,14 +124,14 @@ impl Frontier { pub fn build_selector(&self, policy: &Policy) -> Box { if self.total_mass <= policy.max_block_mass { - Box::new(TakeAllSelector::new(self.search_tree.iter().map(|(k, _)| k.tx.clone()).collect())) + Box::new(TakeAllSelector::new(self.search_tree.ascending_iter().map(|k| k.tx.clone()).collect())) } else if self.total_mass > policy.max_block_mass * COLLISION_FACTOR { let mut rng = rand::thread_rng(); Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, policy), policy.clone())) } else { Box::new(RebalancingWeightedTransactionSelector::new( policy.clone(), - self.search_tree.iter().map(|(k, _)| k.clone()).map(CandidateTransaction::from_key).collect(), + self.search_tree.ascending_iter().cloned().map(CandidateTransaction::from_key).collect(), )) } } @@ -153,14 +145,14 @@ impl Frontier { /// Exposed for benchmarking purposes pub fn build_selector_take_all(&self) -> Box { - Box::new(TakeAllSelector::new(self.search_tree.iter().map(|(k, _)| k.tx.clone()).collect())) + Box::new(TakeAllSelector::new(self.search_tree.ascending_iter().map(|k| k.tx.clone()).collect())) } /// Exposed for benchmarking purposes pub fn build_rebalancing_selector(&self) -> Box { Box::new(RebalancingWeightedTransactionSelector::new( Policy::new(500_000), - self.search_tree.iter().map(|(k, _)| k.clone()).map(CandidateTransaction::from_key).collect(), + self.search_tree.ascending_iter().cloned().map(CandidateTransaction::from_key).collect(), )) } @@ -177,9 +169,9 @@ impl Frontier { let mut estimator = FeerateEstimator::new(self.total_weight(), inclusion_interval); // Search for better estimators by possibly removing extremely high outliers - for key in self.search_tree.iter().rev().map(|(key, ())| key) { + for key in self.search_tree.descending_iter() { // TODO (PR): explain the importance of this visitor for numerical stability - let prefix_weight = self.search_tree.descend_visit(PrefixWeightVisitor::new(key)).unwrap(); + let prefix_weight = self.search_tree.prefix_weight(key); let pending_estimator = FeerateEstimator::new(prefix_weight, inclusion_interval); // Test the pending estimator vs the current one @@ -201,25 +193,9 @@ impl Frontier { #[cfg(test)] mod tests { use super::*; - use itertools::Itertools; - use kaspa_consensus_core::{ - subnets::SUBNETWORK_ID_NATIVE, - tx::{Transaction, TransactionInput, TransactionOutpoint}, - }; - use kaspa_hashes::{HasherBase, TransactionID}; + use feerate_key::tests::build_feerate_key; use rand::thread_rng; - use std::{collections::HashMap, sync::Arc}; - - fn generate_unique_tx(i: u64) -> Arc { - let mut hasher = TransactionID::new(); - let prev = hasher.update(i.to_le_bytes()).clone().finalize(); - let input = TransactionInput::new(TransactionOutpoint::new(prev, 0), vec![], 0, 0); - Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) - } - - fn build_feerate_key(fee: u64, mass: u64, id: u64) -> FeerateTransactionKey { - FeerateTransactionKey::new(fee, mass, generate_unique_tx(id)) - } + use std::collections::HashMap; #[test] pub fn test_highly_irregular_sampling() { @@ -247,88 +223,6 @@ mod tests { // assert_eq!(100, sample.len()); } - #[test] - fn test_feerate_weight_queries() { - let mut btree = FrontierTree::new(Default::default()); - let mass = 2000; - // The btree stores N=64 keys at each node/leaf, so we make sure the tree has more than - // 64^2 keys in order to trigger at least a few intermediate tree nodes - let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); - - #[allow(clippy::mutable_key_type)] - let mut s = HashSet::with_capacity(fees.len()); - for (i, fee) in fees.iter().copied().enumerate() { - let key = build_feerate_key(fee, mass, i as u64); - s.insert(key.clone()); - btree.insert(key, ()); - } - - // Randomly remove 1/6 of the items - let remove = s.iter().take(fees.len() / 6).cloned().collect_vec(); - for r in remove { - s.remove(&r); - btree.remove(&r); - } - - // Collect to vec and sort for reference - let mut v = s.into_iter().collect_vec(); - v.sort(); - - // Test reverse iteration - for (expected, item) in v.iter().rev().zip(btree.iter().rev()) { - assert_eq!(&expected, &item.0); - assert!(expected.cmp(item.0).is_eq()); // Assert Ord equality as well - } - - // Sweep through the tree and verify that weight search queries are handled correctly - let eps: f64 = 0.001; - let mut sum = 0.0; - for expected in v { - let weight = expected.weight(); - let eps = eps.min(weight / 3.0); - let samples = [sum + eps, sum + weight / 2.0, sum + weight - eps]; - for sample in samples { - let key = btree.get_by_argument(sample).unwrap().0; - assert_eq!(&expected, key); - assert!(expected.cmp(key).is_eq()); // Assert Ord equality as well - } - sum += weight; - } - - println!("{}, {}", sum, btree.root_argument().weight()); - - // Test clamped search bounds - assert_eq!(btree.first(), btree.get_by_argument(f64::NEG_INFINITY)); - assert_eq!(btree.first(), btree.get_by_argument(-1.0)); - assert_eq!(btree.first(), btree.get_by_argument(-eps)); - assert_eq!(btree.first(), btree.get_by_argument(0.0)); - assert_eq!(btree.last(), btree.get_by_argument(sum)); - assert_eq!(btree.last(), btree.get_by_argument(sum + eps)); - assert_eq!(btree.last(), btree.get_by_argument(sum + 1.0)); - assert_eq!(btree.last(), btree.get_by_argument(1.0 / 0.0)); - assert_eq!(btree.last(), btree.get_by_argument(f64::INFINITY)); - assert!(btree.get_by_argument(f64::NAN).is_some()); - } - - #[test] - fn test_btree_rev_iter() { - let mut btree = FrontierTree::new(Default::default()); - let mass = 2000; - let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); - let mut v = Vec::with_capacity(fees.len()); - for (i, fee) in fees.iter().copied().enumerate() { - let key = build_feerate_key(fee, mass, i as u64); - v.push(key.clone()); - btree.insert(key, ()); - } - v.sort(); - - for (expected, item) in v.into_iter().rev().zip(btree.iter().rev()) { - assert_eq!(&expected, item.0); - assert!(expected.cmp(item.0).is_eq()); // Assert Ord equality as well - } - } - #[test] pub fn test_mempool_sampling_small() { let mut rng = thread_rng(); diff --git a/mining/src/mempool/model/frontier/feerate_key.rs b/mining/src/mempool/model/frontier/feerate_key.rs index 1bee96f1d..b6b3f181e 100644 --- a/mining/src/mempool/model/frontier/feerate_key.rs +++ b/mining/src/mempool/model/frontier/feerate_key.rs @@ -77,3 +77,26 @@ impl From<&MempoolTransaction> for FeerateTransactionKey { Self::new(tx.mtx.calculated_fee.expect("fee is expected to be populated"), mass, tx.mtx.tx.clone()) } } + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use kaspa_consensus_core::{ + subnets::SUBNETWORK_ID_NATIVE, + tx::{Transaction, TransactionInput, TransactionOutpoint}, + }; + use kaspa_hashes::{HasherBase, TransactionID}; + use std::sync::Arc; + + fn generate_unique_tx(i: u64) -> Arc { + let mut hasher = TransactionID::new(); + let prev = hasher.update(i.to_le_bytes()).clone().finalize(); + let input = TransactionInput::new(TransactionOutpoint::new(prev, 0), vec![], 0, 0); + Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) + } + + /// Test helper for generating a feerate key with a unique tx (per u64 id) + pub(crate) fn build_feerate_key(fee: u64, mass: u64, id: u64) -> FeerateTransactionKey { + FeerateTransactionKey::new(fee, mass, generate_unique_tx(id)) + } +} diff --git a/mining/src/mempool/model/frontier/feerate_weight.rs b/mining/src/mempool/model/frontier/feerate_weight.rs index 85ff6998f..8421c7b8e 100644 --- a/mining/src/mempool/model/frontier/feerate_weight.rs +++ b/mining/src/mempool/model/frontier/feerate_weight.rs @@ -1,11 +1,12 @@ use super::feerate_key::FeerateTransactionKey; use sweep_bptree::tree::visit::{DescendVisit, DescendVisitResult}; use sweep_bptree::tree::{Argument, SearchArgument}; +use sweep_bptree::{BPlusTree, NodeStoreVec}; type FeerateKey = FeerateTransactionKey; #[derive(Clone, Copy, Debug, Default)] -pub struct FeerateWeight(f64); +struct FeerateWeight(f64); impl FeerateWeight { /// Returns the weight value @@ -64,7 +65,7 @@ impl SearchArgument for FeerateWeight { } } -pub struct PrefixWeightVisitor<'a> { +struct PrefixWeightVisitor<'a> { key: &'a FeerateKey, accumulated_weight: f64, } @@ -110,3 +111,155 @@ impl<'a> DescendVisit for PrefixWeightVisitor<'a> Some(self.accumulated_weight) } } + +type InnerTree = BPlusTree>; + +pub struct SearchTree { + tree: InnerTree, +} + +impl Default for SearchTree { + fn default() -> Self { + Self { tree: InnerTree::new(Default::default()) } + } +} + +impl SearchTree { + pub fn new() -> Self { + Self { tree: InnerTree::new(Default::default()) } + } + + pub fn len(&self) -> usize { + self.tree.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn insert(&mut self, key: FeerateKey) -> bool { + self.tree.insert(key, ()).is_none() + } + + pub fn remove(&mut self, key: &FeerateKey) -> bool { + self.tree.remove(key).is_some() + } + + pub fn search(&self, query: f64) -> &FeerateKey { + self.tree.get_by_argument(query).expect("clamped").0 + } + + pub fn total_weight(&self) -> f64 { + self.tree.root_argument().weight() + } + + pub fn prefix_weight(&self, key: &FeerateKey) -> f64 { + self.tree.descend_visit(PrefixWeightVisitor::new(key)).unwrap() + } + + pub fn descending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator { + self.tree.iter().rev().map(|(key, ())| key) + } + + pub fn ascending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator { + self.tree.iter().map(|(key, ())| key) + } + + pub fn first(&self) -> Option<&FeerateKey> { + self.tree.first().map(|(k, ())| k) + } + + pub fn last(&self) -> Option<&FeerateKey> { + self.tree.last().map(|(k, ())| k) + } +} + +#[cfg(test)] +mod tests { + use super::super::feerate_key::tests::build_feerate_key; + use super::*; + use itertools::Itertools; + use std::collections::HashSet; + + #[test] + fn test_feerate_weight_queries() { + let mut btree = SearchTree::new(); + let mass = 2000; + // The btree stores N=64 keys at each node/leaf, so we make sure the tree has more than + // 64^2 keys in order to trigger at least a few intermediate tree nodes + let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); + + #[allow(clippy::mutable_key_type)] + let mut s = HashSet::with_capacity(fees.len()); + for (i, fee) in fees.iter().copied().enumerate() { + let key = build_feerate_key(fee, mass, i as u64); + s.insert(key.clone()); + btree.insert(key); + } + + // Randomly remove 1/6 of the items + let remove = s.iter().take(fees.len() / 6).cloned().collect_vec(); + for r in remove { + s.remove(&r); + btree.remove(&r); + } + + // Collect to vec and sort for reference + let mut v = s.into_iter().collect_vec(); + v.sort(); + + // Test reverse iteration + for (expected, item) in v.iter().rev().zip(btree.descending_iter()) { + assert_eq!(&expected, &item); + assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well + } + + // Sweep through the tree and verify that weight search queries are handled correctly + let eps: f64 = 0.001; + let mut sum = 0.0; + for expected in v { + let weight = expected.weight(); + let eps = eps.min(weight / 3.0); + let samples = [sum + eps, sum + weight / 2.0, sum + weight - eps]; + for sample in samples { + let key = btree.search(sample); + assert_eq!(&expected, key); + assert!(expected.cmp(key).is_eq()); // Assert Ord equality as well + } + sum += weight; + } + + println!("{}, {}", sum, btree.total_weight()); + + // Test clamped search bounds + assert_eq!(btree.first(), Some(btree.search(f64::NEG_INFINITY))); + assert_eq!(btree.first(), Some(btree.search(-1.0))); + assert_eq!(btree.first(), Some(btree.search(-eps))); + assert_eq!(btree.first(), Some(btree.search(0.0))); + assert_eq!(btree.last(), Some(btree.search(sum))); + assert_eq!(btree.last(), Some(btree.search(sum + eps))); + assert_eq!(btree.last(), Some(btree.search(sum + 1.0))); + assert_eq!(btree.last(), Some(btree.search(1.0 / 0.0))); + assert_eq!(btree.last(), Some(btree.search(f64::INFINITY))); + let _ = btree.search(f64::NAN); + } + + #[test] + fn test_btree_rev_iter() { + let mut btree = SearchTree::new(); + let mass = 2000; + let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); + let mut v = Vec::with_capacity(fees.len()); + for (i, fee) in fees.iter().copied().enumerate() { + let key = build_feerate_key(fee, mass, i as u64); + v.push(key.clone()); + btree.insert(key); + } + v.sort(); + + for (expected, item) in v.into_iter().rev().zip(btree.descending_iter()) { + assert_eq!(&expected, item); + assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well + } + } +} From 5159c090b3727b0c8f5275353cd49e81cdd6c9db Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 10:06:00 +0000 Subject: [PATCH 39/74] Rename module --- mining/src/mempool/model/frontier.rs | 4 +- .../{feerate_weight.rs => search_tree.rs} | 38 +++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) rename mining/src/mempool/model/frontier/{feerate_weight.rs => search_tree.rs} (87%) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 3c0b8e45a..337efabb9 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -5,15 +5,15 @@ use crate::{ }; use feerate_key::FeerateTransactionKey; -use feerate_weight::SearchTree; use kaspa_consensus_core::block::TemplateTransactionSelector; use kaspa_core::trace; use rand::{distributions::Uniform, prelude::Distribution, Rng}; +use search_tree::SearchTree; use selectors::{SequenceSelector, SequenceSelectorInput, TakeAllSelector}; use std::collections::HashSet; pub(crate) mod feerate_key; -pub(crate) mod feerate_weight; +pub(crate) mod search_tree; pub(crate) mod selectors; /// If the frontier contains less than 4x the block mass limit, we consider diff --git a/mining/src/mempool/model/frontier/feerate_weight.rs b/mining/src/mempool/model/frontier/search_tree.rs similarity index 87% rename from mining/src/mempool/model/frontier/feerate_weight.rs rename to mining/src/mempool/model/frontier/search_tree.rs index 8421c7b8e..ca4cafbe5 100644 --- a/mining/src/mempool/model/frontier/feerate_weight.rs +++ b/mining/src/mempool/model/frontier/search_tree.rs @@ -183,7 +183,7 @@ mod tests { #[test] fn test_feerate_weight_queries() { - let mut btree = SearchTree::new(); + let mut tree = SearchTree::new(); let mass = 2000; // The btree stores N=64 keys at each node/leaf, so we make sure the tree has more than // 64^2 keys in order to trigger at least a few intermediate tree nodes @@ -194,14 +194,14 @@ mod tests { for (i, fee) in fees.iter().copied().enumerate() { let key = build_feerate_key(fee, mass, i as u64); s.insert(key.clone()); - btree.insert(key); + tree.insert(key); } // Randomly remove 1/6 of the items let remove = s.iter().take(fees.len() / 6).cloned().collect_vec(); for r in remove { s.remove(&r); - btree.remove(&r); + tree.remove(&r); } // Collect to vec and sort for reference @@ -209,7 +209,7 @@ mod tests { v.sort(); // Test reverse iteration - for (expected, item) in v.iter().rev().zip(btree.descending_iter()) { + for (expected, item) in v.iter().rev().zip(tree.descending_iter()) { assert_eq!(&expected, &item); assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well } @@ -222,42 +222,42 @@ mod tests { let eps = eps.min(weight / 3.0); let samples = [sum + eps, sum + weight / 2.0, sum + weight - eps]; for sample in samples { - let key = btree.search(sample); + let key = tree.search(sample); assert_eq!(&expected, key); assert!(expected.cmp(key).is_eq()); // Assert Ord equality as well } sum += weight; } - println!("{}, {}", sum, btree.total_weight()); + println!("{}, {}", sum, tree.total_weight()); // Test clamped search bounds - assert_eq!(btree.first(), Some(btree.search(f64::NEG_INFINITY))); - assert_eq!(btree.first(), Some(btree.search(-1.0))); - assert_eq!(btree.first(), Some(btree.search(-eps))); - assert_eq!(btree.first(), Some(btree.search(0.0))); - assert_eq!(btree.last(), Some(btree.search(sum))); - assert_eq!(btree.last(), Some(btree.search(sum + eps))); - assert_eq!(btree.last(), Some(btree.search(sum + 1.0))); - assert_eq!(btree.last(), Some(btree.search(1.0 / 0.0))); - assert_eq!(btree.last(), Some(btree.search(f64::INFINITY))); - let _ = btree.search(f64::NAN); + assert_eq!(tree.first(), Some(tree.search(f64::NEG_INFINITY))); + assert_eq!(tree.first(), Some(tree.search(-1.0))); + assert_eq!(tree.first(), Some(tree.search(-eps))); + assert_eq!(tree.first(), Some(tree.search(0.0))); + assert_eq!(tree.last(), Some(tree.search(sum))); + assert_eq!(tree.last(), Some(tree.search(sum + eps))); + assert_eq!(tree.last(), Some(tree.search(sum + 1.0))); + assert_eq!(tree.last(), Some(tree.search(1.0 / 0.0))); + assert_eq!(tree.last(), Some(tree.search(f64::INFINITY))); + let _ = tree.search(f64::NAN); } #[test] fn test_btree_rev_iter() { - let mut btree = SearchTree::new(); + let mut tree = SearchTree::new(); let mass = 2000; let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); let mut v = Vec::with_capacity(fees.len()); for (i, fee) in fees.iter().copied().enumerate() { let key = build_feerate_key(fee, mass, i as u64); v.push(key.clone()); - btree.insert(key); + tree.insert(key); } v.sort(); - for (expected, item) in v.into_iter().rev().zip(btree.descending_iter()) { + for (expected, item) in v.into_iter().rev().zip(tree.descending_iter()) { assert_eq!(&expected, item); assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well } From fd18ef81fd35645ce8513f398272f6b12089dbeb Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 11:25:32 +0000 Subject: [PATCH 40/74] documentation and comments --- .../src/mempool/model/frontier/search_tree.rs | 66 +++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/mining/src/mempool/model/frontier/search_tree.rs b/mining/src/mempool/model/frontier/search_tree.rs index ca4cafbe5..cb6760385 100644 --- a/mining/src/mempool/model/frontier/search_tree.rs +++ b/mining/src/mempool/model/frontier/search_tree.rs @@ -5,6 +5,17 @@ use sweep_bptree::{BPlusTree, NodeStoreVec}; type FeerateKey = FeerateTransactionKey; +/// A struct for implementing "weight space" search using the SearchArgument customization. +/// The weight space is the range `[0, total_weight)` and each key has a "logical" interval allocation +/// within this space according to its tree position and weight. +/// +/// We implement the search efficiently by maintaining subtree weights which are updated with each +/// element insertion/removal. Given a search query `p ∈ [0, total_weight)` we then find the corresponding +/// element in log time by walking down from the root and adjusting the query according to subtree weights. +/// For instance if the query point is `123.56` and the top 3 subtrees have weights `120, 10.5 ,100` then we +/// query the middle subtree with the point `123.56 - 120 = 3.56`. +/// +/// See SearchArgument implementation below for more details. #[derive(Clone, Copy, Debug, Default)] struct FeerateWeight(f64); @@ -47,6 +58,8 @@ impl SearchArgument for FeerateWeight { } fn locate_in_inner(mut query: Self::Query, _keys: &[FeerateKey], arguments: &[Self]) -> Option<(usize, Self::Query)> { + // Search algorithm: Locate the next subtree to visit by iterating through `arguments` + // and subtracting the query until the correct range is found for (i, a) in arguments.iter().enumerate() { if query >= a.0 { query -= a.0; @@ -65,8 +78,14 @@ impl SearchArgument for FeerateWeight { } } +/// Visitor struct which accumulates the prefix weight up to a provided key (inclusive) in log time. +/// +/// The basic idea is to use the subtree weights stored in the tree for walking down from the root +/// to the leaf (corresponding to the searched key), and accumulating all weights proceeding the walk-down path struct PrefixWeightVisitor<'a> { + /// The key to search up to key: &'a FeerateKey, + /// This field accumulates the prefix weight during the visit process accumulated_weight: f64, } @@ -75,15 +94,16 @@ impl<'a> PrefixWeightVisitor<'a> { Self { key, accumulated_weight: Default::default() } } + /// Returns the index of the first `key ∈ keys` such that `key > self.key`. If no such key + /// exists, the returned index will be the length of `keys`. fn search_in_keys(&self, keys: &[FeerateKey]) -> usize { match keys.binary_search(self.key) { Err(idx) => { - // The idx is the place where a matching element could be inserted while maintaining - // sorted order, go to left child + // self.key is not in keys, idx is the index of the following key idx } Ok(idx) => { - // Exact match, go to right child. + // Exact match, return the following index idx + 1 } } @@ -95,25 +115,49 @@ impl<'a> DescendVisit for PrefixWeightVisitor<'a> fn visit_inner(&mut self, keys: &[FeerateKey], arguments: &[FeerateWeight]) -> DescendVisitResult { let idx = self.search_in_keys(keys); - // trace!("[visit_inner] {}, {}, {}", keys.len(), arguments.len(), idx); + // Invariants: + // a. arguments.len() == keys.len() + 1 (n inner node keys are the separators between n+1 subtrees) + // b. idx <= keys.len() (hence idx < arguments.len()) + + // Based on the invariants, we first accumulate all the subtree weights up to idx for argument in arguments.iter().take(idx) { self.accumulated_weight += argument.weight(); } + + // ..and then go down to the idx'th subtree DescendVisitResult::GoDown(idx) } fn visit_leaf(&mut self, keys: &[FeerateKey], _values: &[()]) -> Option { + // idx is the index of the key following self.key let idx = self.search_in_keys(keys); - // trace!("[visit_leaf] {}, {}", keys.len(), idx); + // Accumulate all key weights up to idx (which is inclusive if self.key ∈ tree) for key in keys.iter().take(idx) { self.accumulated_weight += key.weight(); } + // ..and return the final result Some(self.accumulated_weight) } } type InnerTree = BPlusTree>; +/// A transaction search tree sorted by feerate order and searchable for probabilistic weighted sampling. +/// +/// All `log(n)` expressions below are in base 64 (based on constants chosen within the sweep_bptree crate). +/// +/// The tree has the following properties: +/// 1. Linear time ordered access (ascending / descending) +/// 2. Insertions/removals in log(n) time +/// 3. Search for a weight point `p ∈ [0, total_weight)` in log(n) time +/// 4. Compute the prefix weight of a key, i.e., the sum of weights up to that key (inclusive) +/// according to key order, in log(n) time +/// 5. Access the total weight in O(1) time. The total weight has numerical stability since it +/// is recomputed from subtree weights for each item insertion/removal +/// +/// Computing the prefix weight is a crucial operation if the tree is used for random sampling and +/// the tree is highly imbalanced in terms of weight variance. See [`Frontier::sample_inplace`] for +/// more details. pub struct SearchTree { tree: InnerTree, } @@ -137,38 +181,50 @@ impl SearchTree { self.len() == 0 } + /// Inserts a key into the tree in log(n) time. Returns `false` is the key was already in the tree. pub fn insert(&mut self, key: FeerateKey) -> bool { self.tree.insert(key, ()).is_none() } + /// Remove a key from the tree in log(n) time. Returns `false` is the key was not in the tree. pub fn remove(&mut self, key: &FeerateKey) -> bool { self.tree.remove(key).is_some() } + /// Search for a weight point `query ∈ [0, total_weight)` in log(n) time pub fn search(&self, query: f64) -> &FeerateKey { self.tree.get_by_argument(query).expect("clamped").0 } + /// Access the total weight in O(1) time pub fn total_weight(&self) -> f64 { self.tree.root_argument().weight() } + /// Computes the prefix weight of a key, i.e., the sum of weights up to that key (inclusive) + /// according to key order, in log(n) time pub fn prefix_weight(&self, key: &FeerateKey) -> f64 { self.tree.descend_visit(PrefixWeightVisitor::new(key)).unwrap() } + /// Iterate the tree in descending key order (going down from the + /// highest key). Linear in the number of keys *actually* iterated. pub fn descending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator { self.tree.iter().rev().map(|(key, ())| key) } + /// Iterate the tree in ascending key order (going up from the + /// lowest key). Linear in the number of keys *actually* iterated. pub fn ascending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator { self.tree.iter().map(|(key, ())| key) } + /// The lowest key in the tree (by key order) pub fn first(&self) -> Option<&FeerateKey> { self.tree.first().map(|(k, ())| k) } + /// The highest key in the tree (by key order) pub fn last(&self) -> Option<&FeerateKey> { self.tree.last().map(|(k, ())| k) } From e5606f64a5cff1b06637adec8ef1e3580e182286 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 11:26:16 +0000 Subject: [PATCH 41/74] optimization: cmp with cached weight rather than compute feerate --- mining/src/mempool/model/frontier/feerate_key.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mining/src/mempool/model/frontier/feerate_key.rs b/mining/src/mempool/model/frontier/feerate_key.rs index b6b3f181e..4be5ce6c8 100644 --- a/mining/src/mempool/model/frontier/feerate_key.rs +++ b/mining/src/mempool/model/frontier/feerate_key.rs @@ -47,13 +47,15 @@ impl PartialOrd for FeerateTransactionKey { impl Ord for FeerateTransactionKey { fn cmp(&self, other: &Self) -> std::cmp::Ordering { - // Our first priority is the feerate - match self.feerate().total_cmp(&other.feerate()) { + // Our first priority is the feerate. + // The weight function is monotonic in feerate so we prefer using it + // since it is cached + match self.weight().total_cmp(&other.weight()) { core::cmp::Ordering::Equal => {} ord => return ord, } - // If feerates are equal, prefer the higher fee in absolute value + // If feerates (and thus weights) are equal, prefer the higher fee in absolute value match self.fee.cmp(&other.fee) { core::cmp::Ordering::Equal => {} ord => return ord, From d534398336cc22df4d2f202df543206e026659b9 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 11:29:58 +0000 Subject: [PATCH 42/74] minor --- mining/src/mempool/model/frontier/search_tree.rs | 6 +++--- mining/src/mempool/model/frontier/selectors.rs | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mining/src/mempool/model/frontier/search_tree.rs b/mining/src/mempool/model/frontier/search_tree.rs index cb6760385..687cc9091 100644 --- a/mining/src/mempool/model/frontier/search_tree.rs +++ b/mining/src/mempool/model/frontier/search_tree.rs @@ -13,7 +13,7 @@ type FeerateKey = FeerateTransactionKey; /// element insertion/removal. Given a search query `p ∈ [0, total_weight)` we then find the corresponding /// element in log time by walking down from the root and adjusting the query according to subtree weights. /// For instance if the query point is `123.56` and the top 3 subtrees have weights `120, 10.5 ,100` then we -/// query the middle subtree with the point `123.56 - 120 = 3.56`. +/// recursively query the middle subtree with the point `123.56 - 120 = 3.56`. /// /// See SearchArgument implementation below for more details. #[derive(Clone, Copy, Debug, Default)] @@ -181,12 +181,12 @@ impl SearchTree { self.len() == 0 } - /// Inserts a key into the tree in log(n) time. Returns `false` is the key was already in the tree. + /// Inserts a key into the tree in log(n) time. Returns `false` if the key was already in the tree. pub fn insert(&mut self, key: FeerateKey) -> bool { self.tree.insert(key, ()).is_none() } - /// Remove a key from the tree in log(n) time. Returns `false` is the key was not in the tree. + /// Remove a key from the tree in log(n) time. Returns `false` if the key was not in the tree. pub fn remove(&mut self, key: &FeerateKey) -> bool { self.tree.remove(key).is_some() } diff --git a/mining/src/mempool/model/frontier/selectors.rs b/mining/src/mempool/model/frontier/selectors.rs index 9183d7696..c37ae81a9 100644 --- a/mining/src/mempool/model/frontier/selectors.rs +++ b/mining/src/mempool/model/frontier/selectors.rs @@ -51,7 +51,7 @@ struct SequenceSelectorSelection { /// that the transactions were already selected via weighted sampling and simply tries them one /// after the other until the block mass limit is reached. pub struct SequenceSelector { - priority_map: SequenceSelectorInput, + input_sequence: SequenceSelectorInput, selected_vec: Vec, selected_map: Option>, total_selected_mass: u64, @@ -61,11 +61,11 @@ pub struct SequenceSelector { } impl SequenceSelector { - pub fn new(priority_map: SequenceSelectorInput, policy: Policy) -> Self { + pub fn new(input_sequence: SequenceSelectorInput, policy: Policy) -> Self { Self { - overall_candidates: priority_map.inner.len(), - selected_vec: Vec::with_capacity(priority_map.inner.len()), - priority_map, + overall_candidates: input_sequence.inner.len(), + selected_vec: Vec::with_capacity(input_sequence.inner.len()), + input_sequence, selected_map: Default::default(), total_selected_mass: Default::default(), overall_rejections: Default::default(), @@ -84,14 +84,14 @@ impl TemplateTransactionSelector for SequenceSelector { fn select_transactions(&mut self) -> Vec { // Remove selections from the previous round if any for selection in self.selected_vec.drain(..) { - self.priority_map.inner.remove(&selection.priority_index); + self.input_sequence.inner.remove(&selection.priority_index); } // Reset selection data structures self.reset_selection(); - let mut transactions = Vec::with_capacity(self.priority_map.inner.len()); + let mut transactions = Vec::with_capacity(self.input_sequence.inner.len()); // Iterate the input sequence in order - for (&priority_index, tx) in self.priority_map.inner.iter() { + for (&priority_index, tx) in self.input_sequence.inner.iter() { if self.total_selected_mass.saturating_add(tx.mass) > self.policy.max_block_mass { // We assume the sequence is relatively small, hence we keep on searching // for transactions with lower mass which might fit into the remaining gap From 8040d4915635a4ff33345a218dadcf0cc14ce80f Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 13:33:51 +0000 Subject: [PATCH 43/74] Finalize build fee estimator and add tests --- mining/src/feerate/mod.rs | 9 +++- mining/src/mempool/model/frontier.rs | 79 ++++++++++++++++++++++------ 2 files changed, 70 insertions(+), 18 deletions(-) diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index 34390d3a1..f90999e80 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -14,7 +14,14 @@ pub struct FeerateEstimations { } pub struct FeerateEstimatorArgs { - pub network_mass_per_second: u64, + pub network_blocks_per_second: u64, + pub maximum_mass_per_block: u64, +} + +impl FeerateEstimatorArgs { + pub fn network_mass_per_second(&self) -> u64 { + self.network_blocks_per_second * self.maximum_mass_per_block + } } pub struct FeerateEstimator { diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 337efabb9..ed572868e 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -25,6 +25,10 @@ const COLLISION_FACTOR: u64 = 4; /// hard limit in order to allow the SequenceSelector to compensate for consensus rejections. const MASS_LIMIT_FACTOR: f64 = 1.2; +/// A rough estimation for the average transaction mass. The usage is a non-important edge case +/// hence we just throw this here (as oppose to performing an accurate estimation) +const TYPICAL_TX_MASS: f64 = 2000.0; + /// Management of the transaction pool frontier, that is, the set of transactions in /// the transaction pool which have no mempool ancestors and are essentially ready /// to enter the next block template. @@ -157,34 +161,44 @@ impl Frontier { } pub fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { - let mut total_mass = self.total_mass(); - let mut mass_per_second = args.network_mass_per_second; - let mut count = self.len(); - let mut average_transaction_mass = match self.len() { - // TODO (PR): remove consts - 0 => 500_000.0 / 300.0, - n => total_mass as f64 / n as f64, + let average_transaction_mass = match self.len() { + 0 => TYPICAL_TX_MASS, + n => self.total_mass() as f64 / n as f64, }; - let mut inclusion_interval = average_transaction_mass / mass_per_second as f64; + let bps = args.network_blocks_per_second as f64; + let mut mass_per_block = args.maximum_mass_per_block as f64; + let mut inclusion_interval = average_transaction_mass / (mass_per_block * bps); let mut estimator = FeerateEstimator::new(self.total_weight(), inclusion_interval); + // Corresponds to the removal of the top item, hence the skip(1) below + mass_per_block -= average_transaction_mass; + inclusion_interval = average_transaction_mass / (mass_per_block * bps); + // Search for better estimators by possibly removing extremely high outliers - for key in self.search_tree.descending_iter() { - // TODO (PR): explain the importance of this visitor for numerical stability + for key in self.search_tree.descending_iter().skip(1) { + // Compute the weight up to, and including, current key let prefix_weight = self.search_tree.prefix_weight(key); let pending_estimator = FeerateEstimator::new(prefix_weight, inclusion_interval); - // Test the pending estimator vs the current one + // Test the pending estimator vs. the current one if pending_estimator.feerate_to_time(1.0) < estimator.feerate_to_time(1.0) { estimator = pending_estimator; + } else { + // The pending estimator is no better, break. Indicates that the reduction in + // network mass per second is more significant than the removed weight + break; + } + + // Update values for the next iteration. In order to remove the outlier from the + // total weight, we must compensate by capturing a block slot. + mass_per_block -= average_transaction_mass; + if mass_per_block <= 0.0 { + // Out of block slots, break (this is rarely reachable code due to dynamics related to the above break) + break; } - // Update values for the next iteration - count -= 1; - total_mass -= key.mass; - mass_per_second -= key.mass; // TODO (PR): remove per block? lower bound? - average_transaction_mass = total_mass as f64 / count as f64; - inclusion_interval = average_transaction_mass / mass_per_second as f64 + // Re-calc the inclusion interval based on the new block "capacity" + inclusion_interval = average_transaction_mass / (mass_per_block * bps); } estimator } @@ -256,4 +270,35 @@ mod tests { let mut selector = frontier.build_selector(&Policy::new(500_000)); selector.select_transactions().iter().map(|k| k.gas).sum::(); } + + #[test] + fn test_feerate_estimator() { + let mut rng = thread_rng(); + let cap = 2000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let mut fee: u64 = rng.gen_range(1..1000000); + let mass: u64 = 1650; + // 304 (~500,000/1650) extreme outliers is an edge case where the build estimator logic should be tested at + if i <= 303 { + // Add an extremely large fee in order to create extremely high variance + fee = i * 10_000_000 * 1_000_000; + } + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + for len in [10, 100, 200, 300, 500, 750, cap / 2, (cap * 2) / 3, (cap * 4) / 5, (cap * 5) / 6, cap] { + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let args = FeerateEstimatorArgs { network_blocks_per_second: 1, maximum_mass_per_block: 500_000 }; + // We are testing that the build function actually returns and is not looping indefinitely + let estimator = frontier.build_feerate_estimator(args); + let _estimations = estimator.calc_estimations(); + // dbg!(_estimations); + } + } } From c931b89ae0048662c8c1019396a671e90ef55ab4 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 13:45:58 +0000 Subject: [PATCH 44/74] updated notebook --- mining/src/feerate/fee_estimation.ipynb | 138 ++++++++++++++++++------ 1 file changed, 106 insertions(+), 32 deletions(-) diff --git a/mining/src/feerate/fee_estimation.ipynb b/mining/src/feerate/fee_estimation.ipynb index 67d69e511..da8d71fed 100644 --- a/mining/src/feerate/fee_estimation.ipynb +++ b/mining/src/feerate/fee_estimation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 43, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -21,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -31,7 +31,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -49,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -61,24 +61,24 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Inclusion time: 0.004\n" + "Inclusion interval: 0.004\n" ] } ], "source": [ - "print('Inclusion time: ', avg_mass/network_mass_rate)" + "print('Inclusion interval: ', avg_mass/network_mass_rate)" ] }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -120,33 +120,33 @@ " `total_weight`: The total probability weight of all current mempool ready \n", " transactions, i.e., Σ_{tx in mempool}(tx.fee/tx.mass)^ALPHA\n", " \n", - " 'inclusion_time': The amortized time between transactions given the current \n", - " transaction masses present in the mempool, i.e., the inverse \n", - " of the transaction inclusion rate. For instance, if the average \n", - " transaction mass is 2500 grams, the block mass limit is 500,000\n", - " and the network has 10 BPS, then this number would be 1/2000 seconds.\n", + " 'inclusion_interval': The amortized time between transactions given the current \n", + " transaction masses present in the mempool, i.e., the inverse \n", + " of the transaction inclusion rate. For instance, if the average \n", + " transaction mass is 2500 grams, the block mass limit is 500,000\n", + " and the network has 10 BPS, then this number would be 1/2000 seconds.\n", " \"\"\"\n", - " def __init__(self, total_weight, inclusion_time):\n", + " def __init__(self, total_weight, inclusion_interval):\n", " self.total_weight = total_weight\n", - " self.inclusion_time = inclusion_time\n", + " self.inclusion_interval = inclusion_interval\n", "\n", " \"\"\"\n", - " Feerate to time function: f(feerate) = inclusion_time * (1/p(feerate))\n", + " Feerate to time function: f(feerate) = inclusion_interval * (1/p(feerate))\n", " where p(feerate) = feerate^ALPHA/(total_weight + feerate^ALPHA) represents \n", " the probability function for drawing `feerate` from the mempool\n", " in a single trial. The inverse 1/p is the expected number of trials until\n", - " success (with repetition), thus multiplied by inclusion_time it provides an\n", + " success (with repetition), thus multiplied by inclusion_interval it provides an\n", " approximation to the overall expected waiting time\n", " \"\"\"\n", " def feerate_to_time(self, feerate):\n", - " c1, c2 = self.inclusion_time, self.total_weight\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", " return c1 * c2 / feerate**ALPHA + c1\n", "\n", " \"\"\"\n", " The inverse function of `feerate_to_time`\n", " \"\"\"\n", " def time_to_feerate(self, time):\n", - " c1, c2 = self.inclusion_time, self.total_weight\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", " return ((c1 * c2 / time) / (1 - c1 / time))**(1 / ALPHA)\n", " \n", " \"\"\"\n", @@ -154,7 +154,7 @@ " feerate_to_time excluding the constant shift `+ c1`\n", " \"\"\"\n", " def feerate_to_time_antiderivative(self, feerate):\n", - " c1, c2 = self.inclusion_time, self.total_weight\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", " return c1 * c2 / (-2.0 * feerate**(ALPHA - 1))\n", " \n", " \"\"\"\n", @@ -162,7 +162,7 @@ " See figures below for illustration\n", " \"\"\"\n", " def quantile(self, lower, upper, frac):\n", - " c1, c2 = self.inclusion_time, self.total_weight\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", " z1 = self.feerate_to_time_antiderivative(lower)\n", " z2 = self.feerate_to_time_antiderivative(upper)\n", " z = frac * z2 + (1.0 - frac) * z1\n", @@ -194,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -214,21 +214,21 @@ "Times:\t\t168.62498827393395, 59.999999999999986, 1.0000000000000004" ] }, - "execution_count": 50, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "estimator = FeerateEstimator(total_weight=total_weight, \n", - " inclusion_time=avg_mass/network_mass_rate)\n", + " inclusion_interval=avg_mass/network_mass_rate)\n", "\n", "pred = estimator.calc_estimations()\n", "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", "y = estimator.feerate_to_time(x)\n", "plt.figure()\n", "plt.plot(x, y)\n", - "plt.fill_between(x, estimator.inclusion_time, y2=y, alpha=0.5)\n", + "plt.fill_between(x, estimator.inclusion_interval, y2=y, alpha=0.5)\n", "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", "plt.show()\n", "pred" @@ -245,7 +245,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -265,7 +265,7 @@ " array([168.62498827, 60. , 1. ]))" ] }, - "execution_count": 51, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -297,7 +297,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -317,21 +317,95 @@ "Times:\t\t2769.889957638353, 60.00000000000002, 1.0000000000000007" ] }, - "execution_count": 52, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "estimator = FeerateEstimator(total_weight=total_weight + 100**ALPHA, \n", - " inclusion_time=avg_mass/network_mass_rate)\n", + " inclusion_interval=avg_mass/network_mass_rate)\n", "\n", "pred = estimator.calc_estimations()\n", "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", "y = estimator.feerate_to_time(x)\n", "plt.figure()\n", "plt.plot(x, y)\n", - "plt.fill_between(x, estimator.inclusion_time, y2=y, alpha=0.5)\n", + "plt.fill_between(x, estimator.inclusion_interval, y2=y, alpha=0.5)\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "pred" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Outliers: solution\n", + "\n", + "Compute the estimator conditioned on the event the the top most transaction captures the first slot. This decreases `total_weight` on the one hand (thus increasing `p`), while increasing `inclusion_interval` on the other, by capturing a block slot. If this estimator gives lower prediction times we switch to it, and then repeat the process with the next highest transaction. The process convegres when the estimator is no longer improving or if all block slots are captured. " + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHh5JREFUeJzt3X2QXXWd5/H39z737ecknRC6ExMkgooo2IP4UK5jxAF1DDMrFo6r0aU2NSvjwzhbik6V7DrljFPjDuqOUhsFjbssQiErcRYHM4CFjAQITyEQICFA0klIOnTSeexO973f/eOcTm6nb3cnffve033P51V1657zO79zzvf60J+c33kyd0dEROInEXUBIiISDQWAiEhMKQBERGJKASAiElMKABGRmFIAiIjElAJARCSmJg0AM7vZzPaa2aYyy/6LmbmZzQvnzcy+b2ZbzWyjmV1c0nelmW0JPyun92eIiMiZOp0jgJ8Cl5/aaGaLgMuA7SXNVwDLws8q4Maw7xzgeuAdwCXA9WbWXknhIiJSmdRkHdz9ATNbUmbRDcBXgLtK2lYAP/Pg9uL1ZtZmZguB9wHr3L0PwMzWEYTKrRPte968eb5kSbldi4jIeB577LF97t4xWb9JA6AcM/sosNPdnzKz0kWdwI6S+Z6wbbz2CS1ZsoQNGzZMpUQRkdgys1dOp98ZB4CZ5YG/Bj5YbnGZNp+gvdz2VxEMH7F48eIzLU9ERE7TVK4Cej2wFHjKzF4GuoDHzewsgn/ZLyrp2wXsmqB9DHdf7e7d7t7d0THpEYyIiEzRGQeAuz/t7vPdfYm7LyH4436xu78KrAU+HV4NdCnQ7+67gXuAD5pZe3jy94Nhm4iIROR0LgO9FXgIOM/Meszsmgm63w1sA7YCPwI+BxCe/P0b4NHw882RE8IiIhINm8nvA+ju7nadBBYROTNm9pi7d0/WT3cCi4jElAJARCSm6jIADg0MccO6F3hyx4GoSxERmbHqMgAKRed7927h8Vf2R12KiMiMVZcB0JgN7m87PDgccSUiIjNXXQZAOpkgl04oAEREJlCXAQDBUcChAQWAiMh46jcAMikODQxFXYaIyIxVtwHQkElqCEhEZAJ1GwD5TJLDGgISERlXXQfAQQ0BiYiMq44DQCeBRUQmUscBoHMAIiITqesAODI4zEx+2qmISJTqNgAasymKDkePF6IuRURkRqrbAMhnkoAeByEiMp66DYDGTPA8IJ0IFhEpr24DQEcAIiITq9sAGHkiqB4HISJSXt0GwIkjAA0BiYiUNWkAmNnNZrbXzDaVtP2DmT1nZhvN7P+aWVvJsq+Z2VYze97M/qik/fKwbauZXTf9P2W0/Mg5AA0BiYiUdTpHAD8FLj+lbR1wgbtfCLwAfA3AzN4EXA28OVznh2aWNLMk8APgCuBNwCfCvlWTzwZHADoJLCJS3qQB4O4PAH2ntP3G3Uf+sq4HusLpFcDP3X3Q3V8CtgKXhJ+t7r7N3Y8DPw/7Vs3IVUAaAhIRKW86zgH8R+DX4XQnsKNkWU/YNl571SQTRjppHB7USWARkXIqCgAz+2tgGLhlpKlMN5+gvdw2V5nZBjPb0NvbW0l5ZFN6HpCIyHimHABmthL4CPBJP/nAnR5gUUm3LmDXBO1juPtqd+929+6Ojo6plgdAJpXgoIaARETKmlIAmNnlwFeBj7r70ZJFa4GrzSxrZkuBZcAjwKPAMjNbamYZghPFaysrfXLppOkcgIjIOFKTdTCzW4H3AfPMrAe4nuCqnyywzswA1rv7n7v7M2Z2O/AswdDQte5eCLfzF8A9QBK42d2fqcLvGSWTTHDwmM4BiIiUM2kAuPsnyjTfNEH/bwHfKtN+N3D3GVVXoWw6Sb8CQESkrLq9Exggl0ooAERExlHXAZBNB+8F1kthRETGqu8ASCUYKjgDQ8WoSxERmXHqOgByqeBxEBoGEhEZq64DIJsOfp4CQERkrPoOgJQCQERkPHUdALm0hoBERMZT1wGgIwARkfHVdQDoCEBEZHx1HQAZHQGIiIyrrgMgYUYurecBiYiUU9cBAME7ARQAIiJjxSAA9DwgEZFyFAAiIjFV9wGQSSU4cFQBICJyqroPgJzeCSAiUlbdB0A2leDggAJARORU9R8A6SSDw0UGhgpRlyIiMqPUfQDkwpvBdCmoiMhodR8ADeHjIPbrRLCIyCiTBoCZ3Wxme81sU0nbHDNbZ2Zbwu/2sN3M7PtmttXMNprZxSXrrAz7bzGzldX5OWONPA+o78jxWu1SRGRWOJ0jgJ8Cl5/Sdh1wr7svA+4N5wGuAJaFn1XAjRAEBnA98A7gEuD6kdCotobMyBGAAkBEpNSkAeDuDwB9pzSvANaE02uAK0vaf+aB9UCbmS0E/ghY5+597r4fWMfYUKmKBh0BiIiUNdVzAAvcfTdA+D0/bO8EdpT06wnbxmuvupEhoP0KABGRUab7JLCVafMJ2sduwGyVmW0wsw29vb0VF5RMGNlUgj4NAYmIjDLVANgTDu0Qfu8N23uARSX9uoBdE7SP4e6r3b3b3bs7OjqmWN5o+UxSRwAiIqeYagCsBUau5FkJ3FXS/unwaqBLgf5wiOge4INm1h6e/P1g2FYTuXSSPl0GKiIySmqyDmZ2K/A+YJ6Z9RBczfNt4HYzuwbYDlwVdr8b+BCwFTgKfBbA3fvM7G+AR8N+33T3U08sV002laDvyGCtdiciMitMGgDu/olxFi0v09eBa8fZzs3AzWdU3TRpSCd1FZCIyCnq/k5gCO4F2H9EQ0AiIqViEQC5dJJjQwU9EE5EpEQsAkA3g4mIjBWPAMgoAEREThWLADhxN7BuBhMROSEWAaAhIBGRsWIVALobWETkpFgEQDadwNARgIhIqVgEQMKMfDZJ72EFgIjIiFgEAEA+k2LfYT0OQkRkRCwC4Ly9v+ZXw3/O/9z2AbjhAth4e9QliYhEbtJnAc125+39NZe9+LekfSBo6N8Bv/pCMH3hx6MrTEQkYnV/BPCe7T8kXRwY3Th0DO79ZjQFiYjMEHUfAM2De8ov6O+pbSEiIjNM3QfAoeyC8gtau2pbiIjIDFP3AfDg4s8xlMiNbkw3wPJvRFOQiMgMUfcngZ+ffwUA73z5B7Qe38tAfiH5K/6bTgCLSOzV/REABCFw49vu4pzBW7jtPXfrj7+ICDEJAIBcOkHCoPeQbgYTEYEYBYCZ0ZTV3cAiIiMqCgAz+0sze8bMNpnZrWaWM7OlZvawmW0xs9vMLBP2zYbzW8PlS6bjB5yJhkxSRwAiIqEpB4CZdQJfALrd/QIgCVwN/D1wg7svA/YD14SrXAPsd/dzgRvCfjXVkE6yVwEgIgJUPgSUAhrMLAXkgd3A+4E7wuVrgCvD6RXhPOHy5WZmFe7/jOQzKQWAiEhoygHg7juB7wDbCf7w9wOPAQfcfTjs1gN0htOdwI5w3eGw/9yp7n8qmrIp+g4fZ7hQrOVuRURmpEqGgNoJ/lW/FDgbaASuKNPVR1aZYFnpdleZ2QYz29Db2zvV8spqzCYpuLNP7wUQEaloCOgDwEvu3uvuQ8CdwLuAtnBICKAL2BVO9wCLAMLlrUDfqRt199Xu3u3u3R0dHRWUN1ZTLihrd/+xad2uiMhsVEkAbAcuNbN8OJa/HHgWuB/4WNhnJXBXOL02nCdcfp+7jzkCqKbmbBqAV/sHJukpIlL/KjkH8DDBydzHgafDba0Gvgp82cy2Eozx3xSuchMwN2z/MnBdBXVPSVN25AhAASAiUtGzgNz9euD6U5q3AZeU6TsAXFXJ/iqVSydIJYw9BxUAIiKxuRMYgruBm3MpHQGIiBCzAABozKZ0ElhEhNgGgI4ARERiFwBN2RR7Dg5QLNb0AiQRkRknlgEwVHD6jupmMBGJt1gGAOheABGR+AVATvcCiIhADAOgOTwC2HVAVwKJSLzFLgDymSSphNGz/2jUpYiIRCp2AWBmtDak2dGnIwARibfYBQAE5wG29+kIQETiLZYB0JJLs0NDQCISc7EMgNaGNIcGhuk/NhR1KSIikYllALSEl4LqRLCIxFk8A6AheDGMTgSLSJzFOgB0BCAicRbLAMilEmRTCXr26whAROIrlgFgZsGVQLoUVERiLJYBANCsewFEJOZiGwCt+TTb+47qvQAiElsVBYCZtZnZHWb2nJltNrN3mtkcM1tnZlvC7/awr5nZ981sq5ltNLOLp+cnTE17Q4bB4SK79YJ4EYmpSo8Avgf8i7ufD7wV2AxcB9zr7suAe8N5gCuAZeFnFXBjhfuuSFs+uBLopd4jUZYhIhKZKQeAmbUA7wVuAnD34+5+AFgBrAm7rQGuDKdXAD/zwHqgzcwWTrnyCrU3ZgB4ad/hqEoQEYlUJUcA5wC9wE/M7Akz+7GZNQIL3H03QPg9P+zfCewoWb8nbItEYyZJJpngRR0BiEhMVRIAKeBi4EZ3vwg4wsnhnnKsTNuYM7BmtsrMNpjZht7e3grKm5iZ0d6Y5qV9CgARiadKAqAH6HH3h8P5OwgCYc/I0E74vbek/6KS9buAXadu1N1Xu3u3u3d3dHRUUN7kWnNptvVqCEhE4mnKAeDurwI7zOy8sGk58CywFlgZtq0E7gqn1wKfDq8GuhToHxkqikpbY4adB44xOFyIsgwRkUikKlz/88AtZpYBtgGfJQiV283sGmA7cFXY927gQ8BW4GjYN1Lt+TRFhx19Rzl3fnPU5YiI1FRFAeDuTwLdZRYtL9PXgWsr2d90a8sHVwK92HtEASAisRPbO4EB5oQBsHWvzgOISPzEOgAyqQStDWmee/VQ1KWIiNRcrAMAYE5jhud2H4y6DBGRmot9AMxtzLBt3xGODxejLkVEpKYUAE0ZCkVnmx4JISIxE/sAmNeUBeB5nQcQkZiJfQC05zMkTAEgIvET+wBIJow5jRkFgIjETuwDAIIrgZ7VlUAiEjMKAKCjKcvu/gH2HzkedSkiIjWjAADmt+QAeHpnf8SViIjUjgIAmN8cXAmkABCROFEAALl0kvZ8mqd7FAAiEh8KgFBHc5aneg5EXYaISM0oAEILmnPs7h9g3+HBqEsREakJBUBofovOA4hIvCgAQh3NWQx4aoeGgUQkHhQAoWwqybymDI+9sj/qUkREakIBUGJhawOPvbKf4YIeDS0i9U8BUOLstgaOHi/oDWEiEgsVB4CZJc3sCTP753B+qZk9bGZbzOw2M8uE7dlwfmu4fEml+55uZ7cFdwQ/+nJfxJWIiFTfdBwBfBHYXDL/98AN7r4M2A9cE7ZfA+x393OBG8J+M0pzLk1rQ5oNL+s8gIjUv4oCwMy6gA8DPw7nDXg/cEfYZQ1wZTi9IpwnXL487D+jnNWS45GX+nD3qEsREamqSo8Avgt8BRg5azoXOODuw+F8D9AZTncCOwDC5f1h/xnl7LYcvYcHefm1o1GXIiJSVVMOADP7CLDX3R8rbS7T1U9jWel2V5nZBjPb0NvbO9XypmzRnDwAD26p/b5FRGqpkiOAdwMfNbOXgZ8TDP18F2gzs1TYpwvYFU73AIsAwuWtwJizre6+2t273b27o6OjgvKmpq0hOA/wwJZ9Nd+3iEgtTTkA3P1r7t7l7kuAq4H73P2TwP3Ax8JuK4G7wum14Tzh8vt8Bg60mxmL2ht46MXXGNL9ACJSx6pxH8BXgS+b2VaCMf6bwvabgLlh+5eB66qw72mxeE6ew4PDeiyEiNS11ORdJufuvwV+G05vAy4p02cAuGo69ldti+bkMYMHtuyje8mcqMsREakK3QlcRi6d5KyWHL99fm/UpYiIVI0CYBxL5jaysaefV/sHoi5FRKQqFADjeH1HIwDrNu+JuBIRkepQAIxjTmOGOfk092x6NepSRESqQgEwDjNjaUcTD217jf5jQ1GXIyIy7RQAE3h9RyOFonPfcxoGEpH6owCYwFktOVpyKe56YtfknUVEZhkFwATMjDcsaOZ3W/bRe2gw6nJERKaVAmAS55/VTMGdXz2lowARqS8KgEnMbcqyoCXLnY/3RF2KiMi0UgCchjcsaGbTroM8r3cFi0gdUQCchjee1UIqYfyv9S9HXYqIyLRRAJyGhkySZfObuPPxnRwa0D0BIlIfFACn6cKuNo4eL3Dn4zujLkVEZFooAE7TWa05zmrJseb3L1Mszrj32IiInDEFwBl466JWtu07ogfEiUhdUACcgTfMb6Ytn+af7tvKDHybpYjIGVEAnIFEwnj74nae3tnP7/TSeBGZ5RQAZ+j8hc0051J8919f0FGAiMxqCoAzlEok+IMlc3h8+wHueUbnAkRk9ppyAJjZIjO738w2m9kzZvbFsH2Oma0zsy3hd3vYbmb2fTPbamYbzezi6foRtfbmhS3Mbczwd7/ezFChGHU5IiJTUskRwDDwV+7+RuBS4FozexNwHXCvuy8D7g3nAa4AloWfVcCNFew7UomE8a5z5/LKa0e5Zf0rUZcjIjIlUw4Ad9/t7o+H04eAzUAnsAJYE3ZbA1wZTq8AfuaB9UCbmS2ccuURWzq3kcVz8nznNy+w56BeHC8is8+0nAMwsyXARcDDwAJ33w1BSADzw26dwI6S1XrCtlnJzPjD8zoYGCrwX9c+E3U5IiJnrOIAMLMm4BfAl9z94ERdy7SNuYzGzFaZ2QYz29Db21tpeVXVls9wydI5/HrTq9zzjF4eLyKzS0UBYGZpgj/+t7j7nWHznpGhnfB7b9jeAywqWb0LGPOWFXdf7e7d7t7d0dFRSXk1cfHidjqas1z3i43s1VCQiMwilVwFZMBNwGZ3/8eSRWuBleH0SuCukvZPh1cDXQr0jwwVzWbJhHH5m8/i8OAwX7rtST0nSERmjUqOAN4NfAp4v5k9GX4+BHwbuMzMtgCXhfMAdwPbgK3Aj4DPVbDvGWVOY4b3Luvg9y++xg/u3xp1OSIipyU11RXd/UHKj+sDLC/T34Frp7q/me7NZ7ew88Ax/vu6F1i2oInLL5i1FziJSEzoTuBpYmYsP38+C1tzfOm2J9m0sz/qkkREJqQAmEapZIIPv2UhmVSCT9/8CC/2Ho66JBGRcSkAplljNsWVb+1kcKjAn/1oPdtfOxp1SSIiZSkAqqC9McOVF3Vy8NgwH1/9EFv3Hoq6JBGRMRQAVTKvKcufXNTJoYEh/v2ND/HYK/ujLklEZBQFQBV1NGe56u2LSBj82Y/W88sn9EJ5EZk5FABV1tqQ5mNv72JeU5Yv3fYk19+1iePDeoS0iERPAVAD+UyKP7mok4sWt7HmoVf40x/+G8+/qvMCIhItBUCNJBPGe5d18OG3LGTbviN85H/8jh/cv1UvlBGRyCgAauzc+U188h2LWTK3kX+453ku/+4D/Pb5vZOvKCIyzRQAEchnUnzoLQv54wsX0nfkOJ/5yaN85uZH2NhzIOrSRCRGpvwsIKncOR1NLJ6b56kd/ax/6TU++k//xh+e18Hnly/j4sXtUZcnInVOARCxVCLB21/XzgWdLTzV08/DL/Vx/w9/z1u7WvnUO5fwkQsXkksng84bb4d7vwn9PdDaBcu/ARd+PNofICKzlgUP6ZyZuru7fcOGDVNa94U9h/h/G2ff6waODxd5dvdBNu3s57Ujx2ltSLPibWfzmeZHWPrQ17GhYyc7pxvgj7+vEBCRUczsMXfvnqyfzgHMMJlUgrctauOT71jMn17UyfyWLP/n4e1kfvut0X/8AYaOBUcEIiJToCGgGcrMWDQnz6I5eQaHC3Q+/FrZft7fw64Dx+hsa6hxhSIy2ykAZoFsKsmh7AJaBse+eH5ncS7v+fZ9LGzN8Y6lc/iDpXO4eHE7585vIp3UAZ6IjE8BMEs8uPhzXPbi35Iunnzx/FAix4OL/jP/zjrYeeAY/7p5L798chcAmWSC885q5oLOVt58dgtvXNjCuR1NtObTUf0EEZlhFACzxPPzrwDgPdt/SPPgHg5lF/Dg4s+xe/4VvA1426I23J0Dx4bYc3CA3kOD9B4e5JdP7OTWR7af2E57Ps2585t4fUfwed3cPJ3tDXS15WlpSGE23ls+RaTeKABmkefnX3EiCMoxM9rzGdrzGc4/K2hzdw4NDLPv8CD7jw6x/+hxdh8Y4NldBzlyvDBq/XwmSWdbA13tDZzd1sDC1hzzmrJ0NAefeU3BJ5PS0JJIPah5AJjZ5cD3gCTwY3f/dq1riBMzo6UhTUvD2KGfgaEC/ceGODgwxKGBYQ4dG+bQ4BDP7j7I+m19HBsqlNkitORSdDRnmduUpbUhTVtDmtbw05YP9tWWz5xoa86laMykyKUTOsIQmUFqGgBmlgR+AFwG9ACPmtlad3+2lnVIIJdOkksnWdCSK7t8uFDk6PFC+Bnm6PECR8Lvo4MFdu4/xkv7jjA4VODYUIGhwsT3lCQMGjJJGjMpGrMpGrNJmrJBOOSzKZqySfKZFA3pJNlUglw6STadIJcKvrOp0fMnvkv6p5NGOpEgkVDQyCwS0U2etT4CuATY6u7bAMzs58AKQAEwA6WSCVoaEmWPHsopFJ2BoQKDw8VR38cLRYaGiwwVPJguFDk+XOTwwDD7jwwxXAzmhwoefhep9PbEpBmpZPBJJxKkk4lgOpkIQiKZKDudKmlLJRIkE5Ac+TYjkTBSieA7aUYyEX7CZcmR5eGykf4nl4fbs2A6EdaZsJMfMzALlhmQSBhBngXfI31O9MVIJE72Nwv6jXwnwqOuke0YJ5ePbGek3Uq2U66WkSO4YDr4z1pHdRXaeDv86gvBfT0A/TuCeah6CNQ6ADqBHSXzPcA7alyDVEkyYeG/7CvbjrtTdBguFikUneGCB99FZ7hYHDtfdAqFYL7gTrHoFN0pFhk1XyhpOz5c5NhQ4cS+gj5h/xPrQNEdH+d7ZHrm3ktfWyMxMBJKIw1W2kZpcIwsHwmykxsaaTu5vo3a/omuJ4Lp5LZKdn1iv3ZqLSUbObW2U4OOMstL1z1Zto1tm2w5cPOBrzO/OM5NnnUWAOX+qTDq/z9mtgpYBbB48eIp7+iceY38p/eeM+X1RU5XsSR4ToRMsUjBTy4rFINPsWS6NJAKRcdLQ4aTQXgyeIL2oK00iABGB9aJdUdtx8esGwTYqe3BMji5/ZFlwZ5Kpwmngwb30rbRfQlrGb08mPETy8dun7CtdPulfRlVy6nbOrkzH7O+U1L6if2Wrs+Y33ryx5T+4Sr3RJ1yfUf9ZxF+d/TtG7syBMNBVVbrAOgBFpXMdwG7Sju4+2pgNQTPAprqjlLJBE26EUpEZrobuoJhn1O1dlV917X+C/kosMzMlppZBrgaWFvjGkREZo7l3wge7Fgq3RC0V1lNjwDcfdjM/gK4h+Ay0Jvd/Zla1iAiMqOMjPPH4Cog3P1u4O5a71dEZMa68OORPNZdg+QiIjGlABARiSkFgIhITCkARERiSgEgIhJTCgARkZhSAIiIxJQCQEQkpszLPcVohjCzXuCVqOuYgnnAOE94qjtx+q2g31vP6um3vs7dOybrNKMDYLYysw3u3h11HbUQp98K+r31LE6/dYSGgEREYkoBICISUwqA6lgddQE1FKffCvq99SxOvxXQOQARkdjSEYCISEwpAKaJmS0ys/vNbLOZPWNmX4y6pmozs6SZPWFm/xx1LdVmZm1mdoeZPRf+d/zOqGuqJjP7y/B/x5vM7FYzy0Vd03Qys5vNbK+ZbSppm2Nm68xsS/jdHmWNtaAAmD7DwF+5+xuBS4FrzexNEddUbV8ENkddRI18D/gXdz8feCt1/LvNrBP4AtDt7hcQvL3v6mirmnY/BS4/pe064F53XwbcG87XNQXANHH33e7+eDh9iOAPRGe0VVWPmXUBHwZ+HHUt1WZmLcB7gZsA3P24ux+ItqqqSwENZpYC8sCuiOuZVu7+ANB3SvMKYE04vQa4sqZFRUABUAVmtgS4CHg42kqq6rvAV4Bi1IXUwDlAL/CTcMjrx2bWGHVR1eLuO4HvANuB3UC/u/8m2qpqYoG774bgH3TA/IjrqToFwDQzsybgF8CX3P1g1PVUg5l9BNjr7o9FXUuNpICLgRvd/SLgCHU8PBCOfa8AlgJnA41m9h+irUqqQQEwjcwsTfDH/xZ3vzPqeqro3cBHzexl4OfA+83sf0dbUlX1AD3uPnJEdwdBINSrDwAvuXuvuw8BdwLvirimWthjZgsBwu+9EddTdQqAaWJmRjBGvNnd/zHqeqrJ3b/m7l3uvoTg5OB97l63/0J091eBHWZ2Xti0HHg2wpKqbTtwqZnlw/9dL6eOT3qXWAusDKdXAndFWEtNpKIuoI68G/gU8LSZPRm2fd3d746wJpk+nwduMbMMsA34bMT1VI27P2xmdwCPE1zd9gR1dpesmd0KvA+YZ2Y9wPXAt4HbzewaghC8KroKa0N3AouIxJSGgEREYkoBICISUwoAEZGYUgCIiMSUAkBEJKYUACIiMaUAEBGJKQWAiEhM/X8k2vFG5y1P8wAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Feerates:\t1.1531420689155165, 2.816548045571761, 11.10120050773006 \n", + "Times:\t\t874.010579873836, 60.00000000000001, 1.0000000000000004" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def build_estimator():\n", + " _feerates = [1.0]*10 + [1.1]*10 + [1.2]*10 + [1.5]*3000 + [2]*3000\\\n", + "+ [2.1]*3000 + [3]*10 + [4]*10 + [5]*10 + [6] + [7] + [10] + [100] + [200]*200\n", + " _total_weight = sum(np.array(_feerates)**ALPHA)\n", + " _network_mass_rate = bps * block_mass_limit\n", + " estimator = FeerateEstimator(total_weight=_total_weight, \n", + " inclusion_interval=avg_mass/_network_mass_rate)\n", + " \n", + " nr = _network_mass_rate\n", + " for i in range(len(_feerates)-1, -1, -1):\n", + " tw = sum(np.array(_feerates[:i])**ALPHA)\n", + " nr -= avg_mass\n", + " if nr <= 0:\n", + " print(\"net mass rate {}\", nr)\n", + " break\n", + " e = FeerateEstimator(total_weight=tw, \n", + " inclusion_interval=avg_mass/nr)\n", + " if e.feerate_to_time(1.0) < estimator.feerate_to_time(1.0):\n", + " # print(\"removing {}\".format(_feerates[i]))\n", + " estimator = e\n", + " else:\n", + " break\n", + " \n", + " return estimator\n", + "\n", + "estimator = build_estimator()\n", + "pred = estimator.calc_estimations()\n", + "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", + "y = estimator.feerate_to_time(x)\n", + "plt.figure()\n", + "plt.plot(x, y)\n", + "plt.fill_between(x, estimator.inclusion_interval, y2=y, alpha=0.5)\n", "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", "plt.show()\n", "pred" From e2024f9654358355aa60737bad0da805be2d3573 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 14:10:00 +0000 Subject: [PATCH 45/74] fee estimator todos --- mining/src/feerate/mod.rs | 60 +++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index f90999e80..2b74fc834 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -1,4 +1,12 @@ +//! See the accompanying fee_estimation.ipynb Jupyter Notebook which details the reasoning +//! behind this fee estimator. + use crate::block_template::selector::ALPHA; +use kaspa_utils::vec::VecExtensions; + +/// The current standard minimum feerate (fee/mass = 1.0 is the current standard minimum). +/// TODO: pass from config +const MIN_FEERATE: f64 = 1.0; #[derive(Clone, Copy, Debug)] pub struct FeerateBucket { @@ -8,9 +16,25 @@ pub struct FeerateBucket { #[derive(Clone, Debug)] pub struct FeerateEstimations { - pub priority_bucket: FeerateBucket, // TODO (PR): priority = sub-second and at least 1.0 feerate (standard) - pub normal_bucket: FeerateBucket, // TODO (PR): change to a vec with sub-minute guarantee for first value - pub low_bucket: FeerateBucket, // TODO (PR): change to a vec with sub-hour guarantee for first value + /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + pub priority_bucket: FeerateBucket, + + /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to + /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + /// between them, once can compose a complete feerate function on the client side. The API makes an effort + /// to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + pub normal_buckets: Vec, + + /// A vector of *low* priority feerate values. The first value of this vector is guaranteed to + /// provide an estimation for sub-*hour* DAG inclusion. + pub low_buckets: Vec, +} + +impl FeerateEstimations { + pub fn ordered_buckets(&self) -> Vec { + vec![self.priority_bucket].merge(self.normal_buckets.clone()).merge(self.low_buckets.clone()) + } } pub struct FeerateEstimatorArgs { @@ -65,13 +89,13 @@ impl FeerateEstimator { } pub fn calc_estimations(&self) -> FeerateEstimations { - let high = self.time_to_feerate(1f64); - let low = self.time_to_feerate(3600f64).max(self.quantile(1f64, high, 0.25)); - let mid = self.time_to_feerate(60f64).max(self.quantile(low, high, 0.5)); + let high = self.time_to_feerate(1f64).max(MIN_FEERATE); + let low = self.time_to_feerate(3600f64).max(MIN_FEERATE).max(self.quantile(1f64, high, 0.25)); + let mid = self.time_to_feerate(60f64).max(MIN_FEERATE).max(self.quantile(low, high, 0.5)); FeerateEstimations { priority_bucket: FeerateBucket { feerate: high, estimated_seconds: self.feerate_to_time(high) }, - normal_bucket: FeerateBucket { feerate: mid, estimated_seconds: self.feerate_to_time(mid) }, - low_bucket: FeerateBucket { feerate: low, estimated_seconds: self.feerate_to_time(low) }, + normal_buckets: vec![FeerateBucket { feerate: mid, estimated_seconds: self.feerate_to_time(mid) }], + low_buckets: vec![FeerateBucket { feerate: low, estimated_seconds: self.feerate_to_time(low) }], } } } @@ -79,15 +103,27 @@ impl FeerateEstimator { #[cfg(test)] mod tests { use super::*; + use itertools::Itertools; #[test] fn test_feerate_estimations() { let estimator = FeerateEstimator { total_weight: 1002283.659, inclusion_interval: 0.004f64 }; let estimations = estimator.calc_estimations(); - assert!(estimations.low_bucket.feerate <= estimations.normal_bucket.feerate); - assert!(estimations.normal_bucket.feerate <= estimations.priority_bucket.feerate); - assert!(estimations.low_bucket.estimated_seconds >= estimations.normal_bucket.estimated_seconds); - assert!(estimations.normal_bucket.estimated_seconds >= estimations.priority_bucket.estimated_seconds); + let buckets = estimations.ordered_buckets(); + for (i, j) in buckets.into_iter().tuple_windows() { + assert!(i.feerate >= j.feerate); + } dbg!(estimations); } + + #[test] + fn test_min_feerate_estimations() { + let estimator = FeerateEstimator { total_weight: 0.00659, inclusion_interval: 0.004f64 }; + let estimations = estimator.calc_estimations(); + let buckets = estimations.ordered_buckets(); + assert!(buckets.last().unwrap().feerate >= MIN_FEERATE); + for (i, j) in buckets.into_iter().tuple_windows() { + assert!(i.feerate >= j.feerate); + } + } } From 25c23337a87e5f2ecd9ea918ac80f529d3df5922 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 5 Aug 2024 14:47:50 +0000 Subject: [PATCH 46/74] expose get_realtime_feerate_estimations from the mining manager --- mining/src/feerate/mod.rs | 4 ++++ mining/src/manager.rs | 13 +++++++++++++ mining/src/mempool/config.rs | 4 ++++ mining/src/mempool/mod.rs | 6 ++++++ mining/src/mempool/model/frontier.rs | 15 +++++++++++++++ mining/src/mempool/model/transactions_pool.rs | 6 ++++++ 6 files changed, 48 insertions(+) diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index 2b74fc834..2337658f7 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -43,6 +43,10 @@ pub struct FeerateEstimatorArgs { } impl FeerateEstimatorArgs { + pub fn new(network_blocks_per_second: u64, maximum_mass_per_block: u64) -> Self { + Self { network_blocks_per_second, maximum_mass_per_block } + } + pub fn network_mass_per_second(&self) -> u64 { self.network_blocks_per_second * self.maximum_mass_per_block } diff --git a/mining/src/manager.rs b/mining/src/manager.rs index c8f0798a1..28b854d5c 100644 --- a/mining/src/manager.rs +++ b/mining/src/manager.rs @@ -2,6 +2,7 @@ use crate::{ block_template::{builder::BlockTemplateBuilder, errors::BuilderError}, cache::BlockTemplateCache, errors::MiningManagerResult, + feerate::{FeerateEstimations, FeerateEstimatorArgs}, mempool::{ config::Config, model::tx::{MempoolTransaction, TransactionPostValidation, TransactionPreValidation, TxRemovalReason}, @@ -201,6 +202,13 @@ impl MiningManager { self.mempool.read().build_selector() } + /// Returns realtime feerate estimations based on internal mempool state + pub(crate) fn get_realtime_feerate_estimations(&self) -> FeerateEstimations { + let args = FeerateEstimatorArgs::new(self.config.network_blocks_per_second, self.config.maximum_mass_per_block); + let estimator = self.mempool.read().build_feerate_estimator(args); + estimator.calc_estimations() + } + /// Clears the block template cache, forcing the next call to get_block_template to build a new block template. #[cfg(test)] pub(crate) fn clear_block_template(&self) { @@ -799,6 +807,11 @@ impl MiningManagerProxy { consensus.clone().spawn_blocking(move |c| self.inner.get_block_template(c, &miner_data)).await } + /// Returns realtime feerate estimations based on internal mempool state + pub async fn get_realtime_feerate_estimations(self) -> FeerateEstimations { + spawn_blocking(move || self.inner.get_realtime_feerate_estimations()).await.unwrap() + } + /// Validates a transaction and adds it to the set of known transactions that have not yet been /// added to any block. /// diff --git a/mining/src/mempool/config.rs b/mining/src/mempool/config.rs index 06945b35a..d0f94980d 100644 --- a/mining/src/mempool/config.rs +++ b/mining/src/mempool/config.rs @@ -45,6 +45,7 @@ pub struct Config { pub minimum_relay_transaction_fee: u64, pub minimum_standard_transaction_version: u16, pub maximum_standard_transaction_version: u16, + pub network_blocks_per_second: u64, } impl Config { @@ -67,6 +68,7 @@ impl Config { minimum_relay_transaction_fee: u64, minimum_standard_transaction_version: u16, maximum_standard_transaction_version: u16, + network_blocks_per_second: u64, ) -> Self { Self { maximum_transaction_count, @@ -86,6 +88,7 @@ impl Config { minimum_relay_transaction_fee, minimum_standard_transaction_version, maximum_standard_transaction_version, + network_blocks_per_second, } } @@ -113,6 +116,7 @@ impl Config { minimum_relay_transaction_fee: DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, minimum_standard_transaction_version: DEFAULT_MINIMUM_STANDARD_TRANSACTION_VERSION, maximum_standard_transaction_version: DEFAULT_MAXIMUM_STANDARD_TRANSACTION_VERSION, + network_blocks_per_second: 1000 / target_milliseconds_per_block, } } diff --git a/mining/src/mempool/mod.rs b/mining/src/mempool/mod.rs index 30e1dfc69..c634a4a5b 100644 --- a/mining/src/mempool/mod.rs +++ b/mining/src/mempool/mod.rs @@ -1,4 +1,5 @@ use crate::{ + feerate::{FeerateEstimator, FeerateEstimatorArgs}, model::{ owner_txs::{GroupedOwnerTransactions, ScriptPublicKeySet}, tx_query::TransactionQuery, @@ -120,6 +121,11 @@ impl Mempool { self.transaction_pool.build_selector() } + /// Builds a feerate estimator based on internal state of the ready transactions frontier + pub(crate) fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { + self.transaction_pool.build_feerate_estimator(args) + } + pub(crate) fn all_transaction_ids_with_priority(&self, priority: Priority) -> Vec { let _sw = Stopwatch::<15>::with_threshold("all_transaction_ids_with_priority op"); self.transaction_pool.all_transaction_ids_with_priority(priority) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index ed572868e..4f4163c24 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -126,6 +126,20 @@ impl Frontier { sequence } + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier. + /// + /// The logic is divided into three cases: + /// 1. The frontier is small and can fit entirely into a block: perform no sampling and return + /// a TakeAllSelector + /// 2. The frontier has at least ~4x the capacity of a block: expected collision rate is low, perform + /// in-place k*log(n) sampling and return a SequenceSelector + /// 3. The frontier has 1-4x capacity of a block. In this case we expect a high collision rate while + /// the number of overall transactions is still low, so we take all of the transactions and use the + /// rebalancing weighted selector (performing the actual sampling out of the mempool lock) + /// + /// The above thresholds were selected based on benchmarks. Overall, this dynamic selection provides + /// full transaction selection in less than 150 µs even if the frontier has 1M entries (!!). See mining/benches + /// for more details. pub fn build_selector(&self, policy: &Policy) -> Box { if self.total_mass <= policy.max_block_mass { Box::new(TakeAllSelector::new(self.search_tree.ascending_iter().map(|k| k.tx.clone()).collect())) @@ -160,6 +174,7 @@ impl Frontier { )) } + /// Builds a feerate estimator based on internal state of the ready transactions frontier pub fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { let average_transaction_mass = match self.len() { 0 => TYPICAL_TX_MASS, diff --git a/mining/src/mempool/model/transactions_pool.rs b/mining/src/mempool/model/transactions_pool.rs index 0d487c22a..1b6a3e532 100644 --- a/mining/src/mempool/model/transactions_pool.rs +++ b/mining/src/mempool/model/transactions_pool.rs @@ -1,4 +1,5 @@ use crate::{ + feerate::{FeerateEstimator, FeerateEstimatorArgs}, mempool::{ config::Config, errors::{RuleError, RuleResult}, @@ -171,6 +172,11 @@ impl TransactionsPool { self.ready_transactions.build_selector(&Policy::new(self.config.maximum_mass_per_block)) } + /// Builds a feerate estimator based on internal state of the ready transactions frontier + pub(crate) fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { + self.ready_transactions.build_feerate_estimator(args) + } + /// Is the mempool transaction identified by `transaction_id` unchained, thus having no successor? pub(crate) fn transaction_is_unchained(&self, transaction_id: &TransactionId) -> bool { if self.all_transactions.contains_key(transaction_id) { From 8e27b402693bbb786937d565f4acd8f13f7d15de Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Tue, 6 Aug 2024 08:31:35 +0000 Subject: [PATCH 47/74] min feerate from config --- mining/src/feerate/mod.rs | 26 +++++++++++++------------- mining/src/manager.rs | 2 +- mining/src/mempool/config.rs | 6 ++++++ mining/src/mempool/model/frontier.rs | 2 +- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index 2337658f7..57078cabc 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -2,11 +2,6 @@ //! behind this fee estimator. use crate::block_template::selector::ALPHA; -use kaspa_utils::vec::VecExtensions; - -/// The current standard minimum feerate (fee/mass = 1.0 is the current standard minimum). -/// TODO: pass from config -const MIN_FEERATE: f64 = 1.0; #[derive(Clone, Copy, Debug)] pub struct FeerateBucket { @@ -33,7 +28,10 @@ pub struct FeerateEstimations { impl FeerateEstimations { pub fn ordered_buckets(&self) -> Vec { - vec![self.priority_bucket].merge(self.normal_buckets.clone()).merge(self.low_buckets.clone()) + std::iter::once(self.priority_bucket) + .chain(self.normal_buckets.iter().copied()) + .chain(self.low_buckets.iter().copied()) + .collect() } } @@ -92,10 +90,11 @@ impl FeerateEstimator { ((c1 * c2) / (-2f64 * z)).powf(1f64 / (ALPHA - 1) as f64) } - pub fn calc_estimations(&self) -> FeerateEstimations { - let high = self.time_to_feerate(1f64).max(MIN_FEERATE); - let low = self.time_to_feerate(3600f64).max(MIN_FEERATE).max(self.quantile(1f64, high, 0.25)); - let mid = self.time_to_feerate(60f64).max(MIN_FEERATE).max(self.quantile(low, high, 0.5)); + pub fn calc_estimations(&self, minimum_standard_feerate: f64) -> FeerateEstimations { + let min = minimum_standard_feerate; + let high = self.time_to_feerate(1f64).max(min); + let low = self.time_to_feerate(3600f64).max(self.quantile(min, high, 0.25)); + let mid = self.time_to_feerate(60f64).max(self.quantile(low, high, 0.5)); FeerateEstimations { priority_bucket: FeerateBucket { feerate: high, estimated_seconds: self.feerate_to_time(high) }, normal_buckets: vec![FeerateBucket { feerate: mid, estimated_seconds: self.feerate_to_time(mid) }], @@ -112,7 +111,7 @@ mod tests { #[test] fn test_feerate_estimations() { let estimator = FeerateEstimator { total_weight: 1002283.659, inclusion_interval: 0.004f64 }; - let estimations = estimator.calc_estimations(); + let estimations = estimator.calc_estimations(1.0); let buckets = estimations.ordered_buckets(); for (i, j) in buckets.into_iter().tuple_windows() { assert!(i.feerate >= j.feerate); @@ -123,9 +122,10 @@ mod tests { #[test] fn test_min_feerate_estimations() { let estimator = FeerateEstimator { total_weight: 0.00659, inclusion_interval: 0.004f64 }; - let estimations = estimator.calc_estimations(); + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); let buckets = estimations.ordered_buckets(); - assert!(buckets.last().unwrap().feerate >= MIN_FEERATE); + assert!(buckets.last().unwrap().feerate >= minimum_feerate); for (i, j) in buckets.into_iter().tuple_windows() { assert!(i.feerate >= j.feerate); } diff --git a/mining/src/manager.rs b/mining/src/manager.rs index 28b854d5c..3818b56f3 100644 --- a/mining/src/manager.rs +++ b/mining/src/manager.rs @@ -206,7 +206,7 @@ impl MiningManager { pub(crate) fn get_realtime_feerate_estimations(&self) -> FeerateEstimations { let args = FeerateEstimatorArgs::new(self.config.network_blocks_per_second, self.config.maximum_mass_per_block); let estimator = self.mempool.read().build_feerate_estimator(args); - estimator.calc_estimations() + estimator.calc_estimations(self.config.minimum_feerate()) } /// Clears the block template cache, forcing the next call to get_block_template to build a new block template. diff --git a/mining/src/mempool/config.rs b/mining/src/mempool/config.rs index d0f94980d..419a4362a 100644 --- a/mining/src/mempool/config.rs +++ b/mining/src/mempool/config.rs @@ -124,4 +124,10 @@ impl Config { self.maximum_transaction_count = (self.maximum_transaction_count as f64 * ram_scale.min(1.0)) as u32; // Allow only scaling down self } + + /// Returns the minimum standard fee/mass ratio currently required by the mempool + pub(crate) fn minimum_feerate(&self) -> f64 { + // The parameter minimum_relay_transaction_fee is in sompi/kg units so divide by 1000 to get sompi/gram + self.minimum_relay_transaction_fee as f64 / 1000.0 + } } diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 4f4163c24..e9262f49e 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -312,7 +312,7 @@ mod tests { let args = FeerateEstimatorArgs { network_blocks_per_second: 1, maximum_mass_per_block: 500_000 }; // We are testing that the build function actually returns and is not looping indefinitely let estimator = frontier.build_feerate_estimator(args); - let _estimations = estimator.calc_estimations(); + let _estimations = estimator.calc_estimations(1.0); // dbg!(_estimations); } } From 8aec49b87013cd450014728d4cf5f550f29181b9 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Tue, 6 Aug 2024 09:44:37 +0000 Subject: [PATCH 48/74] sample_inplace doc --- mining/src/mempool/model/frontier.rs | 32 +++++++++++++++++-- .../src/mempool/model/frontier/search_tree.rs | 2 +- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index e9262f49e..ca2f88748 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -78,6 +78,32 @@ impl Frontier { } } + /// Samples the frontier in-place based on the provided policy and returns a SequenceSelector. + /// + /// This sampling algorithm should be used when frontier total mass is high enough compared to + /// policy mass limit so that the probability of sampling collisions remains low. + /// + /// Convergence analysis: + /// 1. Based on the above we can safely assume that `k << n`, where `n` is the total number of frontier items + /// and `k` is the number of actual samples (since `desired_mass << total_mass` and mass per item is bounded) + /// 2. Indeed, if the weight distribution is not too spread (i.e., `max(weights) = O(min(weights))`), `k << n` means + /// that the probability of collisions is low enough and the sampling process will converge in `O(k log(n))` w.h.p. + /// 3. It remains to deal with the case where the weight distribution is highly biased. The process implemented below + /// keeps track of the top-weight element. If the distribution is highly biased, this element will be sampled twice + /// with sufficient probability (in constant time), in which case we narrow the sampling space to exclude it. We do + /// this by computing the prefix weight up to this top item (exclusive) and then continue the sampling process over + /// the narrowed space. This process is repeated until acquiring the desired mass. + /// 4. Numerical stability. Naively, one would simply subtract `total_weight -= top.weight` in order to narrow the sampling + /// space. However, if `top.weight` is much larger than the remaining weight, the above f64 subtraction will yield a number + /// close or equal to zero. We fix this by implementing a `log(n)` prefix weight operation. + /// 5. Q. Why not just use u64 weights? + /// A. The current weight calculation is `feerate^alpha` with `alpha=3`. Using u64 would mean that the feerate space + /// is limited to `(2^64)^(1/3) = ~2^21 = ~2M` possibilities. Already with current usages, the feerate can vary from `~1/50` (2000 sompi + /// for a transaction with 100K storage mass), to `5M` (100 KAS fee for a transaction with 2000 mass = 100·100_000_000/2000), + /// resulting in a range of 250M points (`5M/(1/50)`). + /// By using floating point arithmetics we gain the adjustment of the probability space to the accuracy level required for + /// current samples. And if the space is highly biased, the repeated elimination of top items and the prefix weight computation + /// will readjust it. pub fn sample_inplace(&self, rng: &mut R, policy: &Policy) -> SequenceSelectorInput where R: Rng + ?Sized, @@ -88,7 +114,7 @@ impl Frontier { // compensate for consensus rejections. // Note: this is a soft limit which is why the loop below might pass it if the // next sampled transaction happens to cross the bound - let extended_mass_limit = (policy.max_block_mass as f64 * MASS_LIMIT_FACTOR) as u64; + let desired_mass = (policy.max_block_mass as f64 * MASS_LIMIT_FACTOR) as u64; let mut distr = Uniform::new(0f64, self.total_weight()); let mut down_iter = self.search_tree.descending_iter(); @@ -98,8 +124,8 @@ impl Frontier { let mut total_selected_mass: u64 = 0; let mut _collisions = 0; - // The sampling process is converging thus the cache will hold all entries eventually, which guarantees loop exit - 'outer: while cache.len() < self.search_tree.len() && total_selected_mass <= extended_mass_limit { + // The sampling process is converging so the cache will eventually hold all entries, which guarantees loop exit + 'outer: while cache.len() < self.search_tree.len() && total_selected_mass <= desired_mass { let query = distr.sample(rng); let item = { let mut item = self.search_tree.search(query); diff --git a/mining/src/mempool/model/frontier/search_tree.rs b/mining/src/mempool/model/frontier/search_tree.rs index 687cc9091..184ddf758 100644 --- a/mining/src/mempool/model/frontier/search_tree.rs +++ b/mining/src/mempool/model/frontier/search_tree.rs @@ -301,7 +301,7 @@ mod tests { } #[test] - fn test_btree_rev_iter() { + fn test_tree_rev_iter() { let mut tree = SearchTree::new(); let mass = 2000; let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); From 310940474cd384581996d3a76389c8a0a5ea1ed9 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Tue, 6 Aug 2024 11:04:21 +0000 Subject: [PATCH 49/74] test_total_mass_tracking --- mining/src/mempool/model/frontier.rs | 52 ++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index ca2f88748..bebefb540 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -98,9 +98,9 @@ impl Frontier { /// close or equal to zero. We fix this by implementing a `log(n)` prefix weight operation. /// 5. Q. Why not just use u64 weights? /// A. The current weight calculation is `feerate^alpha` with `alpha=3`. Using u64 would mean that the feerate space - /// is limited to `(2^64)^(1/3) = ~2^21 = ~2M` possibilities. Already with current usages, the feerate can vary from `~1/50` (2000 sompi - /// for a transaction with 100K storage mass), to `5M` (100 KAS fee for a transaction with 2000 mass = 100·100_000_000/2000), - /// resulting in a range of 250M points (`5M/(1/50)`). + /// is limited to a range of size `(2^64)^(1/3) = ~2^21 = ~2M`. Already with current usages, the feerate can vary + /// from `~1/50` (2000 sompi for a transaction with 100K storage mass), to `5M` (100 KAS fee for a transaction with + /// 2000 mass = 100·100_000_000/2000), resulting in a range of size 250M (`5M/(1/50)`). /// By using floating point arithmetics we gain the adjustment of the probability space to the accuracy level required for /// current samples. And if the space is highly biased, the repeated elimination of top items and the prefix weight computation /// will readjust it. @@ -249,6 +249,7 @@ impl Frontier { mod tests { use super::*; use feerate_key::tests::build_feerate_key; + use itertools::Itertools; use rand::thread_rng; use std::collections::HashMap; @@ -312,6 +313,51 @@ mod tests { selector.select_transactions().iter().map(|k| k.gas).sum::(); } + #[test] + pub fn test_total_mass_tracking() { + let mut rng = thread_rng(); + let cap = 10000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = if i % (cap as u64 / 100) == 0 { 1000000 } else { rng.gen_range(1..10000) }; + let mass: u64 = rng.gen_range(1..100000); // Use distinct mass values to challenge the test + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let len = cap / 2; + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let prev_total_mass = frontier.total_mass(); + // Assert the total mass + assert_eq!(frontier.total_mass(), frontier.search_tree.ascending_iter().map(|k| k.mass).sum::()); + + // Add a bunch of duplicates and make sure the total mass remains the same + let mut dup_items = frontier.search_tree.ascending_iter().take(len / 2).cloned().collect_vec(); + for dup in dup_items.iter().cloned() { + (!frontier.insert(dup)).then_some(()).unwrap(); + } + assert_eq!(prev_total_mass, frontier.total_mass()); + assert_eq!(frontier.total_mass(), frontier.search_tree.ascending_iter().map(|k| k.mass).sum::()); + + // Remove a few elements from the map in order to randomize the iterator + dup_items.iter().take(10).for_each(|k| { + map.remove(&k.tx.id()); + }); + + // Add and remove random elements some of which will be duplicate insertions and some missing removals + for item in map.values().step_by(2) { + frontier.remove(item); + if let Some(item2) = dup_items.pop() { + frontier.insert(item2); + } + } + assert_eq!(frontier.total_mass(), frontier.search_tree.ascending_iter().map(|k| k.mass).sum::()); + } + #[test] fn test_feerate_estimator() { let mut rng = thread_rng(); From 06605967d847191d38898f810986bb1a4d61c9d8 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Tue, 6 Aug 2024 11:24:54 +0000 Subject: [PATCH 50/74] test prefix weights --- .../src/mempool/model/frontier/search_tree.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/mining/src/mempool/model/frontier/search_tree.rs b/mining/src/mempool/model/frontier/search_tree.rs index 184ddf758..beaa6c159 100644 --- a/mining/src/mempool/model/frontier/search_tree.rs +++ b/mining/src/mempool/model/frontier/search_tree.rs @@ -236,6 +236,7 @@ mod tests { use super::*; use itertools::Itertools; use std::collections::HashSet; + use std::ops::Sub; #[test] fn test_feerate_weight_queries() { @@ -273,13 +274,13 @@ mod tests { // Sweep through the tree and verify that weight search queries are handled correctly let eps: f64 = 0.001; let mut sum = 0.0; - for expected in v { + for expected in v.iter() { let weight = expected.weight(); let eps = eps.min(weight / 3.0); let samples = [sum + eps, sum + weight / 2.0, sum + weight - eps]; for sample in samples { let key = tree.search(sample); - assert_eq!(&expected, key); + assert_eq!(expected, key); assert!(expected.cmp(key).is_eq()); // Assert Ord equality as well } sum += weight; @@ -298,6 +299,18 @@ mod tests { assert_eq!(tree.last(), Some(tree.search(1.0 / 0.0))); assert_eq!(tree.last(), Some(tree.search(f64::INFINITY))); let _ = tree.search(f64::NAN); + + // Assert prefix weights + let mut prefix = Vec::with_capacity(v.len()); + prefix.push(v[0].weight()); + for i in 1..v.len() { + prefix.push(prefix[i - 1] + v[i].weight()); + } + let eps = v.iter().map(|k| k.weight()).min_by(f64::total_cmp).unwrap() * 1e-4; + for (expected_prefix, key) in prefix.into_iter().zip(v) { + let prefix = tree.prefix_weight(&key); + assert!(expected_prefix.sub(prefix).abs() < eps); + } } #[test] From 765c940095f5e8114646cab21a4d020c00b169bb Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Tue, 6 Aug 2024 12:28:58 +0000 Subject: [PATCH 51/74] test sequence selector --- mining/src/block_template/selector.rs | 64 ++++++++++++------- .../src/mempool/model/frontier/selectors.rs | 14 ++-- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/mining/src/block_template/selector.rs b/mining/src/block_template/selector.rs index bdf2eb1ed..6acacb22d 100644 --- a/mining/src/block_template/selector.rs +++ b/mining/src/block_template/selector.rs @@ -269,7 +269,13 @@ mod tests { use kaspa_txscript::{pay_to_script_hash_signature_script, test_helpers::op_true_script}; use std::{collections::HashSet, sync::Arc}; - use crate::{mempool::config::DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, model::candidate_tx::CandidateTransaction}; + use crate::{ + mempool::{ + config::DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, + model::frontier::selectors::{SequenceSelector, SequenceSelectorInput, SequenceSelectorTransaction}, + }, + model::candidate_tx::CandidateTransaction, + }; #[test] fn test_reject_transaction() { @@ -277,29 +283,43 @@ mod tests { // Create a vector of transactions differing by output value so they have unique ids let transactions = (0..TX_INITIAL_COUNT).map(|i| create_transaction(SOMPI_PER_KASPA * (i + 1) as u64)).collect_vec(); + let masses: HashMap<_, _> = transactions.iter().map(|tx| (tx.tx.id(), tx.calculated_mass)).collect(); + let sequence: SequenceSelectorInput = + transactions.iter().map(|tx| SequenceSelectorTransaction::new(tx.tx.clone(), tx.calculated_mass)).collect(); + let policy = Policy::new(100_000); - let mut selector = RebalancingWeightedTransactionSelector::new(policy, transactions); - let (mut kept, mut rejected) = (HashSet::new(), HashSet::new()); - let mut reject_count = 32; - for i in 0..10 { - let selected_txs = selector.select_transactions(); - if i > 0 { - assert_eq!( - selected_txs.len(), - reject_count, - "subsequent select calls are expected to only refill the previous rejections" - ); - reject_count /= 2; - } - for tx in selected_txs.iter() { - kept.insert(tx.id()).then_some(()).expect("selected txs should never repeat themselves"); - assert!(!rejected.contains(&tx.id()), "selected txs should never repeat themselves"); + let selectors: [Box; 2] = [ + Box::new(RebalancingWeightedTransactionSelector::new(policy.clone(), transactions)), + Box::new(SequenceSelector::new(sequence, policy.clone())), + ]; + + for mut selector in selectors { + let (mut kept, mut rejected) = (HashSet::new(), HashSet::new()); + let mut reject_count = 32; + let mut total_mass = 0; + for i in 0..10 { + let selected_txs = selector.select_transactions(); + if i > 0 { + assert_eq!( + selected_txs.len(), + reject_count, + "subsequent select calls are expected to only refill the previous rejections" + ); + reject_count /= 2; + } + for tx in selected_txs.iter() { + total_mass += masses[&tx.id()]; + kept.insert(tx.id()).then_some(()).expect("selected txs should never repeat themselves"); + assert!(!rejected.contains(&tx.id()), "selected txs should never repeat themselves"); + } + assert!(total_mass <= policy.max_block_mass); + selected_txs.iter().take(reject_count).for_each(|x| { + total_mass -= masses[&x.id()]; + selector.reject_selection(x.id()); + kept.remove(&x.id()).then_some(()).expect("was just inserted"); + rejected.insert(x.id()).then_some(()).expect("was just verified"); + }); } - selected_txs.iter().take(reject_count).for_each(|x| { - selector.reject_selection(x.id()); - kept.remove(&x.id()).then_some(()).expect("was just inserted"); - rejected.insert(x.id()).then_some(()).expect("was just verified"); - }); } } diff --git a/mining/src/mempool/model/frontier/selectors.rs b/mining/src/mempool/model/frontier/selectors.rs index c37ae81a9..fc8e35efe 100644 --- a/mining/src/mempool/model/frontier/selectors.rs +++ b/mining/src/mempool/model/frontier/selectors.rs @@ -19,19 +19,25 @@ impl SequenceSelectorTransaction { } } -type SequenceSelectorPriorityIndex = u32; +type SequencePriorityIndex = u32; /// The input sequence for the [`SequenceSelector`] transaction selector #[derive(Default)] pub struct SequenceSelectorInput { /// We use the btree map ordered by insertion order in order to follow /// the initial sequence order while allowing for efficient removal of previous selections - inner: BTreeMap, + inner: BTreeMap, +} + +impl FromIterator for SequenceSelectorInput { + fn from_iter>(iter: T) -> Self { + Self { inner: BTreeMap::from_iter(iter.into_iter().enumerate().map(|(i, v)| (i as SequencePriorityIndex, v))) } + } } impl SequenceSelectorInput { pub fn push(&mut self, tx: Arc, mass: u64) { - let idx = self.inner.len() as SequenceSelectorPriorityIndex; + let idx = self.inner.len() as SequencePriorityIndex; self.inner.insert(idx, SequenceSelectorTransaction::new(tx, mass)); } @@ -44,7 +50,7 @@ impl SequenceSelectorInput { struct SequenceSelectorSelection { tx_id: TransactionId, mass: u64, - priority_index: SequenceSelectorPriorityIndex, + priority_index: SequencePriorityIndex, } /// A selector which selects transactions in the order they are provided. The selector assumes From bca599ab51e5b0e0c3f3edaa6b6fd394f2b173de Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Tue, 6 Aug 2024 14:19:27 +0000 Subject: [PATCH 52/74] fix rpc feerate structs + comment --- mining/src/feerate/mod.rs | 2 +- .../src/mempool/model/frontier/feerate_key.rs | 3 +++ rpc/core/src/model/message.rs | 24 ++++++++++++++----- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index 57078cabc..ab0d9ea84 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -17,7 +17,7 @@ pub struct FeerateEstimations { /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating - /// between them, once can compose a complete feerate function on the client side. The API makes an effort + /// between them, one can compose a complete feerate function on the client side. The API makes an effort /// to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. pub normal_buckets: Vec, diff --git a/mining/src/mempool/model/frontier/feerate_key.rs b/mining/src/mempool/model/frontier/feerate_key.rs index 4be5ce6c8..fd2cca091 100644 --- a/mining/src/mempool/model/frontier/feerate_key.rs +++ b/mining/src/mempool/model/frontier/feerate_key.rs @@ -20,6 +20,9 @@ impl PartialEq for FeerateTransactionKey { impl FeerateTransactionKey { pub fn new(fee: u64, mass: u64, tx: Arc) -> Self { + // NOTE: any change to the way this weight is calculated (such as scaling by some factor) + // requires a reversed update to total_weight in `Frontier::build_feerate_estimator`. This + // is because the math methods in FeeEstimator assume this specific weight function. Self { fee, mass, weight: (fee as f64 / mass as f64).powi(ALPHA), tx } } diff --git a/rpc/core/src/model/message.rs b/rpc/core/src/model/message.rs index 2dd7261dd..b91eb08d3 100644 --- a/rpc/core/src/model/message.rs +++ b/rpc/core/src/model/message.rs @@ -860,16 +860,29 @@ pub struct GetFeeEstimateRequest {} #[derive(Clone, Copy, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] pub struct FeerateBucket { + /// The fee/mass ratio estimated to be required for inclusion time <= estimated_seconds pub feerate: f64, - pub estimated_seconds: u64, + + /// The estimated inclusion time for a transaction with fee/mass = feerate + pub estimated_seconds: f64, } #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] pub struct GetFeeEstimateResponse { - pub low_bucket: FeerateBucket, - pub normal_bucket: FeerateBucket, + /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. pub priority_bucket: FeerateBucket, + + /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to + /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + /// between them, one can compose a complete feerate function on the client side. The API makes an effort + /// to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + pub normal_buckets: Vec, + + /// A vector of *low* priority feerate values. The first value of this vector is guaranteed to + /// provide an estimation for sub-*hour* DAG inclusion. + pub low_buckets: Vec, } // pub struct GetFeeEstimateExperimentalRequest { @@ -877,10 +890,9 @@ pub struct GetFeeEstimateResponse { // } // pub struct GetFeeEstimateExperimentalResponse { -// // Same as the usual response -// pub low_bucket: FeerateBucket, -// pub normal_bucket: FeerateBucket, // pub priority_bucket: FeerateBucket, +// pub normal_buckets: Vec, +// pub low_buckets: Vec, // /// Experimental verbose data // pub verbose: Option, From a3eda1882376e9f5fed2adba6ac8c5db6e127397 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Tue, 6 Aug 2024 20:12:55 +0000 Subject: [PATCH 53/74] utils: expiring cache --- Cargo.lock | 1 + utils/Cargo.toml | 1 + utils/src/expiring_cache.rs | 148 ++++++++++++++++++++++++++++++++++++ utils/src/lib.rs | 1 + 4 files changed, 151 insertions(+) create mode 100644 utils/src/expiring_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 81ab7ba80..77a27aca3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3408,6 +3408,7 @@ dependencies = [ name = "kaspa-utils" version = "0.14.1" dependencies = [ + "arc-swap", "async-channel 2.2.1", "async-trait", "bincode", diff --git a/utils/Cargo.toml b/utils/Cargo.toml index a3002afab..dda05cb0e 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true repository.workspace = true [dependencies] +arc-swap.workspace = true parking_lot.workspace = true async-channel.workspace = true borsh.workspace = true diff --git a/utils/src/expiring_cache.rs b/utils/src/expiring_cache.rs new file mode 100644 index 000000000..4ee3e521f --- /dev/null +++ b/utils/src/expiring_cache.rs @@ -0,0 +1,148 @@ +use arc_swap::ArcSwapOption; +use std::{ + future::Future, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; + +struct Entry { + item: T, + timestamp: Instant, +} + +/// An expiring cache for a single object +pub struct ExpiringCache { + store: ArcSwapOption>, + refetch: Duration, + expire: Duration, + fetching: AtomicBool, +} + +impl ExpiringCache { + /// Constructs a new expiring cache where `fetch` is the amount of time required to trigger a data + /// refetch and `expire` is the time duration after which the stored item will never be returned. + pub fn new(refetch: Duration, expire: Duration) -> Self { + assert!(refetch <= expire); + Self { store: Default::default(), refetch, expire, fetching: Default::default() } + } + + pub async fn get(&self, refetch_future: F) -> T + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + let mut fetching = false; + + { + let guard = self.store.load(); + if let Some(entry) = guard.as_ref() { + if let Some(elapsed) = Instant::now().checked_duration_since(entry.timestamp) { + if elapsed < self.refetch { + return entry.item.clone(); + } + // Refetch is triggered, attempt to capture the task + fetching = self.fetching.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst).is_ok(); + // If the fetch task is not captured and expire time is not over yet, return with prev value. Another + // thread is refetching the data but we can return with the not-too-old value + if !fetching && elapsed < self.expire { + return entry.item.clone(); + } + } + // else -- In rare cases where now < timestamp, fall through to re-update the cache + } + } + + // We reach here if either we are the refetching thread or the current data has fully expired + let new_item = refetch_future.await; + let timestamp = Instant::now(); + // Update the store even if we were not in charge of refetching - let the last thread make the final update + self.store.store(Some(Arc::new(Entry { item: new_item.clone(), timestamp }))); + + if fetching { + let result = self.fetching.compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst); + assert!(result.is_ok(), "refetching was captured") + } + + new_item + } +} + +#[cfg(test)] +mod tests { + use super::ExpiringCache; + use std::time::Duration; + use tokio::join; + + #[tokio::test] + #[ignore] + // Tested during development but can be sensitive to runtime machine times so there's no point + // in keeping it part of CI. The test should be activated if the ExpiringCache struct changes. + async fn test_expiring_cache() { + let fetch = Duration::from_millis(500); + let expire = Duration::from_millis(1000); + let mid_point = Duration::from_millis(700); + let expire_point = Duration::from_millis(1200); + let cache: ExpiringCache = ExpiringCache::new(fetch, expire); + + // Test two consecutive calls + let item1 = cache + .get(async move { + println!("first call"); + 1 + }) + .await; + assert_eq!(1, item1); + let item2 = cache + .get(async move { + // cache was just updated with item1, refetch should not be triggered + panic!("should not be called"); + }) + .await; + assert_eq!(1, item2); + + // Test two calls after refetch point + // Sleep until after the refetch point but before expire + tokio::time::sleep(mid_point).await; + let call3 = cache.get(async move { + println!("third call before sleep"); + // keep this refetch busy so that call4 still gets the first item + tokio::time::sleep(Duration::from_millis(100)).await; + println!("third call after sleep"); + 3 + }); + let call4 = cache.get(async move { + // refetch is captured by call3 and we should be before expire + panic!("should not be called"); + }); + let (item3, item4) = join!(call3, call4); + println!("item 3: {}, item 4: {}", item3, item4); + assert_eq!(3, item3); + assert_eq!(1, item4); + + // Test 2 calls after expire + tokio::time::sleep(expire_point).await; + let call5 = cache.get(async move { + println!("5th call before sleep"); + tokio::time::sleep(Duration::from_millis(100)).await; + println!("5th call after sleep"); + 5 + }); + let call6 = cache.get(async move { 6 }); + let (item5, item6) = join!(call5, call6); + println!("item 5: {}, item 6: {}", item5, item6); + assert_eq!(5, item5); + assert_eq!(6, item6); + + let item7 = cache + .get(async move { + // cache was just updated with item5, refetch should not be triggered + panic!("should not be called"); + }) + .await; + // call 5 finished after call 6 + assert_eq!(5, item7); + } +} diff --git a/utils/src/lib.rs b/utils/src/lib.rs index bd3143719..79e96c44f 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -2,6 +2,7 @@ pub mod any; pub mod arc; pub mod binary_heap; pub mod channel; +pub mod expiring_cache; pub mod hashmap; pub mod hex; pub mod iter; From d606c3b955b51ba8bb674766ba99d5c90ea00e45 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Wed, 7 Aug 2024 10:04:59 +0000 Subject: [PATCH 54/74] rpc core fee estimate call --- rpc/core/src/api/ops.rs | 4 ++ rpc/core/src/api/rpc.rs | 16 +++++ rpc/core/src/model/feerate_estimate.rs | 42 ++++++++++++ rpc/core/src/model/message.rs | 67 +++++-------------- rpc/core/src/model/mod.rs | 2 + rpc/grpc/client/src/lib.rs | 11 +++ rpc/grpc/server/src/tests/rpc_core_mock.rs | 11 +++ rpc/service/src/converter/feerate_estimate.rs | 26 +++++++ rpc/service/src/converter/mod.rs | 1 + rpc/service/src/service.rs | 19 ++++++ rpc/wrpc/client/src/client.rs | 2 + utils/src/expiring_cache.rs | 6 +- wallet/core/src/tests/rpc_core_mock.rs | 11 +++ 13 files changed, 165 insertions(+), 53 deletions(-) create mode 100644 rpc/core/src/model/feerate_estimate.rs create mode 100644 rpc/service/src/converter/feerate_estimate.rs diff --git a/rpc/core/src/api/ops.rs b/rpc/core/src/api/ops.rs index d312f499b..6d5357406 100644 --- a/rpc/core/src/api/ops.rs +++ b/rpc/core/src/api/ops.rs @@ -117,6 +117,10 @@ pub enum RpcApiOps { /// Extracts a transaction out of the request message and attempts to replace a matching transaction in the mempool with it, applying a mandatory Replace by Fee policy SubmitTransactionReplacement, + + // Fee estimation related commands + GetFeeEstimate, + GetFeeEstimateExperimental, } impl RpcApiOps { diff --git a/rpc/core/src/api/rpc.rs b/rpc/core/src/api/rpc.rs index f109ef8f6..6d9ab4b86 100644 --- a/rpc/core/src/api/rpc.rs +++ b/rpc/core/src/api/rpc.rs @@ -315,6 +315,22 @@ pub trait RpcApi: Sync + Send + AnySync { request: GetDaaScoreTimestampEstimateRequest, ) -> RpcResult; + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // Fee estimation API + + async fn get_fee_estimate(&self) -> RpcResult { + Ok(self.get_fee_estimate_call(GetFeeEstimateRequest {}).await?.estimate) + } + async fn get_fee_estimate_call(&self, request: GetFeeEstimateRequest) -> RpcResult; + + async fn get_fee_estimate_experimental(&self, verbose: bool) -> RpcResult { + self.get_fee_estimate_experimental_call(GetFeeEstimateExperimentalRequest { verbose }).await + } + async fn get_fee_estimate_experimental_call( + &self, + request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult; + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/core/src/model/feerate_estimate.rs b/rpc/core/src/model/feerate_estimate.rs new file mode 100644 index 000000000..d43530fdd --- /dev/null +++ b/rpc/core/src/model/feerate_estimate.rs @@ -0,0 +1,42 @@ +use borsh::{BorshDeserialize, BorshSerialize}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct FeerateBucket { + /// The fee/mass ratio estimated to be required for inclusion time <= estimated_seconds + pub feerate: f64, + + /// The estimated inclusion time for a transaction with fee/mass = feerate + pub estimated_seconds: f64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct FeeEstimate { + /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + pub priority_bucket: FeerateBucket, + + /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and + /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + /// between them, one can compose a complete feerate function on the client side. The API makes an effort + /// to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + pub normal_buckets: Vec, + + /// A vector of *low* priority feerate values. The first value of this vector is guaranteed to + /// exist and provide an estimation for sub-*hour* DAG inclusion. + pub low_buckets: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct FeeEstimateVerboseExperimentalData { + pub total_mempool_mass: u64, + pub total_mempool_count: u64, + pub network_mass_per_second: u64, + + pub next_block_template_feerate_min: f64, + pub next_block_template_feerate_median: f64, + pub next_block_template_feerate_max: f64, +} diff --git a/rpc/core/src/model/message.rs b/rpc/core/src/model/message.rs index b91eb08d3..88edd08a7 100644 --- a/rpc/core/src/model/message.rs +++ b/rpc/core/src/model/message.rs @@ -857,64 +857,27 @@ impl GetDaaScoreTimestampEstimateResponse { #[serde(rename_all = "camelCase")] pub struct GetFeeEstimateRequest {} -#[derive(Clone, Copy, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] -pub struct FeerateBucket { - /// The fee/mass ratio estimated to be required for inclusion time <= estimated_seconds - pub feerate: f64, +pub struct GetFeeEstimateResponse { + pub estimate: FeeEstimate, +} - /// The estimated inclusion time for a transaction with fee/mass = feerate - pub estimated_seconds: f64, +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateExperimentalRequest { + pub verbose: bool, } #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] -pub struct GetFeeEstimateResponse { - /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. - pub priority_bucket: FeerateBucket, - - /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to - /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation - /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating - /// between them, one can compose a complete feerate function on the client side. The API makes an effort - /// to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. - pub normal_buckets: Vec, - - /// A vector of *low* priority feerate values. The first value of this vector is guaranteed to - /// provide an estimation for sub-*hour* DAG inclusion. - pub low_buckets: Vec, -} - -// pub struct GetFeeEstimateExperimentalRequest { -// pub verbose: bool, -// } - -// pub struct GetFeeEstimateExperimentalResponse { -// pub priority_bucket: FeerateBucket, -// pub normal_buckets: Vec, -// pub low_buckets: Vec, - -// /// Experimental verbose data -// pub verbose: Option, -// } - -// pub struct FeeEstimateVerboseExperimentalData { -// // mempool load factor in relation to tx/s -// // processing capacity -// pub mempool_load_factor: f64, -// // temperature of the mempool water -// pub mempool_water_temperature_celsius: f64, -// // optional internal context data that -// // represents components used to calculate -// // fee estimates and time periods. -// // ... - -// // total_mass, total_fee, etc -// /// Built from Maxim's implementation -// pub next_block_template_feerate_min: f64, -// pub next_block_template_feerate_median: f64, -// pub next_block_template_feerate_max: f64, -// } +pub struct GetFeeEstimateExperimentalResponse { + /// The usual feerate estimate response + pub estimate: FeeEstimate, + + /// Experimental verbose data + pub verbose: Option, +} // ---------------------------------------------------------------------------- // Subscriptions & notifications diff --git a/rpc/core/src/model/mod.rs b/rpc/core/src/model/mod.rs index fd07a109e..8950bd1cb 100644 --- a/rpc/core/src/model/mod.rs +++ b/rpc/core/src/model/mod.rs @@ -1,6 +1,7 @@ pub mod address; pub mod block; pub mod blue_work; +pub mod feerate_estimate; pub mod hash; pub mod header; pub mod hex_cnv; @@ -15,6 +16,7 @@ pub mod tx; pub use address::*; pub use block::*; pub use blue_work::*; +pub use feerate_estimate::*; pub use hash::*; pub use header::*; pub use hex_cnv::*; diff --git a/rpc/grpc/client/src/lib.rs b/rpc/grpc/client/src/lib.rs index 98e72ce3e..8371fb898 100644 --- a/rpc/grpc/client/src/lib.rs +++ b/rpc/grpc/client/src/lib.rs @@ -273,6 +273,17 @@ impl RpcApi for GrpcClient { route!(get_coin_supply_call, GetCoinSupply); route!(get_daa_score_timestamp_estimate_call, GetDaaScoreTimestampEstimate); + // TODO (PR) + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + Err(RpcError::NotImplemented) + } + async fn get_fee_estimate_experimental_call( + &self, + _request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/grpc/server/src/tests/rpc_core_mock.rs b/rpc/grpc/server/src/tests/rpc_core_mock.rs index f0e7bda1d..2f4afa9c9 100644 --- a/rpc/grpc/server/src/tests/rpc_core_mock.rs +++ b/rpc/grpc/server/src/tests/rpc_core_mock.rs @@ -235,6 +235,17 @@ impl RpcApi for RpcCoreMock { Err(RpcError::NotImplemented) } + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + Err(RpcError::NotImplemented) + } + + async fn get_fee_estimate_experimental_call( + &self, + _request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/service/src/converter/feerate_estimate.rs b/rpc/service/src/converter/feerate_estimate.rs new file mode 100644 index 000000000..928f4b5ca --- /dev/null +++ b/rpc/service/src/converter/feerate_estimate.rs @@ -0,0 +1,26 @@ +use kaspa_mining::feerate::{FeerateBucket, FeerateEstimations}; +use kaspa_rpc_core::{FeeEstimate as RpcFeeEstimate, FeerateBucket as RpcFeerateBucket}; + +pub trait FeerateBucketConverter { + fn into_rpc(self) -> RpcFeerateBucket; +} + +impl FeerateBucketConverter for FeerateBucket { + fn into_rpc(self) -> RpcFeerateBucket { + RpcFeerateBucket { feerate: self.feerate, estimated_seconds: self.estimated_seconds } + } +} + +pub trait FeeEstimateConverter { + fn into_rpc(self) -> RpcFeeEstimate; +} + +impl FeeEstimateConverter for FeerateEstimations { + fn into_rpc(self) -> RpcFeeEstimate { + RpcFeeEstimate { + priority_bucket: self.priority_bucket.into_rpc(), + normal_buckets: self.normal_buckets.into_iter().map(FeerateBucketConverter::into_rpc).collect(), + low_buckets: self.low_buckets.into_iter().map(FeerateBucketConverter::into_rpc).collect(), + } + } +} diff --git a/rpc/service/src/converter/mod.rs b/rpc/service/src/converter/mod.rs index 2e1460385..fd167d349 100644 --- a/rpc/service/src/converter/mod.rs +++ b/rpc/service/src/converter/mod.rs @@ -1,3 +1,4 @@ pub mod consensus; +pub mod feerate_estimate; pub mod index; pub mod protocol; diff --git a/rpc/service/src/service.rs b/rpc/service/src/service.rs index c69fda4f1..b3cb3d80a 100644 --- a/rpc/service/src/service.rs +++ b/rpc/service/src/service.rs @@ -1,6 +1,7 @@ //! Core server implementation for ClientAPI use super::collector::{CollectorFromConsensus, CollectorFromIndex}; +use crate::converter::feerate_estimate::FeeEstimateConverter; use crate::converter::{consensus::ConsensusConverter, index::IndexConverter, protocol::ProtocolConverter}; use crate::service::NetworkType::{Mainnet, Testnet}; use async_trait::async_trait; @@ -61,9 +62,11 @@ use kaspa_rpc_core::{ Notification, RpcError, RpcResult, }; use kaspa_txscript::{extract_script_pub_key_address, pay_to_address_script}; +use kaspa_utils::expiring_cache::ExpiringCache; use kaspa_utils::{channel::Channel, triggers::SingleTrigger}; use kaspa_utils_tower::counters::TowerConnectionCounters; use kaspa_utxoindex::api::UtxoIndexProxy; +use std::time::Duration; use std::{ collections::HashMap, iter::once, @@ -109,6 +112,7 @@ pub struct RpcCoreService { perf_monitor: Arc>>, p2p_tower_counters: Arc, grpc_tower_counters: Arc, + fee_estimate_cache: ExpiringCache, } const RPC_CORE: &str = "rpc-core"; @@ -208,6 +212,7 @@ impl RpcCoreService { perf_monitor, p2p_tower_counters, grpc_tower_counters, + fee_estimate_cache: ExpiringCache::new(Duration::from_millis(500), Duration::from_millis(1000)), } } @@ -663,6 +668,20 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(GetDaaScoreTimestampEstimateResponse::new(timestamps)) } + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + let mining_manager = self.mining_manager.clone(); + let estimate = + self.fee_estimate_cache.get(async move { mining_manager.get_realtime_feerate_estimations().await.into_rpc() }).await; + Ok(GetFeeEstimateResponse { estimate }) + } + + async fn get_fee_estimate_experimental_call( + &self, + _request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + todo!() + } + async fn ping_call(&self, _: PingRequest) -> RpcResult { Ok(PingResponse {}) } diff --git a/rpc/wrpc/client/src/client.rs b/rpc/wrpc/client/src/client.rs index 7d8548171..052c51dd9 100644 --- a/rpc/wrpc/client/src/client.rs +++ b/rpc/wrpc/client/src/client.rs @@ -619,6 +619,8 @@ impl RpcApi for KaspaRpcClient { SubmitTransaction, SubmitTransactionReplacement, Unban, + GetFeeEstimate, + GetFeeEstimateExperimental ] ); diff --git a/utils/src/expiring_cache.rs b/utils/src/expiring_cache.rs index 4ee3e521f..175bea548 100644 --- a/utils/src/expiring_cache.rs +++ b/utils/src/expiring_cache.rs @@ -23,12 +23,16 @@ pub struct ExpiringCache { impl ExpiringCache { /// Constructs a new expiring cache where `fetch` is the amount of time required to trigger a data - /// refetch and `expire` is the time duration after which the stored item will never be returned. + /// refetch and `expire` is the time duration after which the stored item is guaranteed not to be returned. + /// + /// Panics if `refetch > expire`. pub fn new(refetch: Duration, expire: Duration) -> Self { assert!(refetch <= expire); Self { store: Default::default(), refetch, expire, fetching: Default::default() } } + /// Returns the cached item or possibly fetches a new one using the `refetch_future` task. The + /// decision whether to refetch depends on the configured expiration and refetch times for this cache. pub async fn get(&self, refetch_future: F) -> T where F: Future + Send + 'static, diff --git a/wallet/core/src/tests/rpc_core_mock.rs b/wallet/core/src/tests/rpc_core_mock.rs index 70d8792dc..1e6d70c2b 100644 --- a/wallet/core/src/tests/rpc_core_mock.rs +++ b/wallet/core/src/tests/rpc_core_mock.rs @@ -252,6 +252,17 @@ impl RpcApi for RpcCoreMock { Err(RpcError::NotImplemented) } + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + Err(RpcError::NotImplemented) + } + + async fn get_fee_estimate_experimental_call( + &self, + _request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API From 85bd72b592921a70e28be17a2139d645af9bade9 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Wed, 7 Aug 2024 11:44:59 +0000 Subject: [PATCH 55/74] fee estimate verbose --- mining/src/feerate/mod.rs | 13 +++++++++ mining/src/manager.rs | 28 ++++++++++++++++++- mining/src/mempool/mod.rs | 8 ++++++ mining/src/mempool/model/transactions_pool.rs | 4 +++ rpc/core/src/model/feerate_estimate.rs | 4 +-- rpc/service/src/converter/feerate_estimate.rs | 27 ++++++++++++++++-- rpc/service/src/service.rs | 18 ++++++++++-- 7 files changed, 94 insertions(+), 8 deletions(-) diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index ab0d9ea84..70168a04a 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -103,6 +103,19 @@ impl FeerateEstimator { } } +#[derive(Clone, Debug)] +pub struct FeeEstimateVerbose { + pub estimations: FeerateEstimations, + + pub mempool_ready_transactions_count: u64, + pub mempool_ready_transactions_total_mass: u64, + pub network_mass_per_second: u64, + + pub next_block_template_feerate_min: f64, + pub next_block_template_feerate_median: f64, + pub next_block_template_feerate_max: f64, +} + #[cfg(test)] mod tests { use super::*; diff --git a/mining/src/manager.rs b/mining/src/manager.rs index 3818b56f3..8bd46c3d8 100644 --- a/mining/src/manager.rs +++ b/mining/src/manager.rs @@ -2,7 +2,7 @@ use crate::{ block_template::{builder::BlockTemplateBuilder, errors::BuilderError}, cache::BlockTemplateCache, errors::MiningManagerResult, - feerate::{FeerateEstimations, FeerateEstimatorArgs}, + feerate::{FeeEstimateVerbose, FeerateEstimations, FeerateEstimatorArgs}, mempool::{ config::Config, model::tx::{MempoolTransaction, TransactionPostValidation, TransactionPreValidation, TxRemovalReason}, @@ -209,6 +209,27 @@ impl MiningManager { estimator.calc_estimations(self.config.minimum_feerate()) } + /// Returns realtime feerate estimations based on internal mempool state with additional verbose data + pub(crate) fn get_realtime_feerate_estimations_verbose(&self) -> FeeEstimateVerbose { + let args = FeerateEstimatorArgs::new(self.config.network_blocks_per_second, self.config.maximum_mass_per_block); + let network_mass_per_second = args.network_mass_per_second(); + let mempool_read = self.mempool.read(); + let estimator = mempool_read.build_feerate_estimator(args); + let ready_transactions_count = mempool_read.ready_transaction_count(); + let ready_transaction_total_mass = mempool_read.ready_transaction_total_mass(); + drop(mempool_read); + FeeEstimateVerbose { + estimations: estimator.calc_estimations(self.config.minimum_feerate()), + network_mass_per_second, + mempool_ready_transactions_count: ready_transactions_count as u64, + mempool_ready_transactions_total_mass: ready_transaction_total_mass, + // TODO: Next PR + next_block_template_feerate_min: -1.0, + next_block_template_feerate_median: -1.0, + next_block_template_feerate_max: -1.0, + } + } + /// Clears the block template cache, forcing the next call to get_block_template to build a new block template. #[cfg(test)] pub(crate) fn clear_block_template(&self) { @@ -812,6 +833,11 @@ impl MiningManagerProxy { spawn_blocking(move || self.inner.get_realtime_feerate_estimations()).await.unwrap() } + /// Returns realtime feerate estimations based on internal mempool state with additional verbose data + pub async fn get_realtime_feerate_estimations_verbose(self) -> FeeEstimateVerbose { + spawn_blocking(move || self.inner.get_realtime_feerate_estimations_verbose()).await.unwrap() + } + /// Validates a transaction and adds it to the set of known transactions that have not yet been /// added to any block. /// diff --git a/mining/src/mempool/mod.rs b/mining/src/mempool/mod.rs index c634a4a5b..e5cd7dbeb 100644 --- a/mining/src/mempool/mod.rs +++ b/mining/src/mempool/mod.rs @@ -115,6 +115,14 @@ impl Mempool { count } + pub(crate) fn ready_transaction_count(&self) -> usize { + self.transaction_pool.ready_transaction_count() + } + + pub(crate) fn ready_transaction_total_mass(&self) -> u64 { + self.transaction_pool.ready_transaction_total_mass() + } + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier pub(crate) fn build_selector(&self) -> Box { let _sw = Stopwatch::<10>::with_threshold("build_selector op"); diff --git a/mining/src/mempool/model/transactions_pool.rs b/mining/src/mempool/model/transactions_pool.rs index 1b6a3e532..bc3469409 100644 --- a/mining/src/mempool/model/transactions_pool.rs +++ b/mining/src/mempool/model/transactions_pool.rs @@ -167,6 +167,10 @@ impl TransactionsPool { self.ready_transactions.len() } + pub(crate) fn ready_transaction_total_mass(&self) -> u64 { + self.ready_transactions.total_mass() + } + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier pub(crate) fn build_selector(&self) -> Box { self.ready_transactions.build_selector(&Policy::new(self.config.maximum_mass_per_block)) diff --git a/rpc/core/src/model/feerate_estimate.rs b/rpc/core/src/model/feerate_estimate.rs index d43530fdd..7a317744f 100644 --- a/rpc/core/src/model/feerate_estimate.rs +++ b/rpc/core/src/model/feerate_estimate.rs @@ -32,8 +32,8 @@ pub struct FeeEstimate { #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] pub struct FeeEstimateVerboseExperimentalData { - pub total_mempool_mass: u64, - pub total_mempool_count: u64, + pub mempool_ready_transactions_count: u64, + pub mempool_ready_transactions_total_mass: u64, pub network_mass_per_second: u64, pub next_block_template_feerate_min: f64, diff --git a/rpc/service/src/converter/feerate_estimate.rs b/rpc/service/src/converter/feerate_estimate.rs index 928f4b5ca..88c5f15c2 100644 --- a/rpc/service/src/converter/feerate_estimate.rs +++ b/rpc/service/src/converter/feerate_estimate.rs @@ -1,5 +1,8 @@ -use kaspa_mining::feerate::{FeerateBucket, FeerateEstimations}; -use kaspa_rpc_core::{FeeEstimate as RpcFeeEstimate, FeerateBucket as RpcFeerateBucket}; +use kaspa_mining::feerate::{FeeEstimateVerbose, FeerateBucket, FeerateEstimations}; +use kaspa_rpc_core::{ + message::GetFeeEstimateExperimentalResponse as RpcFeeEstimateVerboseResponse, FeeEstimate as RpcFeeEstimate, + FeeEstimateVerboseExperimentalData as RpcFeeEstimateVerbose, FeerateBucket as RpcFeerateBucket, +}; pub trait FeerateBucketConverter { fn into_rpc(self) -> RpcFeerateBucket; @@ -24,3 +27,23 @@ impl FeeEstimateConverter for FeerateEstimations { } } } + +pub trait FeeEstimateVerboseConverter { + fn into_rpc(self) -> RpcFeeEstimateVerboseResponse; +} + +impl FeeEstimateVerboseConverter for FeeEstimateVerbose { + fn into_rpc(self) -> RpcFeeEstimateVerboseResponse { + RpcFeeEstimateVerboseResponse { + estimate: self.estimations.into_rpc(), + verbose: Some(RpcFeeEstimateVerbose { + network_mass_per_second: self.network_mass_per_second, + mempool_ready_transactions_count: self.mempool_ready_transactions_count, + mempool_ready_transactions_total_mass: self.mempool_ready_transactions_total_mass, + next_block_template_feerate_min: self.next_block_template_feerate_min, + next_block_template_feerate_median: self.next_block_template_feerate_median, + next_block_template_feerate_max: self.next_block_template_feerate_max, + }), + } + } +} diff --git a/rpc/service/src/service.rs b/rpc/service/src/service.rs index b3cb3d80a..d4dcbdb31 100644 --- a/rpc/service/src/service.rs +++ b/rpc/service/src/service.rs @@ -1,7 +1,7 @@ //! Core server implementation for ClientAPI use super::collector::{CollectorFromConsensus, CollectorFromIndex}; -use crate::converter::feerate_estimate::FeeEstimateConverter; +use crate::converter::feerate_estimate::{FeeEstimateConverter, FeeEstimateVerboseConverter}; use crate::converter::{consensus::ConsensusConverter, index::IndexConverter, protocol::ProtocolConverter}; use crate::service::NetworkType::{Mainnet, Testnet}; use async_trait::async_trait; @@ -113,6 +113,7 @@ pub struct RpcCoreService { p2p_tower_counters: Arc, grpc_tower_counters: Arc, fee_estimate_cache: ExpiringCache, + fee_estimate_verbose_cache: ExpiringCache, } const RPC_CORE: &str = "rpc-core"; @@ -213,6 +214,7 @@ impl RpcCoreService { p2p_tower_counters, grpc_tower_counters, fee_estimate_cache: ExpiringCache::new(Duration::from_millis(500), Duration::from_millis(1000)), + fee_estimate_verbose_cache: ExpiringCache::new(Duration::from_millis(500), Duration::from_millis(1000)), } } @@ -677,9 +679,19 @@ NOTE: This error usually indicates an RPC conversion error between the node and async fn get_fee_estimate_experimental_call( &self, - _request: GetFeeEstimateExperimentalRequest, + request: GetFeeEstimateExperimentalRequest, ) -> RpcResult { - todo!() + if request.verbose { + let mining_manager = self.mining_manager.clone(); + let response = self + .fee_estimate_verbose_cache + .get(async move { mining_manager.get_realtime_feerate_estimations_verbose().await.into_rpc() }) + .await; + Ok(response) + } else { + let estimate = self.get_fee_estimate_call(GetFeeEstimateRequest {}).await?.estimate; + Ok(GetFeeEstimateExperimentalResponse { estimate, verbose: None }) + } } async fn ping_call(&self, _: PingRequest) -> RpcResult { From 797d5d8f981d7e5a3d9182b62333525fbefa3b68 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Wed, 7 Aug 2024 13:18:39 +0000 Subject: [PATCH 56/74] grpc fee estimate calls --- rpc/core/src/model/feerate_estimate.rs | 9 +++ rpc/grpc/client/src/lib.rs | 13 +--- rpc/grpc/core/proto/messages.proto | 4 ++ rpc/grpc/core/proto/rpc.proto | 40 +++++++++++ rpc/grpc/core/src/convert/feerate_estimate.rs | 66 +++++++++++++++++++ rpc/grpc/core/src/convert/kaspad.rs | 4 ++ rpc/grpc/core/src/convert/message.rs | 43 ++++++++++++ rpc/grpc/core/src/convert/mod.rs | 1 + rpc/grpc/core/src/ops.rs | 2 + .../server/src/request_handler/factory.rs | 2 + testing/integration/src/rpc_tests.rs | 27 ++++++++ 11 files changed, 200 insertions(+), 11 deletions(-) create mode 100644 rpc/grpc/core/src/convert/feerate_estimate.rs diff --git a/rpc/core/src/model/feerate_estimate.rs b/rpc/core/src/model/feerate_estimate.rs index 7a317744f..4c199c40f 100644 --- a/rpc/core/src/model/feerate_estimate.rs +++ b/rpc/core/src/model/feerate_estimate.rs @@ -29,6 +29,15 @@ pub struct FeeEstimate { pub low_buckets: Vec, } +impl FeeEstimate { + pub fn ordered_buckets(&self) -> Vec { + std::iter::once(self.priority_bucket) + .chain(self.normal_buckets.iter().copied()) + .chain(self.low_buckets.iter().copied()) + .collect() + } +} + #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] pub struct FeeEstimateVerboseExperimentalData { diff --git a/rpc/grpc/client/src/lib.rs b/rpc/grpc/client/src/lib.rs index 8371fb898..c19a28eb5 100644 --- a/rpc/grpc/client/src/lib.rs +++ b/rpc/grpc/client/src/lib.rs @@ -272,17 +272,8 @@ impl RpcApi for GrpcClient { route!(get_mempool_entries_by_addresses_call, GetMempoolEntriesByAddresses); route!(get_coin_supply_call, GetCoinSupply); route!(get_daa_score_timestamp_estimate_call, GetDaaScoreTimestampEstimate); - - // TODO (PR) - async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { - Err(RpcError::NotImplemented) - } - async fn get_fee_estimate_experimental_call( - &self, - _request: GetFeeEstimateExperimentalRequest, - ) -> RpcResult { - Err(RpcError::NotImplemented) - } + route!(get_fee_estimate_call, GetFeeEstimate); + route!(get_fee_estimate_experimental_call, GetFeeEstimateExperimental); // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/grpc/core/proto/messages.proto b/rpc/grpc/core/proto/messages.proto index 0358f1d19..01075c183 100644 --- a/rpc/grpc/core/proto/messages.proto +++ b/rpc/grpc/core/proto/messages.proto @@ -60,6 +60,8 @@ message KaspadRequest { GetSyncStatusRequestMessage getSyncStatusRequest = 1094; GetDaaScoreTimestampEstimateRequestMessage getDaaScoreTimestampEstimateRequest = 1096; SubmitTransactionReplacementRequestMessage submitTransactionReplacementRequest = 2000; + GetFeeEstimateRequestMessage getFeeEstimateRequest = 1106; + GetFeeEstimateExperimentalRequestMessage getFeeEstimateExperimentalRequest = 1108; } } @@ -120,6 +122,8 @@ message KaspadResponse { GetSyncStatusResponseMessage getSyncStatusResponse = 1095; GetDaaScoreTimestampEstimateResponseMessage getDaaScoreTimestampEstimateResponse = 1097; SubmitTransactionReplacementResponseMessage submitTransactionReplacementResponse = 2001; + GetFeeEstimateResponseMessage getFeeEstimateResponse = 1107; + GetFeeEstimateExperimentalResponseMessage getFeeEstimateExperimentalResponse = 1109; } } diff --git a/rpc/grpc/core/proto/rpc.proto b/rpc/grpc/core/proto/rpc.proto index 52ecfa669..dd19e2546 100644 --- a/rpc/grpc/core/proto/rpc.proto +++ b/rpc/grpc/core/proto/rpc.proto @@ -866,3 +866,43 @@ message GetDaaScoreTimestampEstimateResponseMessage{ repeated uint64 timestamps = 1; RPCError error = 1000; } + +message RpcFeerateBucket { + double feerate = 1; + double estimated_seconds = 2; +} + +message RpcFeeEstimate { + RpcFeerateBucket priority_bucket = 1; + repeated RpcFeerateBucket normal_buckets = 2; + repeated RpcFeerateBucket low_buckets = 3; +} + +message RpcFeeEstimateVerboseExperimentalData { + uint64 mempool_ready_transactions_count = 1; + uint64 mempool_ready_transactions_total_mass = 2; + uint64 network_mass_per_second = 3; + + double next_block_template_feerate_min = 4; + double next_block_template_feerate_median = 5; + double next_block_template_feerate_max = 6; +} + +message GetFeeEstimateRequestMessage { +} + +message GetFeeEstimateResponseMessage { + RpcFeeEstimate estimate = 1; + RPCError error = 1000; +} + +message GetFeeEstimateExperimentalRequestMessage { + bool verbose = 1; +} + +message GetFeeEstimateExperimentalResponseMessage { + RpcFeeEstimate estimate = 1; + RpcFeeEstimateVerboseExperimentalData verbose = 2; + + RPCError error = 1000; +} diff --git a/rpc/grpc/core/src/convert/feerate_estimate.rs b/rpc/grpc/core/src/convert/feerate_estimate.rs new file mode 100644 index 000000000..a71ad14d1 --- /dev/null +++ b/rpc/grpc/core/src/convert/feerate_estimate.rs @@ -0,0 +1,66 @@ +use crate::protowire; +use crate::{from, try_from}; +use kaspa_rpc_core::RpcError; + +// ---------------------------------------------------------------------------- +// rpc_core to protowire +// ---------------------------------------------------------------------------- + +from!(item: &kaspa_rpc_core::FeerateBucket, protowire::RpcFeerateBucket, { + Self { + feerate: item.feerate, + estimated_seconds: item.estimated_seconds, + } +}); + +from!(item: &kaspa_rpc_core::FeeEstimate, protowire::RpcFeeEstimate, { + Self { + priority_bucket: Some((&item.priority_bucket).into()), + normal_buckets: item.normal_buckets.iter().map(|b| b.into()).collect(), + low_buckets: item.low_buckets.iter().map(|b| b.into()).collect(), + } +}); + +from!(item: &kaspa_rpc_core::FeeEstimateVerboseExperimentalData, protowire::RpcFeeEstimateVerboseExperimentalData, { + Self { + network_mass_per_second: item.network_mass_per_second, + mempool_ready_transactions_count: item.mempool_ready_transactions_count, + mempool_ready_transactions_total_mass: item.mempool_ready_transactions_total_mass, + next_block_template_feerate_min: item.next_block_template_feerate_min, + next_block_template_feerate_median: item.next_block_template_feerate_median, + next_block_template_feerate_max: item.next_block_template_feerate_max, + } +}); + +// ---------------------------------------------------------------------------- +// protowire to rpc_core +// ---------------------------------------------------------------------------- + +try_from!(item: &protowire::RpcFeerateBucket, kaspa_rpc_core::FeerateBucket, { + Self { + feerate: item.feerate, + estimated_seconds: item.estimated_seconds, + } +}); + +try_from!(item: &protowire::RpcFeeEstimate, kaspa_rpc_core::FeeEstimate, { + Self { + priority_bucket: item.priority_bucket + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("RpcFeeEstimate".to_string(), "priority_bucket".to_string()))? + .try_into()?, + normal_buckets: item.normal_buckets.iter().map(|b| b.try_into()).collect::, _>>()?, + low_buckets: item.low_buckets.iter().map(|b| b.try_into()).collect::, _>>()?, + } +}); + +try_from!(item: &protowire::RpcFeeEstimateVerboseExperimentalData, kaspa_rpc_core::FeeEstimateVerboseExperimentalData, { + Self { + network_mass_per_second: item.network_mass_per_second, + mempool_ready_transactions_count: item.mempool_ready_transactions_count, + mempool_ready_transactions_total_mass: item.mempool_ready_transactions_total_mass, + next_block_template_feerate_min: item.next_block_template_feerate_min, + next_block_template_feerate_median: item.next_block_template_feerate_median, + next_block_template_feerate_max: item.next_block_template_feerate_max, + } +}); diff --git a/rpc/grpc/core/src/convert/kaspad.rs b/rpc/grpc/core/src/convert/kaspad.rs index 0fc81e0f1..d43442fe3 100644 --- a/rpc/grpc/core/src/convert/kaspad.rs +++ b/rpc/grpc/core/src/convert/kaspad.rs @@ -58,6 +58,8 @@ pub mod kaspad_request_convert { impl_into_kaspad_request!(GetServerInfo); impl_into_kaspad_request!(GetSyncStatus); impl_into_kaspad_request!(GetDaaScoreTimestampEstimate); + impl_into_kaspad_request!(GetFeeEstimate); + impl_into_kaspad_request!(GetFeeEstimateExperimental); impl_into_kaspad_request!(NotifyBlockAdded); impl_into_kaspad_request!(NotifyNewBlockTemplate); @@ -190,6 +192,8 @@ pub mod kaspad_response_convert { impl_into_kaspad_response!(GetServerInfo); impl_into_kaspad_response!(GetSyncStatus); impl_into_kaspad_response!(GetDaaScoreTimestampEstimate); + impl_into_kaspad_response!(GetFeeEstimate); + impl_into_kaspad_response!(GetFeeEstimateExperimental); impl_into_kaspad_notify_response!(NotifyBlockAdded); impl_into_kaspad_notify_response!(NotifyNewBlockTemplate); diff --git a/rpc/grpc/core/src/convert/message.rs b/rpc/grpc/core/src/convert/message.rs index d005c8c79..75606dc9a 100644 --- a/rpc/grpc/core/src/convert/message.rs +++ b/rpc/grpc/core/src/convert/message.rs @@ -401,6 +401,25 @@ from!(item: RpcResult<&kaspa_rpc_core::GetDaaScoreTimestampEstimateResponse>, pr Self { timestamps: item.timestamps.clone(), error: None } }); +// Fee estimate API + +from!(&kaspa_rpc_core::GetFeeEstimateRequest, protowire::GetFeeEstimateRequestMessage); +from!(item: RpcResult<&kaspa_rpc_core::GetFeeEstimateResponse>, protowire::GetFeeEstimateResponseMessage, { + Self { estimate: Some((&item.estimate).into()), error: None } +}); +from!(item: &kaspa_rpc_core::GetFeeEstimateExperimentalRequest, protowire::GetFeeEstimateExperimentalRequestMessage, { + Self { + verbose: item.verbose + } +}); +from!(item: RpcResult<&kaspa_rpc_core::GetFeeEstimateExperimentalResponse>, protowire::GetFeeEstimateExperimentalResponseMessage, { + Self { + estimate: Some((&item.estimate).into()), + verbose: item.verbose.as_ref().map(|x| x.into()), + error: None + } +}); + from!(&kaspa_rpc_core::PingRequest, protowire::PingRequestMessage); from!(RpcResult<&kaspa_rpc_core::PingResponse>, protowire::PingResponseMessage); @@ -818,6 +837,30 @@ try_from!(item: &protowire::GetDaaScoreTimestampEstimateResponseMessage, RpcResu Self { timestamps: item.timestamps.clone() } }); +try_from!(&protowire::GetFeeEstimateRequestMessage, kaspa_rpc_core::GetFeeEstimateRequest); +try_from!(item: &protowire::GetFeeEstimateResponseMessage, RpcResult, { + Self { + estimate: item.estimate + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("GetFeeEstimateResponseMessage".to_string(), "estimate".to_string()))? + .try_into()? + } +}); +try_from!(item: &protowire::GetFeeEstimateExperimentalRequestMessage, kaspa_rpc_core::GetFeeEstimateExperimentalRequest, { + Self { + verbose: item.verbose + } +}); +try_from!(item: &protowire::GetFeeEstimateExperimentalResponseMessage, RpcResult, { + Self { + estimate: item.estimate + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("GetFeeEstimateExperimentalResponseMessage".to_string(), "estimate".to_string()))? + .try_into()?, + verbose: item.verbose.as_ref().map(|x| x.try_into()).transpose()? + } +}); + try_from!(&protowire::PingRequestMessage, kaspa_rpc_core::PingRequest); try_from!(&protowire::PingResponseMessage, RpcResult); diff --git a/rpc/grpc/core/src/convert/mod.rs b/rpc/grpc/core/src/convert/mod.rs index 2f3252d22..d4948f57d 100644 --- a/rpc/grpc/core/src/convert/mod.rs +++ b/rpc/grpc/core/src/convert/mod.rs @@ -1,6 +1,7 @@ pub mod address; pub mod block; pub mod error; +pub mod feerate_estimate; pub mod header; pub mod kaspad; pub mod mempool; diff --git a/rpc/grpc/core/src/ops.rs b/rpc/grpc/core/src/ops.rs index 20ebf6ab4..605d27efd 100644 --- a/rpc/grpc/core/src/ops.rs +++ b/rpc/grpc/core/src/ops.rs @@ -82,6 +82,8 @@ pub enum KaspadPayloadOps { GetServerInfo, GetSyncStatus, GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, // Subscription commands for starting/stopping notifications NotifyBlockAdded, diff --git a/rpc/grpc/server/src/request_handler/factory.rs b/rpc/grpc/server/src/request_handler/factory.rs index c598b759c..a70fb629f 100644 --- a/rpc/grpc/server/src/request_handler/factory.rs +++ b/rpc/grpc/server/src/request_handler/factory.rs @@ -76,6 +76,8 @@ impl Factory { GetServerInfo, GetSyncStatus, GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, NotifyBlockAdded, NotifyNewBlockTemplate, NotifyFinalityConflict, diff --git a/testing/integration/src/rpc_tests.rs b/testing/integration/src/rpc_tests.rs index c584e6b64..dc26b539f 100644 --- a/testing/integration/src/rpc_tests.rs +++ b/testing/integration/src/rpc_tests.rs @@ -557,6 +557,33 @@ async fn sanity_test() { }) } + KaspadPayloadOps::GetFeeEstimate => { + let rpc_client = client.clone(); + tst!(op, { + let response = rpc_client.get_fee_estimate().await.unwrap(); + info!("{:?}", response.priority_bucket); + assert!(!response.normal_buckets.is_empty()); + assert!(!response.low_buckets.is_empty()); + for bucket in response.ordered_buckets() { + info!("{:?}", bucket); + } + }) + } + + KaspadPayloadOps::GetFeeEstimateExperimental => { + let rpc_client = client.clone(); + tst!(op, { + let response = rpc_client.get_fee_estimate_experimental(true).await.unwrap(); + assert!(!response.estimate.normal_buckets.is_empty()); + assert!(!response.estimate.low_buckets.is_empty()); + for bucket in response.estimate.ordered_buckets() { + info!("{:?}", bucket); + } + assert!(response.verbose.is_some()); + info!("{:?}", response.verbose); + }) + } + KaspadPayloadOps::NotifyBlockAdded => { let rpc_client = client.clone(); let id = listener_id; From 18a8fc45349f7cb0e919fd314a67a6ff3eb768f6 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 11 Aug 2024 12:46:54 +0000 Subject: [PATCH 57/74] Benchmark worst-case collision cases + an optimization addressing these cases --- mining/benches/bench.rs | 66 +++++++++++++++++-- mining/src/mempool/model/frontier.rs | 35 ++++++---- .../src/mempool/model/frontier/feerate_key.rs | 3 +- 3 files changed, 83 insertions(+), 21 deletions(-) diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index 4a605b8b7..16cfcc234 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -109,7 +109,7 @@ pub fn bench_mempool_sampling(c: &mut Criterion) { group.bench_function("mempool one-shot sample", |b| { b.iter(|| { black_box({ - let selected = frontier.sample_inplace(&mut rng, &Policy::new(500_000)); + let selected = frontier.sample_inplace(&mut rng, &Policy::new(500_000), &mut 0); selected.iter().map(|k| k.mass).sum::() }) }) @@ -170,9 +170,9 @@ pub fn bench_mempool_sampling(c: &mut Criterion) { group.finish(); } -pub fn bench_mempool_sampling_small(c: &mut Criterion) { +pub fn bench_mempool_selectors(c: &mut Criterion) { let mut rng = thread_rng(); - let mut group = c.benchmark_group("mempool sampling small"); + let mut group = c.benchmark_group("mempool selectors"); let cap = 1_000_000; let mut map = HashMap::with_capacity(cap); for i in 0..cap as u64 { @@ -182,7 +182,7 @@ pub fn bench_mempool_sampling_small(c: &mut Criterion) { map.insert(key.tx.id(), key); } - for len in [100, 300, 350, 500, 1000, 2000, 5000, 10_000, 100_000, 500_000, 1_000_000] { + for len in [100, 300, 350, 500, 1000, 2000, 5000, 10_000, 100_000, 500_000, 1_000_000].into_iter().rev() { let mut frontier = Frontier::default(); for item in map.values().take(len).cloned() { frontier.insert(item).then_some(()).unwrap(); @@ -197,15 +197,23 @@ pub fn bench_mempool_sampling_small(c: &mut Criterion) { }) }); + let mut collisions = 0; + let mut n = 0; + group.bench_function(format!("sample inplace selector ({})", len), |b| { b.iter(|| { black_box({ - let mut selector = frontier.build_selector_sample_inplace(); + let mut selector = frontier.build_selector_sample_inplace(&mut collisions); + n += 1; selector.select_transactions().iter().map(|k| k.gas).sum::() }) }) }); + if n > 0 { + println!("---------------------- \n Avg collisions: {}", collisions / n); + } + if frontier.total_mass() <= 500_000 { group.bench_function(format!("take all selector ({})", len), |b| { b.iter(|| { @@ -230,5 +238,51 @@ pub fn bench_mempool_sampling_small(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_mempool_sampling, bench_mempool_sampling_small, bench_compare_topological_index_fns); +pub fn bench_inplace_sampling_worst_case(c: &mut Criterion) { + let mut group = c.benchmark_group("mempool inplace sampling"); + let max_fee = u64::MAX; + let fee_steps = (0..10).map(|i| max_fee / 100u64.pow(i)).collect_vec(); + for subgroup_size in [300, 200, 100, 80, 50, 30] { + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = if i < 300 { fee_steps[i as usize / subgroup_size] } else { 1 }; + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let mut frontier = Frontier::default(); + for item in map.values().cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let mut collisions = 0; + let mut n = 0; + + group.bench_function(format!("inplace sampling worst case (subgroup size: {})", subgroup_size), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_sample_inplace(&mut collisions); + n += 1; + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + if n > 0 { + println!("---------------------- \n Avg collisions: {}", collisions / n); + } + } + + group.finish(); +} + +criterion_group!( + benches, + bench_mempool_sampling, + bench_mempool_selectors, + bench_inplace_sampling_worst_case, + bench_compare_topological_index_fns +); criterion_main!(benches); diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index bebefb540..f0b0f8839 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -104,7 +104,7 @@ impl Frontier { /// By using floating point arithmetics we gain the adjustment of the probability space to the accuracy level required for /// current samples. And if the space is highly biased, the repeated elimination of top items and the prefix weight computation /// will readjust it. - pub fn sample_inplace(&self, rng: &mut R, policy: &Policy) -> SequenceSelectorInput + pub fn sample_inplace(&self, rng: &mut R, policy: &Policy, _collisions: &mut u64) -> SequenceSelectorInput where R: Rng + ?Sized, { @@ -122,7 +122,7 @@ impl Frontier { let mut cache = HashSet::new(); let mut sequence = SequenceSelectorInput::default(); let mut total_selected_mass: u64 = 0; - let mut _collisions = 0; + let mut collisions = 0; // The sampling process is converging so the cache will eventually hold all entries, which guarantees loop exit 'outer: while cache.len() < self.search_tree.len() && total_selected_mass <= desired_mass { @@ -130,12 +130,18 @@ impl Frontier { let item = { let mut item = self.search_tree.search(query); while !cache.insert(item.tx.id()) { - _collisions += 1; - if top == item { - // Narrow the search to reduce further sampling collisions - match down_iter.next() { - Some(next) => top = next, - None => break 'outer, + collisions += 1; + // Try to narrow the sampling space in order to reduce further sampling collisions + if cache.contains(&top.tx.id()) { + loop { + match down_iter.next() { + Some(next) => top = next, + None => break 'outer, + } + // Loop until finding a top item which was not sampled yet + if !cache.contains(&top.tx.id()) { + break; + } } let remaining_weight = self.search_tree.prefix_weight(top); distr = Uniform::new(0f64, remaining_weight); @@ -148,7 +154,8 @@ impl Frontier { sequence.push(item.tx.clone(), item.mass); total_selected_mass += item.mass; // Max standard mass + Mempool capacity bound imply this will not overflow } - trace!("[mempool frontier sample inplace] collisions: {_collisions}, cache: {}", cache.len()); + trace!("[mempool frontier sample inplace] collisions: {collisions}, cache: {}", cache.len()); + *_collisions += collisions; sequence } @@ -171,7 +178,7 @@ impl Frontier { Box::new(TakeAllSelector::new(self.search_tree.ascending_iter().map(|k| k.tx.clone()).collect())) } else if self.total_mass > policy.max_block_mass * COLLISION_FACTOR { let mut rng = rand::thread_rng(); - Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, policy), policy.clone())) + Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, policy, &mut 0), policy.clone())) } else { Box::new(RebalancingWeightedTransactionSelector::new( policy.clone(), @@ -181,10 +188,10 @@ impl Frontier { } /// Exposed for benchmarking purposes - pub fn build_selector_sample_inplace(&self) -> Box { + pub fn build_selector_sample_inplace(&self, _collisions: &mut u64) -> Box { let mut rng = rand::thread_rng(); let policy = Policy::new(500_000); - Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, &policy), policy)) + Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, &policy, _collisions), policy)) } /// Exposed for benchmarking purposes @@ -275,7 +282,7 @@ mod tests { frontier.insert(item).then_some(()).unwrap(); } - let _sample = frontier.sample_inplace(&mut rng, &Policy::new(500_000)); + let _sample = frontier.sample_inplace(&mut rng, &Policy::new(500_000), &mut 0); // assert_eq!(100, sample.len()); } @@ -303,7 +310,7 @@ mod tests { let mut selector = frontier.build_rebalancing_selector(); selector.select_transactions().iter().map(|k| k.gas).sum::(); - let mut selector = frontier.build_selector_sample_inplace(); + let mut selector = frontier.build_selector_sample_inplace(&mut 0); selector.select_transactions().iter().map(|k| k.gas).sum::(); let mut selector = frontier.build_selector_take_all(); diff --git a/mining/src/mempool/model/frontier/feerate_key.rs b/mining/src/mempool/model/frontier/feerate_key.rs index fd2cca091..843ef0ff1 100644 --- a/mining/src/mempool/model/frontier/feerate_key.rs +++ b/mining/src/mempool/model/frontier/feerate_key.rs @@ -78,8 +78,9 @@ impl Ord for FeerateTransactionKey { impl From<&MempoolTransaction> for FeerateTransactionKey { fn from(tx: &MempoolTransaction) -> Self { let mass = tx.mtx.tx.mass(); + let fee = tx.mtx.calculated_fee.expect("fee is expected to be populated"); assert_ne!(mass, 0, "mass field is expected to be set when inserting to the mempool"); - Self::new(tx.mtx.calculated_fee.expect("fee is expected to be populated"), mass, tx.mtx.tx.clone()) + Self::new(fee, mass, tx.mtx.tx.clone()) } } From 40a1df5f32514978661d193cc9523510f89e6486 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 11 Aug 2024 14:31:27 +0000 Subject: [PATCH 58/74] Expose SearchTree --- mining/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mining/src/lib.rs b/mining/src/lib.rs index d1e2cf69a..745fb63f9 100644 --- a/mining/src/lib.rs +++ b/mining/src/lib.rs @@ -17,7 +17,7 @@ pub mod monitor; // Exposed for benchmarks pub use block_template::{policy::Policy, selector::RebalancingWeightedTransactionSelector}; -pub use mempool::model::frontier::{feerate_key::FeerateTransactionKey, Frontier}; +pub use mempool::model::frontier::{feerate_key::FeerateTransactionKey, search_tree::SearchTree, Frontier}; #[cfg(test)] pub mod testutils; From 00509a5915539baf130e9f2bf7d56c7ca0c7f829 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Sun, 11 Aug 2024 14:41:26 +0000 Subject: [PATCH 59/74] cli support (with @coderofstuff) --- cli/src/modules/rpc.rs | 9 +++++++++ rpc/wrpc/server/src/router.rs | 2 ++ 2 files changed, 11 insertions(+) diff --git a/cli/src/modules/rpc.rs b/cli/src/modules/rpc.rs index c84915480..53478c174 100644 --- a/cli/src/modules/rpc.rs +++ b/cli/src/modules/rpc.rs @@ -229,6 +229,15 @@ impl Rpc { } } } + RpcApiOps::GetFeeEstimate => { + let result = rpc.get_fee_estimate_call(GetFeeEstimateRequest {}).await?; + self.println(&ctx, result); + } + RpcApiOps::GetFeeEstimateExperimental => { + let verbose = if argv.is_empty() { false } else { argv.remove(0).parse().unwrap_or(false) }; + let result = rpc.get_fee_estimate_experimental_call(GetFeeEstimateExperimentalRequest { verbose }).await?; + self.println(&ctx, result); + } _ => { tprintln!(ctx, "rpc method exists but is not supported by the cli: '{op_str}'\r\n"); return Ok(()); diff --git a/rpc/wrpc/server/src/router.rs b/rpc/wrpc/server/src/router.rs index 4c0d2b3aa..024f6a9c6 100644 --- a/rpc/wrpc/server/src/router.rs +++ b/rpc/wrpc/server/src/router.rs @@ -68,6 +68,8 @@ impl Router { SubmitTransaction, SubmitTransactionReplacement, Unban, + GetFeeEstimate, + GetFeeEstimateExperimental, ] ); From 715bcf28a0d417a2738a2be027d28d0ab4aa7f4e Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Tue, 13 Aug 2024 18:32:30 +0000 Subject: [PATCH 60/74] addressing a few minor review comments --- Cargo.lock | 1 - mining/Cargo.toml | 3 +-- mining/src/mempool/model/frontier.rs | 7 ++----- mining/src/mempool/model/frontier/selectors.rs | 2 ++ 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77a27aca3..bcd372013 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3031,7 +3031,6 @@ version = "0.14.1" dependencies = [ "criterion", "futures-util", - "indexmap 2.2.6", "itertools 0.11.0", "kaspa-addresses", "kaspa-consensus-core", diff --git a/mining/Cargo.toml b/mining/Cargo.toml index 4fa786cda..0c7eb2525 100644 --- a/mining/Cargo.toml +++ b/mining/Cargo.toml @@ -27,10 +27,9 @@ parking_lot.workspace = true rand.workspace = true serde.workspace = true smallvec.workspace = true +sweep-bptree = "0.4.1" thiserror.workspace = true -indexmap.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "macros", "signal"] } -sweep-bptree = "0.4.1" [dev-dependencies] kaspa-txscript.workspace = true diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index f0b0f8839..f185c23a5 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -276,14 +276,12 @@ mod tests { map.insert(key.tx.id(), key); } - let len = cap; let mut frontier = Frontier::default(); - for item in map.values().take(len).cloned() { + for item in map.values().cloned() { frontier.insert(item).then_some(()).unwrap(); } let _sample = frontier.sample_inplace(&mut rng, &Policy::new(500_000), &mut 0); - // assert_eq!(100, sample.len()); } #[test] @@ -298,9 +296,8 @@ mod tests { map.insert(key.tx.id(), key); } - let len = cap; let mut frontier = Frontier::default(); - for item in map.values().take(len).cloned() { + for item in map.values().cloned() { frontier.insert(item).then_some(()).unwrap(); } diff --git a/mining/src/mempool/model/frontier/selectors.rs b/mining/src/mempool/model/frontier/selectors.rs index fc8e35efe..a30ecc145 100644 --- a/mining/src/mempool/model/frontier/selectors.rs +++ b/mining/src/mempool/model/frontier/selectors.rs @@ -59,6 +59,7 @@ struct SequenceSelectorSelection { pub struct SequenceSelector { input_sequence: SequenceSelectorInput, selected_vec: Vec, + /// Maps from selected tx ids to tx mass so that the total used mass can be subtracted on tx reject selected_map: Option>, total_selected_mass: u64, overall_candidates: usize, @@ -114,6 +115,7 @@ impl TemplateTransactionSelector for SequenceSelector { // Lazy-create the map only when there are actual rejections let selected_map = self.selected_map.get_or_insert_with(|| self.selected_vec.iter().map(|tx| (tx.tx_id, tx.mass)).collect()); let mass = selected_map.remove(&tx_id).expect("only previously selected txs can be rejected (and only once)"); + // Selections must be counted in total selected mass, so this subtraction cannot underflow self.total_selected_mass -= mass; self.overall_rejections += 1; } From d74f831a36c47608ef62afc6f846218f87f67222 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Tue, 13 Aug 2024 21:32:56 +0000 Subject: [PATCH 61/74] feerate estimator - handle various edge cases (with @tiram88) --- mining/src/feerate/mod.rs | 58 +++++++++++++++++++++++++++- mining/src/mempool/model/frontier.rs | 27 ++++++++++--- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index 70168a04a..bb6f12b08 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -50,11 +50,13 @@ impl FeerateEstimatorArgs { } } +#[derive(Debug, Clone)] pub struct FeerateEstimator { - /// The total probability weight of all current mempool ready transactions, i.e., Σ_{tx in mempool}(tx.fee/tx.mass)^alpha + /// The total probability weight of current mempool ready transactions, i.e., `Σ_{tx in mempool}(tx.fee/tx.mass)^alpha`. + /// Note that some estimators might consider a reduced weight which excludes outliers. See [`Frontier::build_feerate_estimator`] total_weight: f64, - /// The amortized time between transactions given the current transaction masses present in the mempool. Or in + /// The amortized time **in seconds** between transactions, given the current transaction masses present in the mempool. Or in /// other words, the inverse of the transaction inclusion rate. For instance, if the average transaction mass is 2500 grams, /// the block mass limit is 500,000 and the network has 10 BPS, then this number would be 1/2000 seconds. inclusion_interval: f64, @@ -62,6 +64,8 @@ pub struct FeerateEstimator { impl FeerateEstimator { pub fn new(total_weight: f64, inclusion_interval: f64) -> Self { + assert!(total_weight >= 0.0); + assert!((0f64..1f64).contains(&inclusion_interval)); Self { total_weight, inclusion_interval } } @@ -72,29 +76,48 @@ impl FeerateEstimator { fn time_to_feerate(&self, time: f64) -> f64 { let (c1, c2) = (self.inclusion_interval, self.total_weight); + assert!(c1 < time, "{c1}, {time}"); ((c1 * c2 / time) / (1f64 - c1 / time)).powf(1f64 / ALPHA as f64) } /// The antiderivative function of [`feerate_to_time`] excluding the constant shift `+ c1` + #[inline] fn feerate_to_time_antiderivative(&self, feerate: f64) -> f64 { let (c1, c2) = (self.inclusion_interval, self.total_weight); c1 * c2 / (-2f64 * feerate.powi(ALPHA - 1)) } + /// Returns the feerate value for which the integral area is `frac` of the total area between `lower` and `upper`. fn quantile(&self, lower: f64, upper: f64, frac: f64) -> f64 { assert!((0f64..=1f64).contains(&frac)); + assert!(0.0 < lower && lower <= upper, "{lower}, {upper}"); let (c1, c2) = (self.inclusion_interval, self.total_weight); + if c1 == 0.0 || c2 == 0.0 { + // if c1 · c2 == 0.0, the integral area is empty, so we simply return `lower` + return lower; + } let z1 = self.feerate_to_time_antiderivative(lower); let z2 = self.feerate_to_time_antiderivative(upper); + // Get the total area corresponding to `frac` of the integral area between `lower` and `upper` + // which can be expressed as z1 + frac * (z2 - z1) let z = frac * z2 + (1f64 - frac) * z1; + // Calc the x value (feerate) corresponding to said area ((c1 * c2) / (-2f64 * z)).powf(1f64 / (ALPHA - 1) as f64) } pub fn calc_estimations(&self, minimum_standard_feerate: f64) -> FeerateEstimations { let min = minimum_standard_feerate; + // Choose `high` such that it provides sub-second waiting time let high = self.time_to_feerate(1f64).max(min); + // Choose `low` feerate such that it provides sub-hour waiting time AND it covers (at least) the 0.25 quantile let low = self.time_to_feerate(3600f64).max(self.quantile(min, high, 0.25)); + // Choose `mid` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.5 quantile between low and high. let mid = self.time_to_feerate(60f64).max(self.quantile(low, high, 0.5)); + /* Intuition for the above: + 1. The quantile calculations make sure that we return interesting points on the `feerate_to_time` curve. + 2. They also ensure that the times don't diminish too high if small increments to feerate would suffice + to cover large fractions of the integral area (reflecting the position within the waiting-time distribution) + */ FeerateEstimations { priority_bucket: FeerateBucket { feerate: high, estimated_seconds: self.feerate_to_time(high) }, normal_buckets: vec![FeerateBucket { feerate: mid, estimated_seconds: self.feerate_to_time(mid) }], @@ -141,6 +164,37 @@ mod tests { assert!(buckets.last().unwrap().feerate >= minimum_feerate); for (i, j) in buckets.into_iter().tuple_windows() { assert!(i.feerate >= j.feerate); + assert!(i.estimated_seconds <= j.estimated_seconds); + } + } + + #[test] + fn test_zero_values() { + let estimator = FeerateEstimator { total_weight: 0.0, inclusion_interval: 0.0 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + let buckets = estimations.ordered_buckets(); + for bucket in buckets { + assert_eq!(minimum_feerate, bucket.feerate); + assert_eq!(0.0, bucket.estimated_seconds); + } + + let estimator = FeerateEstimator { total_weight: 0.0, inclusion_interval: 0.1 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + let buckets = estimations.ordered_buckets(); + for bucket in buckets { + assert_eq!(minimum_feerate, bucket.feerate); + assert_eq!(estimator.inclusion_interval, bucket.estimated_seconds); + } + + let estimator = FeerateEstimator { total_weight: 0.1, inclusion_interval: 0.0 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + let buckets = estimations.ordered_buckets(); + for bucket in buckets { + assert_eq!(minimum_feerate, bucket.feerate); + assert_eq!(0.0, bucket.estimated_seconds); } } } diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index f185c23a5..93c911ec6 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -240,12 +240,13 @@ impl Frontier { // Update values for the next iteration. In order to remove the outlier from the // total weight, we must compensate by capturing a block slot. mass_per_block -= average_transaction_mass; - if mass_per_block <= 0.0 { - // Out of block slots, break (this is rarely reachable code due to dynamics related to the above break) + if mass_per_block <= average_transaction_mass { + // Out of block slots, break break; } - // Re-calc the inclusion interval based on the new block "capacity" + // Re-calc the inclusion interval based on the new block "capacity". + // Note that inclusion_interval < 1.0 as required by the estimator, since mass_per_block > average_transaction_mass (by condition above) and bps >= 1 inclusion_interval = average_transaction_mass / (mass_per_block * bps); } estimator @@ -379,7 +380,7 @@ mod tests { map.insert(key.tx.id(), key); } - for len in [10, 100, 200, 300, 500, 750, cap / 2, (cap * 2) / 3, (cap * 4) / 5, (cap * 5) / 6, cap] { + for len in [0, 10, 100, 200, 300, 500, 750, cap / 2, (cap * 2) / 3, (cap * 4) / 5, (cap * 5) / 6, cap] { let mut frontier = Frontier::default(); for item in map.values().take(len).cloned() { frontier.insert(item).then_some(()).unwrap(); @@ -388,8 +389,22 @@ mod tests { let args = FeerateEstimatorArgs { network_blocks_per_second: 1, maximum_mass_per_block: 500_000 }; // We are testing that the build function actually returns and is not looping indefinitely let estimator = frontier.build_feerate_estimator(args); - let _estimations = estimator.calc_estimations(1.0); - // dbg!(_estimations); + dbg!(len, estimator.clone()); + let estimations = estimator.calc_estimations(1.0); + dbg!(estimations.clone()); + + let buckets = estimations.ordered_buckets(); + // Test for the absence of NaN, infinite or zero values in buckets + for b in buckets.iter() { + assert!( + b.feerate.is_normal() && b.feerate >= 1.0, + "bucket feerate must be a finite number greater or equal to the minimum standard feerate" + ); + assert!( + b.estimated_seconds.is_normal() && b.estimated_seconds > 0.0, + "bucket estimated seconds must be a finite number greater than zero" + ); + } } } } From 073d6b350d8c105f02d820cfccaa410ec34911b0 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Tue, 13 Aug 2024 22:02:09 +0000 Subject: [PATCH 62/74] one more test (with @tiram88) --- mining/src/mempool/model/frontier.rs | 43 +++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 93c911ec6..29fe9cc97 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -380,7 +380,7 @@ mod tests { map.insert(key.tx.id(), key); } - for len in [0, 10, 100, 200, 300, 500, 750, cap / 2, (cap * 2) / 3, (cap * 4) / 5, (cap * 5) / 6, cap] { + for len in [0, 1, 10, 100, 200, 300, 500, 750, cap / 2, (cap * 2) / 3, (cap * 4) / 5, (cap * 5) / 6, cap] { let mut frontier = Frontier::default(); for item in map.values().take(len).cloned() { frontier.insert(item).then_some(()).unwrap(); @@ -407,4 +407,45 @@ mod tests { } } } + + #[test] + fn test_constant_feerate_estimator() { + const MIN_FEERATE: f64 = 1.0; + let cap = 20_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let mass: u64 = 1650; + let fee = (mass as f64 * MIN_FEERATE) as u64; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + for len in [0, 1, 10, 100, 200, 300, 500, 750, cap / 2, (cap * 2) / 3, (cap * 4) / 5, (cap * 5) / 6, cap] { + println!(); + println!("Testing a frontier with {} txs...", len.min(cap)); + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let args = FeerateEstimatorArgs { network_blocks_per_second: 1, maximum_mass_per_block: 500_000 }; + // We are testing that the build function actually returns and is not looping indefinitely + let estimator = frontier.build_feerate_estimator(args); + let estimations = estimator.calc_estimations(MIN_FEERATE); + let buckets = estimations.ordered_buckets(); + // Test for the absence of NaN, infinite or zero values in buckets + for b in buckets.iter() { + assert!( + b.feerate.is_normal() && b.feerate >= 1.0, + "bucket feerate must be a finite number greater or equal to the minimum standard feerate" + ); + assert!( + b.estimated_seconds.is_normal() && b.estimated_seconds > 0.0, + "bucket estimated seconds must be a finite number greater than zero" + ); + } + // dbg!(estimations); + // dbg!(estimator); + } + } } From 9faff73f2b00ba69d77942588199c702c08433b0 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Wed, 14 Aug 2024 14:33:06 +0000 Subject: [PATCH 63/74] build_feerate_estimator - fix edge case of not trying the estimator without all frontier txs (+loop logic is more streamlined now) --- mining/src/mempool/model/frontier.rs | 42 ++++++++++--------- .../src/mempool/model/frontier/search_tree.rs | 5 ++- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 29fe9cc97..442d19f14 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -218,14 +218,24 @@ impl Frontier { let mut inclusion_interval = average_transaction_mass / (mass_per_block * bps); let mut estimator = FeerateEstimator::new(self.total_weight(), inclusion_interval); - // Corresponds to the removal of the top item, hence the skip(1) below - mass_per_block -= average_transaction_mass; - inclusion_interval = average_transaction_mass / (mass_per_block * bps); - // Search for better estimators by possibly removing extremely high outliers - for key in self.search_tree.descending_iter().skip(1) { - // Compute the weight up to, and including, current key - let prefix_weight = self.search_tree.prefix_weight(key); + let mut down_iter = self.search_tree.descending_iter().skip(1); + loop { + // Update values for the next iteration. In order to remove the outlier from the + // total weight, we must compensate by capturing a block slot. + mass_per_block -= average_transaction_mass; + if mass_per_block <= average_transaction_mass { + // Out of block slots, break + break; + } + + // Re-calc the inclusion interval based on the new block "capacity". + // Note that inclusion_interval < 1.0 as required by the estimator, since mass_per_block > average_transaction_mass (by condition above) and bps >= 1 + inclusion_interval = average_transaction_mass / (mass_per_block * bps); + + // Compute the weight up to, and including, current key (or use zero weight if next is none) + let next = down_iter.next(); + let prefix_weight = next.map(|key| self.search_tree.prefix_weight(key)).unwrap_or_default(); let pending_estimator = FeerateEstimator::new(prefix_weight, inclusion_interval); // Test the pending estimator vs. the current one @@ -237,17 +247,9 @@ impl Frontier { break; } - // Update values for the next iteration. In order to remove the outlier from the - // total weight, we must compensate by capturing a block slot. - mass_per_block -= average_transaction_mass; - if mass_per_block <= average_transaction_mass { - // Out of block slots, break + if next.is_none() { break; } - - // Re-calc the inclusion interval based on the new block "capacity". - // Note that inclusion_interval < 1.0 as required by the estimator, since mass_per_block > average_transaction_mass (by condition above) and bps >= 1 - inclusion_interval = average_transaction_mass / (mass_per_block * bps); } estimator } @@ -389,9 +391,7 @@ mod tests { let args = FeerateEstimatorArgs { network_blocks_per_second: 1, maximum_mass_per_block: 500_000 }; // We are testing that the build function actually returns and is not looping indefinitely let estimator = frontier.build_feerate_estimator(args); - dbg!(len, estimator.clone()); let estimations = estimator.calc_estimations(1.0); - dbg!(estimations.clone()); let buckets = estimations.ordered_buckets(); // Test for the absence of NaN, infinite or zero values in buckets @@ -405,6 +405,8 @@ mod tests { "bucket estimated seconds must be a finite number greater than zero" ); } + dbg!(len, estimator); + dbg!(estimations); } } @@ -444,8 +446,8 @@ mod tests { "bucket estimated seconds must be a finite number greater than zero" ); } - // dbg!(estimations); - // dbg!(estimator); + dbg!(len, estimator); + dbg!(estimations); } } } diff --git a/mining/src/mempool/model/frontier/search_tree.rs b/mining/src/mempool/model/frontier/search_tree.rs index beaa6c159..fc18b2118 100644 --- a/mining/src/mempool/model/frontier/search_tree.rs +++ b/mining/src/mempool/model/frontier/search_tree.rs @@ -1,4 +1,5 @@ use super::feerate_key::FeerateTransactionKey; +use std::iter::FusedIterator; use sweep_bptree::tree::visit::{DescendVisit, DescendVisitResult}; use sweep_bptree::tree::{Argument, SearchArgument}; use sweep_bptree::{BPlusTree, NodeStoreVec}; @@ -209,13 +210,13 @@ impl SearchTree { /// Iterate the tree in descending key order (going down from the /// highest key). Linear in the number of keys *actually* iterated. - pub fn descending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator { + pub fn descending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator + FusedIterator { self.tree.iter().rev().map(|(key, ())| key) } /// Iterate the tree in ascending key order (going up from the /// lowest key). Linear in the number of keys *actually* iterated. - pub fn ascending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator { + pub fn ascending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator + FusedIterator { self.tree.iter().map(|(key, ())| key) } From fa5ea38c183c73b164f4ff4bdc5a5db7cd671671 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 15 Aug 2024 21:51:24 +0000 Subject: [PATCH 64/74] monitor feerate estimations (debug print every 10 secs) --- kaspad/src/daemon.rs | 5 +++-- mining/src/feerate/mod.rs | 16 ++++++++++++++++ mining/src/monitor.rs | 8 +++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/kaspad/src/daemon.rs b/kaspad/src/daemon.rs index 0950ad8fa..ce4c19033 100644 --- a/kaspad/src/daemon.rs +++ b/kaspad/src/daemon.rs @@ -419,15 +419,16 @@ do you confirm? (answer y/n or pass --yes to the Kaspad command line to confirm let (address_manager, port_mapping_extender_svc) = AddressManager::new(config.clone(), meta_db, tick_service.clone()); - let mining_monitor = Arc::new(MiningMonitor::new(mining_counters.clone(), tx_script_cache_counters.clone(), tick_service.clone())); let mining_manager = MiningManagerProxy::new(Arc::new(MiningManager::new_with_extended_config( config.target_time_per_block, false, config.max_block_mass, config.ram_scale, config.block_template_cache_lifetime, - mining_counters, + mining_counters.clone(), ))); + let mining_monitor = + Arc::new(MiningMonitor::new(mining_manager.clone(), mining_counters, tx_script_cache_counters.clone(), tick_service.clone())); let flow_context = Arc::new(FlowContext::new( consensus_manager.clone(), diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index bb6f12b08..f8f61d9ce 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -2,6 +2,8 @@ //! behind this fee estimator. use crate::block_template::selector::ALPHA; +use itertools::Itertools; +use std::fmt::Display; #[derive(Clone, Copy, Debug)] pub struct FeerateBucket { @@ -9,6 +11,12 @@ pub struct FeerateBucket { pub estimated_seconds: f64, } +impl Display for FeerateBucket { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "({:.4}, {:.4}s)", self.feerate, self.estimated_seconds) + } +} + #[derive(Clone, Debug)] pub struct FeerateEstimations { /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. @@ -35,6 +43,14 @@ impl FeerateEstimations { } } +impl Display for FeerateEstimations { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "(fee/mass, secs) priority: {}, ", self.priority_bucket)?; + write!(f, "normal: {}, ", self.normal_buckets.iter().format(", "))?; + write!(f, "low: {}", self.low_buckets.iter().format(", ")) + } +} + pub struct FeerateEstimatorArgs { pub network_blocks_per_second: u64, pub maximum_mass_per_block: u64, diff --git a/mining/src/monitor.rs b/mining/src/monitor.rs index 517bd8276..876ce9b7a 100644 --- a/mining/src/monitor.rs +++ b/mining/src/monitor.rs @@ -1,4 +1,5 @@ use super::MiningCounters; +use crate::manager::MiningManagerProxy; use kaspa_core::{ debug, info, task::{ @@ -13,6 +14,8 @@ use std::{sync::Arc, time::Duration}; const MONITOR: &str = "mempool-monitor"; pub struct MiningMonitor { + mining_manager: MiningManagerProxy, + // Counters counters: Arc, @@ -24,11 +27,12 @@ pub struct MiningMonitor { impl MiningMonitor { pub fn new( + mining_manager: MiningManagerProxy, counters: Arc, tx_script_cache_counters: Arc, tick_service: Arc, ) -> MiningMonitor { - MiningMonitor { counters, tx_script_cache_counters, tick_service } + MiningMonitor { mining_manager, counters, tx_script_cache_counters, tick_service } } pub async fn worker(self: &Arc) { @@ -62,6 +66,8 @@ impl MiningMonitor { delta.low_priority_tx_counts, delta.tx_accepted_counts, ); + let feerate_estimations = self.mining_manager.clone().get_realtime_feerate_estimations().await; + debug!("Realtime feerate estimations: {}", feerate_estimations); } if tx_script_cache_snapshot != last_tx_script_cache_snapshot { debug!( From ebe6f94826dd496abbf0b40d96e426c3fa255d47 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 15 Aug 2024 22:01:46 +0000 Subject: [PATCH 65/74] follow rpc naming conventions --- rpc/core/src/api/rpc.rs | 2 +- rpc/core/src/model/feerate_estimate.rs | 16 ++++++++-------- rpc/core/src/model/message.rs | 6 +++--- rpc/grpc/core/src/convert/feerate_estimate.rs | 12 ++++++------ rpc/service/src/converter/feerate_estimate.rs | 4 ++-- rpc/service/src/service.rs | 2 +- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/rpc/core/src/api/rpc.rs b/rpc/core/src/api/rpc.rs index 6d9ab4b86..8fd26b058 100644 --- a/rpc/core/src/api/rpc.rs +++ b/rpc/core/src/api/rpc.rs @@ -318,7 +318,7 @@ pub trait RpcApi: Sync + Send + AnySync { // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Fee estimation API - async fn get_fee_estimate(&self) -> RpcResult { + async fn get_fee_estimate(&self) -> RpcResult { Ok(self.get_fee_estimate_call(GetFeeEstimateRequest {}).await?.estimate) } async fn get_fee_estimate_call(&self, request: GetFeeEstimateRequest) -> RpcResult; diff --git a/rpc/core/src/model/feerate_estimate.rs b/rpc/core/src/model/feerate_estimate.rs index 4c199c40f..442cf1e4d 100644 --- a/rpc/core/src/model/feerate_estimate.rs +++ b/rpc/core/src/model/feerate_estimate.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Copy, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] -pub struct FeerateBucket { +pub struct RpcFeerateBucket { /// The fee/mass ratio estimated to be required for inclusion time <= estimated_seconds pub feerate: f64, @@ -13,24 +13,24 @@ pub struct FeerateBucket { #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] -pub struct FeeEstimate { +pub struct RpcFeeEstimate { /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. - pub priority_bucket: FeerateBucket, + pub priority_bucket: RpcFeerateBucket, /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating /// between them, one can compose a complete feerate function on the client side. The API makes an effort /// to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. - pub normal_buckets: Vec, + pub normal_buckets: Vec, /// A vector of *low* priority feerate values. The first value of this vector is guaranteed to /// exist and provide an estimation for sub-*hour* DAG inclusion. - pub low_buckets: Vec, + pub low_buckets: Vec, } -impl FeeEstimate { - pub fn ordered_buckets(&self) -> Vec { +impl RpcFeeEstimate { + pub fn ordered_buckets(&self) -> Vec { std::iter::once(self.priority_bucket) .chain(self.normal_buckets.iter().copied()) .chain(self.low_buckets.iter().copied()) @@ -40,7 +40,7 @@ impl FeeEstimate { #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] -pub struct FeeEstimateVerboseExperimentalData { +pub struct RpcFeeEstimateVerboseExperimentalData { pub mempool_ready_transactions_count: u64, pub mempool_ready_transactions_total_mass: u64, pub network_mass_per_second: u64, diff --git a/rpc/core/src/model/message.rs b/rpc/core/src/model/message.rs index 88edd08a7..5c003546c 100644 --- a/rpc/core/src/model/message.rs +++ b/rpc/core/src/model/message.rs @@ -860,7 +860,7 @@ pub struct GetFeeEstimateRequest {} #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] pub struct GetFeeEstimateResponse { - pub estimate: FeeEstimate, + pub estimate: RpcFeeEstimate, } #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] @@ -873,10 +873,10 @@ pub struct GetFeeEstimateExperimentalRequest { #[serde(rename_all = "camelCase")] pub struct GetFeeEstimateExperimentalResponse { /// The usual feerate estimate response - pub estimate: FeeEstimate, + pub estimate: RpcFeeEstimate, /// Experimental verbose data - pub verbose: Option, + pub verbose: Option, } // ---------------------------------------------------------------------------- diff --git a/rpc/grpc/core/src/convert/feerate_estimate.rs b/rpc/grpc/core/src/convert/feerate_estimate.rs index a71ad14d1..d1bff8f45 100644 --- a/rpc/grpc/core/src/convert/feerate_estimate.rs +++ b/rpc/grpc/core/src/convert/feerate_estimate.rs @@ -6,14 +6,14 @@ use kaspa_rpc_core::RpcError; // rpc_core to protowire // ---------------------------------------------------------------------------- -from!(item: &kaspa_rpc_core::FeerateBucket, protowire::RpcFeerateBucket, { +from!(item: &kaspa_rpc_core::RpcFeerateBucket, protowire::RpcFeerateBucket, { Self { feerate: item.feerate, estimated_seconds: item.estimated_seconds, } }); -from!(item: &kaspa_rpc_core::FeeEstimate, protowire::RpcFeeEstimate, { +from!(item: &kaspa_rpc_core::RpcFeeEstimate, protowire::RpcFeeEstimate, { Self { priority_bucket: Some((&item.priority_bucket).into()), normal_buckets: item.normal_buckets.iter().map(|b| b.into()).collect(), @@ -21,7 +21,7 @@ from!(item: &kaspa_rpc_core::FeeEstimate, protowire::RpcFeeEstimate, { } }); -from!(item: &kaspa_rpc_core::FeeEstimateVerboseExperimentalData, protowire::RpcFeeEstimateVerboseExperimentalData, { +from!(item: &kaspa_rpc_core::RpcFeeEstimateVerboseExperimentalData, protowire::RpcFeeEstimateVerboseExperimentalData, { Self { network_mass_per_second: item.network_mass_per_second, mempool_ready_transactions_count: item.mempool_ready_transactions_count, @@ -36,14 +36,14 @@ from!(item: &kaspa_rpc_core::FeeEstimateVerboseExperimentalData, protowire::RpcF // protowire to rpc_core // ---------------------------------------------------------------------------- -try_from!(item: &protowire::RpcFeerateBucket, kaspa_rpc_core::FeerateBucket, { +try_from!(item: &protowire::RpcFeerateBucket, kaspa_rpc_core::RpcFeerateBucket, { Self { feerate: item.feerate, estimated_seconds: item.estimated_seconds, } }); -try_from!(item: &protowire::RpcFeeEstimate, kaspa_rpc_core::FeeEstimate, { +try_from!(item: &protowire::RpcFeeEstimate, kaspa_rpc_core::RpcFeeEstimate, { Self { priority_bucket: item.priority_bucket .as_ref() @@ -54,7 +54,7 @@ try_from!(item: &protowire::RpcFeeEstimate, kaspa_rpc_core::FeeEstimate, { } }); -try_from!(item: &protowire::RpcFeeEstimateVerboseExperimentalData, kaspa_rpc_core::FeeEstimateVerboseExperimentalData, { +try_from!(item: &protowire::RpcFeeEstimateVerboseExperimentalData, kaspa_rpc_core::RpcFeeEstimateVerboseExperimentalData, { Self { network_mass_per_second: item.network_mass_per_second, mempool_ready_transactions_count: item.mempool_ready_transactions_count, diff --git a/rpc/service/src/converter/feerate_estimate.rs b/rpc/service/src/converter/feerate_estimate.rs index 88c5f15c2..8df695c0c 100644 --- a/rpc/service/src/converter/feerate_estimate.rs +++ b/rpc/service/src/converter/feerate_estimate.rs @@ -1,7 +1,7 @@ use kaspa_mining::feerate::{FeeEstimateVerbose, FeerateBucket, FeerateEstimations}; use kaspa_rpc_core::{ - message::GetFeeEstimateExperimentalResponse as RpcFeeEstimateVerboseResponse, FeeEstimate as RpcFeeEstimate, - FeeEstimateVerboseExperimentalData as RpcFeeEstimateVerbose, FeerateBucket as RpcFeerateBucket, + message::GetFeeEstimateExperimentalResponse as RpcFeeEstimateVerboseResponse, RpcFeeEstimate, + RpcFeeEstimateVerboseExperimentalData as RpcFeeEstimateVerbose, RpcFeerateBucket, }; pub trait FeerateBucketConverter { diff --git a/rpc/service/src/service.rs b/rpc/service/src/service.rs index d4dcbdb31..e15fa3afa 100644 --- a/rpc/service/src/service.rs +++ b/rpc/service/src/service.rs @@ -112,7 +112,7 @@ pub struct RpcCoreService { perf_monitor: Arc>>, p2p_tower_counters: Arc, grpc_tower_counters: Arc, - fee_estimate_cache: ExpiringCache, + fee_estimate_cache: ExpiringCache, fee_estimate_verbose_cache: ExpiringCache, } From abf30d39120e572a8b43e1be89d64916e71640be Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 15 Aug 2024 22:06:19 +0000 Subject: [PATCH 66/74] proto leave blank index range --- rpc/grpc/core/proto/rpc.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rpc/grpc/core/proto/rpc.proto b/rpc/grpc/core/proto/rpc.proto index dd19e2546..0683cafcd 100644 --- a/rpc/grpc/core/proto/rpc.proto +++ b/rpc/grpc/core/proto/rpc.proto @@ -883,9 +883,9 @@ message RpcFeeEstimateVerboseExperimentalData { uint64 mempool_ready_transactions_total_mass = 2; uint64 network_mass_per_second = 3; - double next_block_template_feerate_min = 4; - double next_block_template_feerate_median = 5; - double next_block_template_feerate_max = 6; + double next_block_template_feerate_min = 11; + double next_block_template_feerate_median = 12; + double next_block_template_feerate_max = 13; } message GetFeeEstimateRequestMessage { From 6731521cede590eec9bb98e8f53758edf6f4c93d Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 15 Aug 2024 22:08:00 +0000 Subject: [PATCH 67/74] insert in correct abc location (keeping rest of the array as is for easier omega merge) --- rpc/wrpc/server/src/router.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rpc/wrpc/server/src/router.rs b/rpc/wrpc/server/src/router.rs index 024f6a9c6..ff72cbdb3 100644 --- a/rpc/wrpc/server/src/router.rs +++ b/rpc/wrpc/server/src/router.rs @@ -44,9 +44,11 @@ impl Router { GetBlockTemplate, GetCoinSupply, GetConnectedPeerInfo, + GetCurrentNetwork, GetDaaScoreTimestampEstimate, GetServerInfo, - GetCurrentNetwork, + GetFeeEstimate, + GetFeeEstimateExperimental, GetHeaders, GetInfo, GetInfo, @@ -68,8 +70,6 @@ impl Router { SubmitTransaction, SubmitTransactionReplacement, Unban, - GetFeeEstimate, - GetFeeEstimateExperimental, ] ); From c1981898086c3d6a07979089735260b8499acc93 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 15 Aug 2024 22:16:39 +0000 Subject: [PATCH 68/74] fix comment to reflect the most updated final algo --- mining/src/mempool/model/frontier.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index 442d19f14..b8fbde74a 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -89,10 +89,11 @@ impl Frontier { /// 2. Indeed, if the weight distribution is not too spread (i.e., `max(weights) = O(min(weights))`), `k << n` means /// that the probability of collisions is low enough and the sampling process will converge in `O(k log(n))` w.h.p. /// 3. It remains to deal with the case where the weight distribution is highly biased. The process implemented below - /// keeps track of the top-weight element. If the distribution is highly biased, this element will be sampled twice - /// with sufficient probability (in constant time), in which case we narrow the sampling space to exclude it. We do - /// this by computing the prefix weight up to this top item (exclusive) and then continue the sampling process over - /// the narrowed space. This process is repeated until acquiring the desired mass. + /// keeps track of the top-weight element. If the distribution is highly biased, this element will be sampled with + /// sufficient probability (in constant time). Following each sampling collision we search for a consecutive range of + /// top elements which were already sampled and narrow the sampling space to exclude them all . We do this by computing + /// the prefix weight up to the top most item which wasn't sampled yet (inclusive) and then continue the sampling process + /// over the narrowed space. This process is repeated until acquiring the desired mass. /// 4. Numerical stability. Naively, one would simply subtract `total_weight -= top.weight` in order to narrow the sampling /// space. However, if `top.weight` is much larger than the remaining weight, the above f64 subtraction will yield a number /// close or equal to zero. We fix this by implementing a `log(n)` prefix weight operation. From 79790d355a09a8ca488c7d2fd84806f2265f6fa4 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 15 Aug 2024 22:33:09 +0000 Subject: [PATCH 69/74] document feerate --- mining/src/feerate/mod.rs | 13 +++++++++++-- rpc/core/src/model/feerate_estimate.rs | 4 ++++ rpc/grpc/core/proto/rpc.proto | 16 ++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index f8f61d9ce..4dedf711b 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -5,6 +5,11 @@ use crate::block_template::selector::ALPHA; use itertools::Itertools; use std::fmt::Display; +/// A type representing fee/mass of a transaction in `sompi/gram` units. +/// Given a feerate value recommendation, calculate the required fee by +/// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` +pub type Feerate = f64; + #[derive(Clone, Copy, Debug)] pub struct FeerateBucket { pub feerate: f64, @@ -20,9 +25,13 @@ impl Display for FeerateBucket { #[derive(Clone, Debug)] pub struct FeerateEstimations { /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + /// + /// Note: for all buckets, feerate values represent fee/mass of a transaction in `sompi/gram` units. + /// Given a feerate value recommendation, calculate the required fee by + /// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` pub priority_bucket: FeerateBucket, - /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to + /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating /// between them, one can compose a complete feerate function on the client side. The API makes an effort @@ -30,7 +39,7 @@ pub struct FeerateEstimations { pub normal_buckets: Vec, /// A vector of *low* priority feerate values. The first value of this vector is guaranteed to - /// provide an estimation for sub-*hour* DAG inclusion. + /// exist and provide an estimation for sub-*hour* DAG inclusion. pub low_buckets: Vec, } diff --git a/rpc/core/src/model/feerate_estimate.rs b/rpc/core/src/model/feerate_estimate.rs index 442cf1e4d..f9de9de90 100644 --- a/rpc/core/src/model/feerate_estimate.rs +++ b/rpc/core/src/model/feerate_estimate.rs @@ -15,6 +15,10 @@ pub struct RpcFeerateBucket { #[serde(rename_all = "camelCase")] pub struct RpcFeeEstimate { /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + /// + /// Note: for all buckets, feerate values represent fee/mass of a transaction in `sompi/gram` units. + /// Given a feerate value recommendation, calculate the required fee by + /// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` pub priority_bucket: RpcFeerateBucket, /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and diff --git a/rpc/grpc/core/proto/rpc.proto b/rpc/grpc/core/proto/rpc.proto index 0683cafcd..a0bed8977 100644 --- a/rpc/grpc/core/proto/rpc.proto +++ b/rpc/grpc/core/proto/rpc.proto @@ -868,13 +868,29 @@ message GetDaaScoreTimestampEstimateResponseMessage{ } message RpcFeerateBucket { + // Fee/mass of a transaction in `sompi/gram` units double feerate = 1; double estimated_seconds = 2; } +// Data required for making fee estimates. +// +// Feerate values represent fee/mass of a transaction in `sompi/gram` units. +// Given a feerate value recommendation, calculate the required fee by +// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` message RpcFeeEstimate { + // Top-priority feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. RpcFeerateBucket priority_bucket = 1; + + // A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and + // provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + // times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + // between them, one can compose a complete feerate function on the client side. The API makes an effort + // to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. repeated RpcFeerateBucket normal_buckets = 2; + + // A vector of *low* priority feerate values. The first value of this vector is guaranteed to + // exist and provide an estimation for sub-*hour* DAG inclusion. repeated RpcFeerateBucket low_buckets = 3; } From 095baab048b9cd107a0c60b4fc5f7256543b9779 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 15 Aug 2024 22:48:59 +0000 Subject: [PATCH 70/74] update notebook --- mining/src/feerate/fee_estimation.ipynb | 81 +++++++++++++++++++------ 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/mining/src/feerate/fee_estimation.ipynb b/mining/src/feerate/fee_estimation.ipynb index da8d71fed..020baf036 100644 --- a/mining/src/feerate/fee_estimation.ipynb +++ b/mining/src/feerate/fee_estimation.ipynb @@ -1,8 +1,20 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Feerates\n", + "\n", + "The feerate value represents the fee/mass ratio of a transaction in `sompi/gram` units.\n", + "Given a feerate value recommendation, one should calculate the required fee by taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)`. \n", + "\n", + "This notebook makes an effort to implement and illustrate the feerate estimator method we used. The corresponding Rust implementation is more comprehencive and addresses some additional edge cases, but the code in this notebook highly reflects it." + ] + }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -12,26 +24,33 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ - "ALPHA = 3.0" + "feerates = [1.0, 1.1, 1.2]*10 + [1.5]*3000 + [2]*3000 + [2.1]*3000 + [3, 4, 5]*10\n", + "# feerates = [1.0, 1.1, 1.2] + [1.1]*100 + [1.2]*100 + [1.3]*100 # + [3, 4, 5, 100]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We compute the probability weight of each transaction by raising `feerate` to the power of `alpha` (currently set to `3`). Essentially, alpha represents the amount of bias we want towards higher feerate transactions. " ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ - "feerates = [1.0, 1.1, 1.2]*10 + [1.5]*3000 + [2]*3000 + [2.1]*3000 + [3, 4, 5]*10\n", - "# feerates = [1.0, 1.1, 1.2] + [1.1]*100 + [1.2]*100 + [1.3]*100 # + [3, 4, 5, 100]" + "ALPHA = 3.0" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -49,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -61,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -78,7 +97,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -174,7 +193,7 @@ " # Choose `low` feerate such that it provides sub-hour waiting time AND it covers (at least) the 0.25 quantile\n", " low = max(self.time_to_feerate(3600.0), self.quantile(1.0, high, 0.25))\n", " # Choose `mid` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.5 quantile\n", - " mid = max(self.time_to_feerate(60.0), self.quantile(1.0, high, 0.5))\n", + " mid = max(self.time_to_feerate(60.0), self.quantile(low, high, 0.5))\n", " # low = self.quantile(1.0, high, 0.25)\n", " # mid = self.quantile(1.0, high, 0.5)\n", " return FeerateEstimations(\n", @@ -183,18 +202,40 @@ " FeerateBucket(high, self.feerate_to_time(high)))" ] }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "estimator = FeerateEstimator(total_weight=0, inclusion_interval=1/100)\n", + "# estimator.quantile(2, 3, 0.5)\n", + "estimator.time_to_feerate(1)" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Feerate estimation\n", "\n", - "The figure below illustrates the estimator selectoin. We first estimate the `feerate_to_time` function and then select 3 meaningfull points by analyzing the curve and its integral (see `calc_estimations`). " + "The figure below illustrates the estimator selection. We first estimate the `feerate_to_time` function and then select 3 meaningfull points by analyzing the curve and its integral (see `calc_estimations`). " ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -214,7 +255,7 @@ "Times:\t\t168.62498827393395, 59.999999999999986, 1.0000000000000004" ] }, - "execution_count": 8, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -245,7 +286,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -265,7 +306,7 @@ " array([168.62498827, 60. , 1. ]))" ] }, - "execution_count": 9, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -297,7 +338,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -317,7 +358,7 @@ "Times:\t\t2769.889957638353, 60.00000000000002, 1.0000000000000007" ] }, - "execution_count": 10, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -348,7 +389,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -368,7 +409,7 @@ "Times:\t\t874.010579873836, 60.00000000000001, 1.0000000000000004" ] }, - "execution_count": 11, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } From 49759f132959961190f349934b546b3f2f612b43 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 15 Aug 2024 23:21:39 +0000 Subject: [PATCH 71/74] add an additional point to normal feerate buckets (between normal and low) --- mining/src/feerate/fee_estimation.ipynb | 82 ++++++++++++++----------- mining/src/feerate/mod.rs | 12 +++- 2 files changed, 56 insertions(+), 38 deletions(-) diff --git a/mining/src/feerate/fee_estimation.ipynb b/mining/src/feerate/fee_estimation.ipynb index 020baf036..610d8fcd0 100644 --- a/mining/src/feerate/fee_estimation.ipynb +++ b/mining/src/feerate/fee_estimation.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 97, "metadata": {}, "outputs": [], "source": [ @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 98, "metadata": {}, "outputs": [], "source": [ @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 99, "metadata": {}, "outputs": [], "source": [ @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 100, "metadata": {}, "outputs": [ { @@ -68,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 101, "metadata": {}, "outputs": [], "source": [ @@ -80,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 102, "metadata": {}, "outputs": [ { @@ -97,7 +97,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 103, "metadata": {}, "outputs": [], "source": [ @@ -108,21 +108,26 @@ " \n", "\n", "class FeerateEstimations:\n", - " def __init__(self, low_bucket, normal_bucket, priority_bucket):\n", + " def __init__(self, low_bucket, mid_bucket, normal_bucket, priority_bucket):\n", " self.low_bucket = low_bucket \n", + " self.mid_bucket = mid_bucket \n", " self.normal_bucket = normal_bucket\n", " self.priority_bucket = priority_bucket\n", " \n", " def __repr__(self):\n", - " return 'Feerates:\\t{}, {}, {} \\nTimes:\\t\\t{}, {}, {}'.format(self.low_bucket.feerate, \n", + " return 'Feerates:\\t{}, {}, {}, {} \\nTimes:\\t\\t{}, {}, {}, {}'.format(\n", + " self.low_bucket.feerate, \n", + " self.mid_bucket.feerate, \n", " self.normal_bucket.feerate,\n", " self.priority_bucket.feerate, \n", " self.low_bucket.estimated_seconds, \n", + " self.mid_bucket.estimated_seconds, \n", " self.normal_bucket.estimated_seconds, \n", " self.priority_bucket.estimated_seconds)\n", " def feerates(self):\n", " return np.array([\n", " self.low_bucket.feerate, \n", + " self.mid_bucket.feerate, \n", " self.normal_bucket.feerate,\n", " self.priority_bucket.feerate\n", " ])\n", @@ -130,6 +135,7 @@ " def times(self):\n", " return np.array([\n", " self.low_bucket.estimated_seconds, \n", + " self.mid_bucket.estimated_seconds, \n", " self.normal_bucket.estimated_seconds,\n", " self.priority_bucket.estimated_seconds\n", " ])\n", @@ -190,21 +196,27 @@ " def calc_estimations(self):\n", " # Choose `high` such that it provides sub-second waiting time\n", " high = self.time_to_feerate(1.0)\n", + " \n", " # Choose `low` feerate such that it provides sub-hour waiting time AND it covers (at least) the 0.25 quantile\n", " low = max(self.time_to_feerate(3600.0), self.quantile(1.0, high, 0.25))\n", - " # Choose `mid` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.5 quantile\n", - " mid = max(self.time_to_feerate(60.0), self.quantile(low, high, 0.5))\n", - " # low = self.quantile(1.0, high, 0.25)\n", - " # mid = self.quantile(1.0, high, 0.5)\n", + " \n", + " # Choose `normal` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.75\n", + " # quantile between low and high\n", + " normal = max(self.time_to_feerate(60.0), self.quantile(low, high, 0.66))\n", + " \n", + " # Choose an additional point between normal and low\n", + " mid = max(self.time_to_feerate(1800.0), self.quantile(1.0, high, 0.5))\n", + " \n", " return FeerateEstimations(\n", " FeerateBucket(low, self.feerate_to_time(low)),\n", " FeerateBucket(mid, self.feerate_to_time(mid)),\n", + " FeerateBucket(normal, self.feerate_to_time(normal)),\n", " FeerateBucket(high, self.feerate_to_time(high)))" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 104, "metadata": {}, "outputs": [ { @@ -213,7 +225,7 @@ "0.0" ] }, - "execution_count": 20, + "execution_count": 104, "metadata": {}, "output_type": "execute_result" } @@ -235,12 +247,12 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 105, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHTlJREFUeJzt3XuUnHWd5/H3t+7V9066k3TSucEE5GICoY2szDqMDCqKgq7L4Jxx0OOezDniqmfnnDnq2XVcj5z1rKvs4io7ICiut2UUVxzwwkZGiCChgxDIBUjIpTu37tz7kr5U13f/qKdDJ3SnO91Vebqe+rzOqfM8z6+ep+pbXD7P07/6Pb8yd0dERKIrFnYBIiJSWgp6EZGIU9CLiEScgl5EJOIU9CIiEaegFxGJOAW9iEjEKehFRCJOQS8iEnGJsAsAaGpq8mXLloVdhohIWdm4ceMhd2+ebL9ZEfTLli2jvb097DJERMqKme2eyn7quhERiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4so66F8+0MNXf72NY/1DYZciIjJrlXXQ7z7cxzcf30HHkZNhlyIiMmtNGvRmttjMHjezrWa22cw+HbR/0cz2mtnzweM9Y475nJltN7OXzexdpSp+fl0GgIMnBkr1FiIiZW8qUyDkgL9z9+fMrBbYaGaPBc/d6e7/bezOZnYpcCtwGbAQ+H9mdpG7jxSzcBgT9D0KehGRiUx6Re/u+939uWC9B9gKLDrLITcBP3b3QXffCWwH1hSj2DM11aQw4OCJwVK8vIhIJJxTH72ZLQOuBJ4Jmj5pZpvM7H4zawzaFgEdYw7r5OwnhmlLxGM0VCXpUteNiMiEphz0ZlYD/BT4jLufAO4GLgSuAPYDXxvddZzDfZzXW2tm7WbW3t3dfc6Fj6rPJtVHLyJyFlMKejNLUgj5H7j7QwDuftDdR9w9D9zL690zncDiMYe3AvvOfE13v8fd29y9rbl50umUJ1SfTXJAQS8iMqGpjLox4D5gq7t/fUx7y5jdPgC8FKw/DNxqZmkzWw6sADYUr+TTNVQl1UcvInIWUxl1cw3wEeBFM3s+aPs88GEzu4JCt8wu4G8B3H2zmT0IbKEwYuf2Uoy4GdVQleJI3xBDuTypRFnfFiAiUhKTBr27r2f8fvdHz3LMHcAdM6hryhqqkgB09w6yqCF7Pt5SRKSslP0lcGMQ9PpCVkRkfGUf9A3ZFICGWIqITKD8g/7UFb2+kBURGU/ZB31NJkHM1HUjIjKRsg/6mBk16YSu6EVEJlD2QQ9QnU7QpYnNRETGFYmgr0rF2X9MQS8iMp5IBH11OqFpEEREJhCJoK9JJ+gdzNE/lAu7FBGRWScSQV+bKdzgu0/dNyIibxCNoE8XxtLvO6bfjhUROVM0gv7UFb2CXkTkTJEI+up0AkNBLyIynkgEfTxm1GYS7DuuPnoRkTNFIuihMBWCruhFRN4oOkGfSrD3qIJeRORM0Qn6TIL9xwdwf8PvkIuIVLTIBH1tJsnQSJ7DfUNhlyIiMqtEKOg1xFJEZDzRCfq07o4VERlPdII+o7tjRUTGE5mgzyRjJOOmoBcROUNkgt7MqMsk2a+bpkREThOZoAeoTsfpPNofdhkiIrNKpIK+NpOkUzdNiYicJlJBX5dNcrhvSD9AIiIyRqSCvj4YeaOrehGR10Ur6LOFoN9zWP30IiKjIhX0ddnCTVMd+kJWROSUSAV9NhknlYix54iCXkRk1KRBb2aLzexxM9tqZpvN7NNB+xwze8zMXg2WjUG7mdldZrbdzDaZ2epSf4gxtVKfTdKhoBcROWUqV/Q54O/c/RLgauB2M7sU+Cywzt1XAOuCbYAbgBXBYy1wd9GrPovadEJX9CIiY0wa9O6+392fC9Z7gK3AIuAm4IFgtweAm4P1m4DvecEfgAYzayl65ROoyybZc6Rf89KLiATOqY/ezJYBVwLPAPPdfT8UTgbAvGC3RUDHmMM6g7YzX2utmbWbWXt3d/e5Vz6B+mySgWHNSy8iMmrKQW9mNcBPgc+4+4mz7TpO2xsur939Hndvc/e25ubmqZYxqdGRN+q+EREpmFLQm1mSQsj/wN0fCpoPjnbJBMuuoL0TWDzm8FZgX3HKndzoTVP6QlZEpGAqo24MuA/Y6u5fH/PUw8BtwfptwM/HtP9NMPrmauD4aBfP+VCXVdCLiIyVmMI+1wAfAV40s+eDts8DXwEeNLOPA3uAfxs89yjwHmA70A98rKgVTyIZj1GjkTciIqdMGvTuvp7x+90Brhtnfwdun2FdM1KXSbBb0yCIiAARuzN2VH1VktcO9YVdhojIrBDJoG+sStHdM0jfoKYrFhGJZNA3VBW+kN2pq3oRkWgGfWNVClDQi4hARIO+IasrehGRUZEM+kQ8Rn02qaAXESGiQQ+FOW9e6+4NuwwRkdBFNugbsoUhlprFUkQqXXSDvipJz0COI5rFUkQqXGSDXiNvREQKIhv0o2PpdYesiFS6yAZ9XSZJ3ExX9CJS8SIb9LGY0VCVZHuXRt6ISGWLbNADNFanePlAT9hliIiEKtJBP7c6RceRfgaGR8IuRUQkNJEO+neNPMGTqU+RvmMu3Hk5bHow7JJERM67qfzCVFm6uOuX/EX310jFBgsNxzvgF58qrK+8JbzCRETOs8he0f/pnm+R8sHTG4dPwrovhVOQiEhIIhv0tYMHx3/ieOf5LUREJGSRDfqe9Pzxn6hvPb+FiIiELLJBv37JJxiOZU5vTGbhui+EU5CISEgi+2Xsy/NuAOCtO/8njcPd5GoXknrnF/VFrIhUnMgGPRTC/qmqd/D9Z/Zw582r+MBKdduISOWJbNfNqIaqFHEzXj6gqRBEpDJFPujjMWNuTYptB06EXYqISCgiH/QAc2tSvLT3eNhliIiEoiKCvrkmzaHeIbp6BsIuRUTkvKuMoK9NA7B1v2ayFJHKUxlBX1MI+s371H0jIpVn0qA3s/vNrMvMXhrT9kUz22tmzweP94x57nNmtt3MXjazd5Wq8HORTsZpyCbZsk9fyIpI5ZnKFf13gXeP036nu18RPB4FMLNLgVuBy4JjvmVm8WIVOxNza1JsVtCLSAWaNOjd/QngyBRf7ybgx+4+6O47ge3AmhnUVzTNNWl2HeqjbzAXdikiIufVTProP2lmm4KuncagbRHQMWafzqDtDcxsrZm1m1l7d3f3DMqYmubaNA5s008LikiFmW7Q3w1cCFwB7Ae+FrTbOPv6eC/g7ve4e5u7tzU3N0+zjKkbHXmzZb+6b0Skskwr6N39oLuPuHseuJfXu2c6gcVjdm0F9s2sxOKoSSeoSsV5sfNY2KWIiJxX0wp6M2sZs/kBYHREzsPArWaWNrPlwApgw8xKLA4zY15tmuc7FPQiUlkmnb3SzH4EXAs0mVkn8A/AtWZ2BYVumV3A3wK4+2YzexDYAuSA2919pDSln7v5dRk27DxC72COmnSkJ+4UETll0rRz9w+P03zfWfa/A7hjJkWVyoK6DA68tPc4V18wN+xyRETOi4q4M3bU/LrCL069oO4bEakgFRX02VScxqqk+ulFpKJUVNBDYZjlHxX0IlJBKi7oF9RlOHB8gK4TmrJYRCpDxQX9aD+9um9EpFJUXNDPq00TMwW9iFSOigv6RDzGvNoMz+6a6jxtIiLlreKCHmBhQ4bnO44xMDxr7uUSESmZCg36LMMjzqZO/eKUiERfxQY9oO4bEakIFRn02WScppoUG3Yq6EUk+ioy6AEW1GfYuPsoI/lxp8sXEYmMig36RQ1ZegdzbNUPkYhIxFVs0KufXkQqRcUGfV0mSX02ydM7DoddiohISVVs0AO0NmZ5asdhciP5sEsRESmZig76JXOq6B3M8eJejacXkeiq6KBvbSz0069/9VDIlYiIlE5FB31VKsG8ujTrtyvoRSS6KjroARY3VLFx91H6h3JhlyIiUhIK+jlZcnnnGd0lKyIRVfFBv6ghSyJm6qcXkciq+KBPxGMsasjy221dYZciIlISFR/0AMuaqtl5qI+dh/rCLkVEpOgU9MAFTdUArNt6MORKRESKT0EP1GWTNNekWbdV3TciEj0K+sDSuVVs2HWE4yeHwy5FRKSoFPSB5U3VjOSdJ17pDrsUEZGiUtAHFtRnqErF1U8vIpEzadCb2f1m1mVmL41pm2Nmj5nZq8GyMWg3M7vLzLab2SYzW13K4ospZsayudU8tvUgg7mRsMsRESmaqVzRfxd49xltnwXWufsKYF2wDXADsCJ4rAXuLk6Z58eK+TX0DY7w5Cu6eUpEomPSoHf3J4Az5we4CXggWH8AuHlM+/e84A9Ag5m1FKvYUlvcWEU2GeeRF/eHXYqISNFMt49+vrvvBwiW84L2RUDHmP06g7Y3MLO1ZtZuZu3d3bPjC9B4zFjeVM1jWw4yMKzuGxGJhmJ/GWvjtPl4O7r7Pe7e5u5tzc3NRS5j+i6aX0PvYI4nNfeNiETEdIP+4GiXTLAcvdOoE1g8Zr9WYN/0yzv/Wke7bzaVVdkiIhOabtA/DNwWrN8G/HxM+98Eo2+uBo6PdvGUi3jMuKC5mt9sOag56kUkEqYyvPJHwNPAxWbWaWYfB74CXG9mrwLXB9sAjwKvAduBe4FPlKTqErtkQR39QyP86qUDYZciIjJjicl2cPcPT/DUdePs68DtMy0qbAsbMjRUJfmn9k4+uLo17HJERGZEd8aOw8x404Jann7tMJ1H+8MuR0RkRhT0E7hkQR0ADz23N+RKRERmRkE/gbpsksWNWf6pvYNCj5SISHlS0J/FJS11dBw9yVM7DoddiojItCnoz2LFvBqqUnG+9/SusEsREZk2Bf1ZJOIxLmmp47EtB9l37GTY5YiITIuCfhIrF9XjDj98Zk/YpYiITIuCfhJ12STLm6r54YY9mqdeRMqSgn4KVrbWc6RviH9+oaxmcxARART0U7JkThVNNSn+1+92kM9rqKWIlBcF/RSYGVctaeTVrl4ef7lr8gNERGYRBf0UrZhfS302yd3/siPsUkREzomCforiMeOKxQ207z7Ks7vO/GVFEZHZS0F/Di5bWEdVKs431r0adikiIlOmoD8HyXiM1UsaeeLVQ2zYqat6ESkPCvpztLK1npp0gq/+epsmOxORsqCgP0fJeIy2ZY08u+soT+gHxEWkDCjop+HyhfXUZ5P8119t07h6EZn1FPTTEI8Zb10+h837TvDQH/XDJCIyuynop+lNC2ppqc/wlV9upXcwF3Y5IiITUtBPk5nx9hXNHOod4hu/1XBLEZm9FPQzsKA+w6Uttdz35E52HuoLuxwRkXEp6GfobRc2cVP899R86wr8iw1w5+Ww6cGwyxIROSURdgHlbvXxx7gucS/p/GCh4XgH/OJThfWVt4RXmIhIQFf0M/Sne75F2gdPbxw+Ceu+FE5BIiJnUNDPUO3gwfGfON55fgsREZmAgn6GetLzx3+ivvX8FiIiMgEF/QytX/IJhmOZ09pOeop9bX8fUkUiIqdT0M/Qy/Nu4LELP8+J9AIc41hqAV9gLR9tX8rAsH5MXETCN6NRN2a2C+gBRoCcu7eZ2Rzg/wDLgF3ALe5+dGZlzm4vz7uBl+fdcGp7+HAfrzy/j//4f1/iqx9aiZmFWJ2IVLpiXNH/ubtf4e5twfZngXXuvgJYF2xXlKVzq1mzfA4/2djJ957eHXY5IlLhStF1cxPwQLD+AHBzCd5j1rt6+RwuaKrmS7/YwtM7DoddjohUsJkGvQO/MbONZrY2aJvv7vsBguW8Gb5HWTIz3nnZfBqqknziBxs1RYKIhGamQX+Nu68GbgBuN7O3T/VAM1trZu1m1t7d3T3DMmandCLOe1e2MJjL85H7nqGrZyDskkSkAs0o6N19X7DsAn4GrAEOmlkLQLDsmuDYe9y9zd3bmpubZ1LGrNZYleJ9qxbSdWKQ2+7fQM/AcNgliUiFmXbQm1m1mdWOrgPvBF4CHgZuC3a7Dfj5TIssdwvqMrznzQt4+UAP/+6BdvqHNH+9iJw/M7minw+sN7MXgA3AI+7+K+ArwPVm9ipwfbBd8ZbOreadly5gw64jfPT+Z+nTj5WIyHky7XH07v4asGqc9sPAdTMpKqouXlALwK+3HOCj39nAdz+2huq0JhAVkdLSnbHn2cULann3ZQvYuPsof3XvHzjcOzj5QSIiM6CgD8FF82t5z5tb2LzvBB/81lPsPqyhlyJSOgr6kFzYXMMHVy+iu3eQm7/5e57bE+lZIkQkRAr6ELXUZ/nQ6lYc+Mt/fJofbdgTdkkiEkEK+pA1Vqf4y7bFLGzI8rmHXuRzD21iMKdZL0WkeBT0s0AmGef9qxbylmWN/GhDBx/45lNs7+oJuywRiQgF/SwRM+NtFzbxvpUt7Drcx3vvWs//fnoX7h52aSJS5hT0s8wFzTX81ZoltNRn+E8/38xHv/MsHUf6wy5LRMqYgn4Wqk4neP+qhVx7UTNP7zjM9Xf+jn/83Q6GR/JhlyYiZUhBP0uZGasWN/DXVy9hUUOW//LLbdz4jfWa215EzpmCfparzSS5ceVCblzZwv5jJ/nwvX/g4w88qy9rRWTKNNFKmbiwuYalc6p4vuMY6189xLu2Pcktb1nM7X9+Ia2NVWGXJyKzmIK+jCTiMdqWzeHShXVs2HmEB5/t4MH2Dv7N6kV84to/YVlTddglisgspKAvQ1WpBNdePI+rljaycfdRHnpuLz/Z2MmNKxfysWuWceWSxrBLFJFZREFfxmozSa69eB5vWTaH5/Yc5debD/DwC/tYuaiej16zjPeubCGdiIddpoiEzGbDDTltbW3e3t4+rWN/u+0gL3QcL3JF5Wkol2fr/hNs2nucI31DNFYl+cCVrXzoqlYuXVgXdnkiUmRmttHd2ybbT1f0EZJKxFi1uIGVrfXsOdLPS/tO8MDTu7j/9zt504JaPnRVK+9ftZB5dZmwSxWR80hBH0FmxtK51SydW83J4RFeOdDDtgM9fPmRrdzxyFZWL23khssX8K7LFrB4jkbsiESdgj7issk4qxY3sGpxA0f6htje1cuO7l6+/MhWvvzIVi5fWMc7LpnPn13UzKrWehJx3VohEjUK+goypzrFmuVzWLN8Dsf6h9jR3ceO7l6+8dtXuWvdq9RmEvzrFU382UXNvO3CJlobs5hZ2GWLyAwp6CtUQ1WKq5amuGppIwPDI+w50s/uw/08+cohHn3xAAAL6jKsWT6Htyyfw5plc1gxr4ZYTMEvUm4U9EImGeei+bVcNL8Wd+dw3xB7j55k77GTPL6ti4df2AdAfTbJqtZ6VrY28ObWela21rOgLqOrfpFZTkEvpzEzmmrSNNWkWbW4AXfnxECOvcdOsu/YSbYd6GH99kPkg1G5c6pTrGqt5/JF9ayYX8vF82tZ3lRNKqG+fpHZQkEvZ2Vm1GeT1GeTXNpSGIufG8nT3TtI14lBDvYM8OLe4/zule5T4R+PGcubqrk4+CvhT+bVsKypiqVzq6lJ6z85kfNN/9fJOUvEY7TUZ2mpz55qy43kOdo/zOG+QY70DXG4d4indhzi0Rf3M/aWvLnVKZY3FYZ+LptbxdKmapbOqaKlIUNTdVrfAYiUgIJeiiIRj9Fcm6a5Nn1a+/BInmP9wxzrH+LYyWGOnxzmwIkBXjnYw4mB3Gn7JuNGS32WhQ0ZFjZkWVifZWFDlpaGDC31GZpr0jRWpXQyEDlHCnopqeQEJwAonASOB+HfO5CjZzBHz8AwHUf62bLvBL2DuVPdQaPiZsytSZ16zeaawrJpzLKxOkljVYqGqqTm+hFBQS8hSsZjp774HU8+7/QN5egZyNE7mKN/aIT+oRx9gyP0DeY43DvEc0NHxz0hjMom4zRUFYK/sTpJQ1WKxmC7PlvYrs0kCo90ktpMgppgWycJiQoFvcxasZhRm0lSm0medT93Z2A4XzgJDI0wMDz6yJ9a7x/KcbR/iMFcDwPDI5wcGmGy6fxS8RjV6Ti1mSR12dNPBDXpBNlUnOpUgqpUnGwqXlgmC9uvt415PhnXnccSipIFvZm9G/gfQBz4trt/pVTvJZXNzMgGYTp3ise4O4O5wolgKJdnMJdnaCTPUC7/+nYuz+BI4fnegRxH+oYYzvnr+43kGZnoT4kJJONGNlmoNZ2Ik07EyCTjZJNx0skY6UTsVHthO1hPxEgnx6wn3rh/MhEjETOS8Rip8dYTMVLxwno8Zrr/IQybHoR1X4LjnVDfCtd9AVbeUvK3LUnQm1kc+CZwPdAJPGtmD7v7llK8n8i5MjMyyTiZ5My6Z0byTi6fZ3jEGR7JkwuWhYeTC5bD+Te25fKFE8WJgWGO9g+Rd2ck78FrOiMjheXo6xeTUeg6S8QLJ4Nk3EjEYiQTVjgZxAsnhdGTxOgyHosRj0EiFiMeMxIxIxYs37j9+kklPub509djk7xGoa5YrPD9TCxmxMyIGcRs9IRVGNI7tv3UI8a463EzLNiOW+E1Rl8vZpTmJLjpQfjFp2D4ZGH7eEdhG0oe9qW6ol8DbHf31wDM7MfATYCCXiKlEFpxSn17gLuTd06dHHKjJ4QRP3WyyXvhxJN3J593RtzJ5wmWfsby9fZTJ5jgPfJ5ZziXZ3B45NS+eS/s507hQbBv8BrukOeN++aD1yxHY08Ap50wTrW9flKI2+vrMePUCcl4ve37PZ9nvp88/U2GTxau8Ms06BcBHWO2O4G3luKN5lanWa7fShWZtUZPUu7+hhPH6IllJDgxjD35jN1vZMxJ5tQJBz9je8z6mPc8vT1YZ2r7TPR6p72vO/ng9caeCEdrGj2m2Q+N/w/oeGfJ/x2UKujH+7vntPO6ma0F1gIsWbJk2m80OgWviMisdmdrobvmTPWtJX/rUg0B6AQWj9luBfaN3cHd73H3Nndva25uLlEZIiKzxHVfgGT29LZkttBeYqUK+meBFWa23MxSwK3AwyV6LxGR2W/lLfC+u6B+MWCF5fvuKt9RN+6eM7NPAr+mMLzyfnffXIr3EhEpGytvOS/BfqaSjRVw90eBR0v1+iIiMjW6TU9EJOIU9CIiEaegFxGJOAW9iEjEKehFRCJOQS8iEnEKehGRiDP38KeWM7NuYPc0D28CJpgtKDKi/hn1+cqbPl94lrr7pHPIzIqgnwkza3f3trDrKKWof0Z9vvKmzzf7qetGRCTiFPQiIhEXhaC/J+wCzoOof0Z9vvKmzzfLlX0fvYiInF0UruhFROQsyjbozex+M+sys5fCrqUUzGyxmT1uZlvNbLOZfTrsmorJzDJmtsHMXgg+338Ou6ZSMLO4mf3RzP457FpKwcx2mdmLZva8mbWHXU+xmVmDmf3EzLYF/y/+q7Brmo6y7boxs7cDvcD33P3ysOspNjNrAVrc/TkzqwU2Aje7+5aQSysKMzOg2t17zSwJrAc+7e5/CLm0ojKz/wC0AXXufmPY9RSbme0C2twn+uXr8mZmDwBPuvu3g1/Lq3L3Y2HXda7K9ore3Z8AjoRdR6m4+353fy5Y7wG2AovCrap4vKA32EwGj/K86piAmbUC7wW+HXYtcu7MrA54O3AfgLsPlWPIQxkHfSUxs2XAlcAz4VZSXEG3xvNAF/CYu0fq8wH/Hfh7IB92ISXkwG/MbKOZrQ27mCK7AOgGvhN0v33bzKrDLmo6FPSznJnVAD8FPuPuJ8Kup5jcfcTdrwBagTVmFpkuODO7Eehy941h11Ji17j7auAG4PagSzUqEsBq4G53vxLoAz4bbknTo6CfxYK+658CP3D3h8Kup1SCP4f/BXh3yKUU0zXA+4M+7B8D7zCz74dbUvG5+75g2QX8DFgTbkVF1Ql0jvlL8ycUgr/sKOhnqeDLyvuAre7+9bDrKTYzazazhmA9C/wFsC3cqorH3T/n7q3uvgy4Ffitu/91yGUVlZlVBwMFCLo03glEZhScux8AOszs4qDpOqAsB0Mkwi5guszsR8C1QJOZdQL/4O73hVtVUV0DfAR4MejHBvi8uz8aYk3F1AI8YGZxChccD7p7JIcgRth84GeFaxISwA/d/VfhllR0/x74QTDi5jXgYyHXMy1lO7xSRESmRl03IiIRp6AXEYk4Bb2ISMQp6EVEIk5BLyIScQp6EZGIU9CLiEScgl5EJOL+P2hKqYG7wKxGAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHVlJREFUeJzt3XuQnHW95/H3t+8990lmMplkQhIwoIAJlxj14PFwRNR4OcA5LoVb60HLLU4VeNTaU7Wl7JaHtWTX2rPKrq5yFgTFEnVTikcUvLABBY5ASBASyIUk5DKT20xuc0vm0tPf/aOfCZNkkpnMdOeZfvrzqup6nufXTz/9bS6f3zO//j1Pm7sjIiLRFQu7ABERKS0FvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4hT0IiIRp6AXEYm4RNgFADQ1NfmiRYvCLkNEpKysW7fuoLs3T7TfjAj6RYsWsXbt2rDLEBEpK2a2azL7aehGRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYgr66Dfsr+Xf/rtZo4eGwq7FBGRGausg37XoX6+/dR22g8fD7sUEZEZa8KgN7MFZvaUmW0ys9fM7PNB+11mtsfMXg4eHx7zmi+Z2TYz22JmHyxV8S11GQAO9AyU6i1ERMreZG6BkAP+wd1fMrNaYJ2ZPRE8d4+7/4+xO5vZpcAtwGXAPOD/mdnF7j5SzMJhTND3KuhFRM5kwjN6d9/n7i8F673AJmD+WV5yA/ATdx909x3ANmBFMYo9VVNNCgMO9AyW4vAiIpFwTmP0ZrYIuBJ4IWj6rJmtN7MHzawxaJsPtI95WQdn7ximLBGP0VCVpFNDNyIiZzTpoDezGuBnwBfcvQe4F7gIuALYB3x9dNdxXu7jHO82M1trZmu7urrOufBR9dmkxuhFRM5iUkFvZkkKIf+wuz8C4O4H3H3E3fPA/bw5PNMBLBjz8jZg76nHdPf73H25uy9vbp7wdspnVJ9Nsl9BLyJyRpOZdWPAA8Amd//GmPbWMbvdBLwarD8K3GJmaTNbDCwB1hSv5JM1VCU1Ri8ichaTmXVzDfBJYIOZvRy03Ql8wsyuoDAssxP4OwB3f83MVgEbKczYuaMUM25GNVSlONw/xFAuTypR1pcFiIiUxIRB7+7PMv64++Nnec3dwN3TqGvSGqqSAHT1DTK/IXs+3lJEpKyU/SlwYxD0+kJWRGR8ZR/0DdkUgKZYioicQfkH/Ykzen0hKyIynrIP+ppMgphp6EZE5EzKPuhjZtSkEzqjFxE5g7IPeoDqdIJO3dhMRGRckQj6qlScfUcV9CIi44lE0FenE7oNgojIGUQi6GvSCfoGcxwbyoVdiojIjBOJoK/NFC7w3avhGxGR00Qj6NOFufR7j+q3Y0VEThWNoD9xRq+gFxE5VSSCvjqdwFDQi4iMJxJBH48ZtZkEe7s1Ri8icqpIBD0UboWgM3oRkdNFJ+hTCfYcUdCLiJwqOkGfSbCvewD3036HXESkokUm6GszSYZG8hzqHwq7FBGRGSVCQa8pliIi44lO0Kd1dayIyHiiE/QZXR0rIjKeyAR9JhkjGTcFvYjIKSIT9GZGXSbJPl00JSJyksgEPUB1Ok7HkWNhlyEiMqNEKuhrM0k6dNGUiMhJIhX0ddkkh/qH9AMkIiJjRCro64OZNzqrFxF5U7SCPlsI+t2HNE4vIjIqUkFfly1cNNWuL2RFRE6IVNBnk3FSiRi7DyvoRURGTRj0ZrbAzJ4ys01m9pqZfT5on2VmT5jZ1mDZGLSbmX3TzLaZ2Xozu6rUH2JMrdRnk7Qr6EVETpjMGX0O+Ad3fxvwLuAOM7sU+CKw2t2XAKuDbYCVwJLgcRtwb9GrPovadEJn9CIiY0wY9O6+z91fCtZ7gU3AfOAG4KFgt4eAG4P1G4AfeMHzQIOZtRa98jOoyybZffiY7ksvIhI4pzF6M1sEXAm8ALS4+z4odAbAnGC3+UD7mJd1BG2nHus2M1trZmu7urrOvfIzqM8mGRjWfelFREZNOujNrAb4GfAFd+85267jtJ12eu3u97n7cndf3tzcPNkyJjQ680bDNyIiBZMKejNLUgj5h939kaD5wOiQTLDsDNo7gAVjXt4G7C1OuRMbvWhKX8iKiBRMZtaNAQ8Am9z9G2OeehS4NVi/FfjFmPa/DWbfvAvoHh3iOR/qsgp6EZGxEpPY5xrgk8AGM3s5aLsT+Bqwysw+A+wG/k3w3OPAh4FtwDHg00WteALJeIwazbwRETlhwqB392cZf9wd4Lpx9nfgjmnWNS11mQS7dBsEEREgYlfGjqqvSvLGwf6wyxARmREiGfSNVSm6egfpH9TtikVEIhn0DVWFL2R36KxeRCSaQd9YlQIU9CIiENGgb8jqjF5EZFQkgz4Rj1GfTSroRUSIaNBD4Z43b3T1hV2GiEjoIhv0DdnCFEvdxVJEKl10g74qSe9AjsO6i6WIVLjIBr1m3oiIFEQ26Efn0usKWRGpdJEN+rpMkriZzuhFpOJFNuhjMaOhKsm2Ts28EZHKFtmgB2isTrFlf2/YZYiIhCrSQT+7OkX74WMMDI+EXYqISGgiHfQfHHmaZ1KfI333bLjncli/KuySRETOu8n8wlRZuqTz17y/6+ukYoOFhu52+OXnCutLbw6vMBGR8yyyZ/Tv2f0dUj54cuPwcVj9lXAKEhEJSWSDvnbwwPhPdHec30JEREIW2aDvTbeM/0R92/ktREQkZJEN+mcvuJ3hWObkxmQWrvtyOAWJiIQksl/GbpmzEoB37vjfNA53kaudR+oDd+mLWBGpOJENeiiE/R+r3scPX9jNPTcu46alGrYRkcoT2aGbUQ1VKeJmbNmvWyGISGWKfNDHY8bsmhSb9/eEXYqISCgiH/QAs2tSvLqnO+wyRERCURFB31yT5mDfEJ29A2GXIiJy3lVG0NemAdi0T3eyFJHKUxlBX1MI+tf2avhGRCrPhEFvZg+aWaeZvTqm7S4z22NmLwePD4957ktmts3MtpjZB0tV+LlIJ+M0ZJNs3KsvZEWk8kzmjP77wIfGab/H3a8IHo8DmNmlwC3AZcFrvmNm8WIVOx2za1K8pqAXkQo0YdC7+9PA4Uke7wbgJ+4+6O47gG3AimnUVzTNNWl2HuynfzAXdikiIufVdMboP2tm64OhncagbT7QPmafjqDtNGZ2m5mtNbO1XV1d0yhjcppr0ziwWT8tKCIVZqpBfy9wEXAFsA/4etBu4+zr4x3A3e9z9+Xuvry5uXmKZUze6Mybjfs0fCMilWVKQe/uB9x9xN3zwP28OTzTASwYs2sbsHd6JRZHTTpBVSrOho6jYZciInJeTSnozax1zOZNwOiMnEeBW8wsbWaLgSXAmumVWBxmxpzaNC+3K+hFpLJMePdKM/sxcC3QZGYdwD8C15rZFRSGZXYCfwfg7q+Z2SpgI5AD7nD3kdKUfu5a6jKs2XGYvsEcNelI37hTROSECdPO3T8xTvMDZ9n/buDu6RRVKnPrMjjw6p5u3nXh7LDLERE5LyriythRLXWFX5x6RcM3IlJBKiros6k4jVVJjdOLSEWpqKCHwjTLPynoRaSCVFzQz63LsL97gM4e3bJYRCpDxQX96Di9hm9EpFJUXNDPqU0TMwW9iFSOigv6RDzGnNoML+6c7H3aRETKW8UFPcC8hgwvtx9lYHjGXMslIlIyFRr0WYZHnPUd+sUpEYm+ig16QMM3IlIRKjLos8k4TTUp1uxQ0ItI9FVk0APMrc+wbtcRRvLj3i5fRCQyKjbo5zdk6RvMsUk/RCIiEVexQa9xehGpFBUb9HWZJPXZJM9tPxR2KSIiJVWxQQ/Q1pjlj9sPkRvJh12KiEjJVHTQXzCrir7BHBv2aD69iERXRQd9W2NhnP7ZrQdDrkREpHQqOuirUgnm1KV5dpuCXkSiq6KDHmBBQxXrdh3h2FAu7FJEREpCQT8rSy7vvKCrZEUkoio+6Oc3ZEnETOP0IhJZFR/0iXiM+Q1ZntzcGXYpIiIlUfFBD7CoqZodB/vZcbA/7FJERIpOQQ9c2FQNwOpNB0KuRESk+BT0QF02SXNNmtWbNHwjItGjoA8snF3Fmp2H6T4+HHYpIiJFpaAPLG6qZiTvPP16V9iliIgUlYI+MLc+Q1UqrnF6EYmcCYPezB40s04ze3VM2ywze8LMtgbLxqDdzOybZrbNzNab2VWlLL6YYmYsml3NE5sOMJgbCbscEZGimcwZ/feBD53S9kVgtbsvAVYH2wArgSXB4zbg3uKUeX4saamhf3CEZ17XxVMiEh0TBr27Pw2cen+AG4CHgvWHgBvHtP/AC54HGsystVjFltqCxiqyyTiPbdgXdikiIkUz1TH6FnffBxAs5wTt84H2Mft1BG2nMbPbzGytma3t6poZX4DGY8bipmqe2HiAgWEN34hINBT7y1gbp83H29Hd73P35e6+vLm5uchlTN3FLTX0DeZ4Rve+EZGImGrQHxgdkgmWo1cadQALxuzXBuydennnX1tjFR9P/ZGrH/lzuKsB7rkc1q8KuywRkSmbatA/CtwarN8K/GJM+98Gs2/eBXSPDvGUi0sP/oavxu5nVu4A4NDdDr/8nMJeRMrWZKZX/hh4DrjEzDrM7DPA14DrzWwrcH2wDfA48AawDbgfuL0kVZfQe3Z/hwyDJzcOH4fVXwmnIBGRaUpMtIO7f+IMT103zr4O3DHdosJUO3iGC6a6O85vISIiRaIrY0/Rm24Z/4n6tvNbiIhIkSjoT/HsBbczHMuc3JjMwnVfDqcgEZFpmnDoptJsmbMSKIzV1wweoNOaaPnYf8WW3hxyZSIiU6OgH8eWOSvZMmclm/b18LuNB3i46p1cE3ZRIiJTpKGbs1gyp4aqVJwfPLcz7FJERKZMQX8WiXiMt7XW8cTGA+w9ejzsckREpkRBP4Gl8+txhx+9sDvsUkREpkRBP4G6bJLFTdX8aM1u3adeRMqSgn4SlrbVc7h/iF+9UlZ3cxARART0k3LBrCqaalL88x+2k8+PezNOEZEZS0E/CWbG1Rc0srWzj6e2dE78AhGRGURBP0lLWmqpzya59/fbwy5FROScKOgnKR4zrljQwNpdR3hx56m/rCgiMnMp6M/BZfPqqErF+dbqrWGXIiIyaQr6c5CMx7jqgkae3nqQNTt0Vi8i5UFBf46WttVTk07wT7/dTOH2+yIiM5uC/hwl4zGWL2rkxZ1HeFo/IC4iZUBBPwWXz6unPpvkv/9ms+bVi8iMp6CfgnjMeOfiWby2t4dH/rQn7HJERM5KQT9Fb51bS2t9hq/9ehN9g7mwyxEROSMF/RSZGe9d0szBviG+9aSmW4rIzKWgn4a59Rkuba3lgWd2sONgf9jliIiMS0E/TX92URPxmPGln63XdEsRmZEU9NNUnU7wnrc08fyOw/zkxfawyxEROY2Cvggum1fHgsYsdz+2if3dA2GXIyJyEgV9EZgZ73vrHAZzI9z58w0awhGRGUVBXyQNVSnefeFsntzcyQ/1+7IiMoMo6IvoigUNLJpdxVd/tZEt+3vDLkdEBFDQF5WZ8f63tZCIG3//45cYGNaPiYtI+KYV9Ga208w2mNnLZrY2aJtlZk+Y2dZg2VicUstDdTrB9W9r4fUDffznf3lV4/UiErpinNH/pbtf4e7Lg+0vAqvdfQmwOtiuKAtnV7Ni8Sx+uq6DHzy3K+xyRKTClWLo5gbgoWD9IeDGErzHjPeuxbO4sKmar/xyI89tPxR2OSJSwaYb9A78zszWmdltQVuLu+8DCJZzpvkeZcnM+MBlLTRUJbn94XW6RYKIhGa6QX+Nu18FrATuMLP3TvaFZnabma01s7VdXV3TLGNmSififGRpK4O5PJ984AU6e3UxlYicf9MKenffGyw7gZ8DK4ADZtYKECw7z/Da+9x9ubsvb25unk4ZM1pjVYqPLZtHZ88gtz64ht6B4bBLEpEKM+WgN7NqM6sdXQc+ALwKPArcGux2K/CL6RZZ7ubWZfjw2+eyZX8v//6htRwb0v3rReT8mc4ZfQvwrJm9AqwBHnP33wBfA643s63A9cF2xVs4u5oPXDqXNTsP86kHX6RfP1YiIudJYqovdPc3gGXjtB8CrptOUVF1ydxaAH67cT+f+t4avv/pFVSnp/yvQERkUnRl7Hl2ydxaPnTZXNbtOsK/vf95DvUNhl2SiEScgj4EF7fU8uG3t/La3h7++jt/ZNchTb0UkdJR0IfkouYa/vqq+XT1DXLjt/+VHU99D+65HO5qKCzXrwq7RBGJCA0Qh6i1PsvHr2pj+OX/S8vv/xlsqPBEdzv88nOF9aU3h1egiESCzuhD1lid4oupVVSNhvyo4eOw+ivhFCUikaKgnwHqhg6M/0R3x/ktREQiSUE/A/SmW8Zt78vM1W2ORWTaFPQzwLMX3M5wLHNS2wBp7uy5iU9970XaDx8LqTIRiQIF/QywZc5KnrjoTnrSc3GMnvRcVi/5T/S85Sae236I6+/5A//nD9sZHsmHXaqIlCHNupkhtsxZyZY5K09qWwZc2FzNH17v4r/9ejOP/GkPd33sMt590exwihSRsqQz+hmuNpPko0vn8dGlrew7epxP3P88n3noRbZ16sfHRWRydEZfJi5qrmHhrCpebj/Ks1sP8sHNz3DzOxZwx19eRFtjVdjlicgMpqAvI4l4jOWLZnHpvDrW7DjMqhfbWbW2nb+5aj63X/sWFjVVh12iiMxACvoyVJVKcO0lc7h6YSPrdh3hkZf28NN1HXx06Tw+fc0irrygMewSRWQGUdCXsdpMkmsvmcM7Fs3ipd1H+O1r+3n0lb0snV/Pp65ZxEeWtpJOxMMuU0RCZjPhgpzly5f72rVrp/TaJzcf4JX27iJXVJ6Gcnk27eth/Z5uDvcP0ViV5KYr2/j41W1cOq8u7PJEpMjMbJ27L59oP53RR0gqEWPZggaWttWz+/AxXt3bw0PP7eTBf93BW+fW8vGr2/irZfOYU5eZ8FgiEh0K+ggyMxbOrmbh7GqOD4/w+v5eNu/v5auPbeLuxzZx1cJGVl4+lw9eNpcFszRjRyTqFPQRl03GWbaggWULGjjcP8S2zj62d/Xx1cc28dXHNnH5vDre97YW/uLiZpa11ZOI69IKkahR0FeQWdUpViyexYrFszh6bIjtXf1s7+rjW09u5Zurt1KbSfDnS5r4i4ub+bOLmmhrzGJmYZctItOkoK9QDVUprl6Y4uqFjQwMj7D78DF2HTrGM68f5PEN+wGYW5dhxeJZvGPxLFYsmsWSOTXEYgp+kXKjoBcyyTgXt9RycUst7s6h/iH2HDnOnqPHeWpzJ4++sheA+mySZW31LG1r4O1t9Sxtq2duXUZn/SIznIJeTmJmNNWkaapJs2xBA+5Oz0COPUePs/focTbv7+XZbQfJB7NyZ1WnWNZWz+Xz61nSUsslLbUsbqomldBYv8hMoaCXszIz6rNJ6rNJLm0tzMXPjeTp6huks2eQA70DbNjTzR9e7zoR/vGYsbipmkuCvxLeMqeGRU1VLJxdTU1a/8mJnG/6v07OWSIeo7U+S2t99kRbbiTPkWPDHOof5HD/EIf6hvjj9oM8vmEfYy/Jm12dYnFTYernotlVLGyqZuGsKlobMjRVp/UdgEgJKOilKBLxGM21aZpr0ye1D4/kOXpsmKPHhjh6fJju48Ps7xng9QO99AzkTto3GTda67PMa8gwryHLvPos8xqytDZkaK3P0FyTprEqpc5A5Bwp6KWkkmfoAKDQCXQH4d83kKN3MEfvwDDth4+xcW8PfYO5E8NBo+JmzK5JnThmc01h2TRm2VidpLEqRUNVUvf6EUFBLyFKxmMnvvgdTz7v9A/l6B3I0TeY49jQCMeGcvQPjtA/mONQ3xAvDR0Zt0MYlU3GaagqBH9jdZKGqhSNwXZ9trBdm0kUHukktZkENcG2OgmJCgW9zFixmFGbSVKbSZ51P3dnYDhf6ASGRhgYHn3kT6wfG8px5NgQg7leBoZHOD40wkS380vFY1Sn49RmktRlT+4IatIJsqk41akEVak42VS8sEwWtt9sG/N8Mq4rjyUUJQt6M/sQ8L+AOPBdd/9aqd5LKpuZkQ3CdLK/puvuDOYKHcFQLs9gLs/QSJ6hXP7N7VyewZHC830DOQ73DzGc8zf3G8kzcqY/Jc4gGTeyyUKt6UScdCJGJhknm4yTTsZIJ2In2gvbwXoiRjo5Zj1x+v7JRIxEzEjGY6TGW0/ESMUL6/GY6fqHMKxfBau/At0dUN8G130Zlt5c8rctSdCbWRz4NnA90AG8aGaPuvvGUryfyLkyMzLJOJnk9IZnRvJOLp9neMQZHsmTC5aFh5MLlsP509ty+UJH0TMwzJFjQ+TdGcl7cExnZKSwHD1+MRmFobNEvNAZJONGIhYjmbBCZxAvdAqjncToMh6LEY9BIhYjHjMSMSMWLE/ffrNTiY95/uT12ATHKNQVixW+n4nFjJgZMYOYjXZYhSm9Y9tPPGKMux43w4LtuBWOMXq8mFGaTnD9Kvjl52D4eGG7u72wDSUP+1Kd0a8Atrn7GwBm9hPgBkBBL5FSCK04pb48wN3JOyc6h9xohzDiJzqbvBc6nrw7+bwz4k4+T7D0U5Zvtp/oYIL3yOed4VyeweGRE/vmvbCfO4UHwb7BMdwhz+n75oNjlqOxHcBJHcaJtjc7hbi9uR4zTnRIxpttP+y9kxY/fvKbDB8vnOGXadDPB9rHbHcA7yzFG82uTrNYv5UqMmONdlLuflrHMdqxjAQdw9jOZ+x+I2M6mRMdDn7K9pj1Me95cnuwzuT2OdPxTnpfd/LB8cZ2hKM1jb6m2Q+O/w+ou6Pk/w5KFfTj/d1zUr9uZrcBtwFccMEFU36j0VvwiojMaPe0FYZrTlXfVvK3LtUUgA5gwZjtNmDv2B3c/T53X+7uy5ubm0tUhojIDHHdlyGZPbktmS20l1ipgv5FYImZLTazFHAL8GiJ3ktEZOZbejN87JtQvwCwwvJj3yzfWTfunjOzzwK/pTC98kF3f60U7yUiUjaW3nxegv1UJZsr4O6PA4+X6vgiIjI5ukxPRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYgz9/BvLWdmXcCuKb68CTjD3YIiI+qfUZ+vvOnzhWehu094D5kZEfTTYWZr3X152HWUUtQ/oz5fedPnm/k0dCMiEnEKehGRiItC0N8XdgHnQdQ/oz5fedPnm+HKfoxeRETOLgpn9CIichZlG/Rm9qCZdZrZq2HXUgpmtsDMnjKzTWb2mpl9PuyaisnMMma2xsxeCT7ffwm7plIws7iZ/cnMfhV2LaVgZjvNbIOZvWxma8Oup9jMrMHMfmpmm4P/F98ddk1TUbZDN2b2XqAP+IG7Xx52PcVmZq1Aq7u/ZGa1wDrgRnffGHJpRWFmBlS7e5+ZJYFngc+7+/Mhl1ZUZvYfgOVAnbt/NOx6is3MdgLL3c/0y9flzcweAp5x9+8Gv5ZX5e5Hw67rXJXtGb27Pw0cDruOUnH3fe7+UrDeC2wC5odbVfF4QV+wmQwe5XnWcQZm1gZ8BPhu2LXIuTOzOuC9wAMA7j5UjiEPZRz0lcTMFgFXAi+EW0lxBcMaLwOdwBPuHqnPB/xP4D8C+bALKSEHfmdm68zstrCLKbILgS7ge8Hw23fNrDrsoqZCQT/DmVkN8DPgC+7eE3Y9xeTuI+5+BdAGrDCzyAzBmdlHgU53Xxd2LSV2jbtfBawE7giGVKMiAVwF3OvuVwL9wBfDLWlqFPQzWDB2/TPgYXd/JOx6SiX4c/j3wIdCLqWYrgH+KhjD/gnwPjP7YbglFZ+77w2WncDPgRXhVlRUHUDHmL80f0oh+MuOgn6GCr6sfADY5O7fCLueYjOzZjNrCNazwPuBzeFWVTzu/iV3b3P3RcAtwJPu/u9CLquozKw6mChAMKTxASAys+DcfT/QbmaXBE3XAWU5GSIRdgFTZWY/Bq4FmsysA/hHd38g3KqK6hrgk8CGYBwb4E53fzzEmoqpFXjIzOIUTjhWuXskpyBGWAvw88I5CQngR+7+m3BLKrq/Bx4OZty8AXw65HqmpGynV4qIyORo6EZEJOIU9CIiEaegFxGJOAW9iEjEKehFRCJOQS8iEnEKehGRiFPQi4hE3P8H1DStq24uP4EAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] @@ -251,11 +263,11 @@ { "data": { "text/plain": [ - "Feerates:\t1.1499744606513134, 1.6228733928138133, 6.361686926992798 \n", - "Times:\t\t168.62498827393395, 59.999999999999986, 1.0000000000000004" + "Feerates:\t1.1499744606513134, 1.3970589103224236, 1.9124681884207781, 6.361686926992798 \n", + "Times:\t\t168.62498827393395, 94.04820895845543, 36.664092522353194, 1.0000000000000004" ] }, - "execution_count": 21, + "execution_count": 105, "metadata": {}, "output_type": "execute_result" } @@ -286,12 +298,12 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 106, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHS1JREFUeJzt3XlwXOWZ7/Hv04t2WYslG1uSsQFj9tiOMGQ8YZIwhG0qmCxzSSaEbOXcDLmVzORyb8i9U8nUrdRQxUwySyUwBLiBbFyGEMIkJIYQwpIQbBmMjTEGGxssy7ZkeZOstdXP/aOPHNmWrK1bRzr9+1R19em3z+l+muV3Xr399nvM3RERkeiKhV2AiIjkloJeRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRFwi7AIAampqfOHChWGXISIyo6xfv36/u9eOtt+0CPqFCxfS1NQUdhkiIjOKmb01lv00dCMiEnEKehGRiFPQi4hEnIJeRCTiFPQiIhGnoBcRiTgFvYhIxM3ooN+6t4Pb17zGoa6+sEsREZm2ZnTQv9V+lG8/tZ1dB7rDLkVEZNoaNejNrMHMnjKzLWa22cy+GLR/3cx2m9mG4HbNkGNuNbNtZrbVzK7MVfFzZxUBsO9IT67eQkRkxhvLEggp4Mvu/qKZlQPrzeyJ4Llvufs/Dt3ZzM4DbgDOB+YDvzazs919IJuFw5Cg71DQi4iMZNQevbvvcfcXg+0OYAtQd4pDrgMecPded98BbANWZKPYE9WUFWAG+4705uLlRUQiYVxj9Ga2EFgGvBA0fcHMNprZvWZWFbTVAbuGHNbMqU8ME5aIx6gpK6RVQzciIiMac9CbWRnwE+BL7n4EuAM4E1gK7AH+aXDXYQ73YV5vtZk1mVlTW1vbuAsfNHdWocboRUROYUxBb2ZJMiH/Q3d/GMDd97n7gLunge/yx+GZZqBhyOH1QMuJr+nud7l7o7s31taOupzyiOaWF2noRkTkFMYy68aAe4At7v7NIe3zhux2PfBKsP0ocIOZFZrZImAxsDZ7JR9vzqwiWvVlrIjIiMYy62YlcCOwycw2BG1fBT5qZkvJDMvsBD4H4O6bzexB4FUyM3ZuzsWMm0FzZxWyv7OPvlSagsSM/lmAiEhOjBr07v4cw4+7P3aKY74BfGMSdY3Z4BTLts5e6iqLp+ItRURmlBnfBT5NP5oSETmlGR/0c2YVAmiKpYjICGZ80P9xGQTNvBERGc6MD/rqkgISMdPQjYjICGZ80MdixpzyQvXoRURGMOODHjSXXkTkVCIR9FoGQURkZBEJ+iL2HlbQi4gMJxJBf1pFEUd6UnT1pcIuRURk2olE0M+vyPwituWQevUiIieKRtBXDga9rh0rInKiiAR95kdTCnoRkZNFIujnzirCDFr0hayIyEkiEfTJeIy55UXq0YuIDCMSQQ+Z4RsFvYjIySIU9MXs0dCNiMhJIhX0uw91437SdchFRPJadIK+ooi+VJr2o31hlyIiMq1EJ+g1l15EZFgRDHqN04uIDBXBoFePXkRkqMgEfVVJkqJkTEEvInKCyAS9mTG/QlMsRUROFJmgh8zwTbN69CIix4lU0NdVFrP7oIJeRGSoSAX9gtkl7O/s1QVIRESGiFTQN1SXANCsXr2IyDHRCvqqzBTLt9u7Qq5ERGT6iFTQLwh69LsOKuhFRAZFKuirSwsoKYjz9gEFvYjIoFGD3swazOwpM9tiZpvN7ItBe7WZPWFmbwT3VUG7mdm/mtk2M9toZstz/SGG1MqC6hJ2HdAYvYjIoLH06FPAl939XOBS4GYzOw/4CvCkuy8GngweA1wNLA5uq4E7sl71KTRUl7BLPXoRkWNGDXp33+PuLwbbHcAWoA64Drgv2O0+YFWwfR1wv2f8Aag0s3lZr3wEDVUlvH2gS+vSi4gExjVGb2YLgWXAC8Bcd98DmZMBMCfYrQ7YNeSw5qDtxNdabWZNZtbU1tY2/spHsKC6mO7+Aa1LLyISGHPQm1kZ8BPgS+5+5FS7DtN2Uvfa3e9y90Z3b6ytrR1rGaManEuvL2RFRDLGFPRmliQT8j9094eD5n2DQzLBfWvQ3gw0DDm8HmjJTrmjOzbFUkEvIgKMbdaNAfcAW9z9m0OeehS4Kdi+CfjZkPZPBLNvLgUODw7xTIX6KgW9iMhQiTHssxK4EdhkZhuCtq8CtwEPmtlngLeBjwTPPQZcA2wDuoBPZbXiURQXxKktL9QUSxGRwKhB7+7PMfy4O8Dlw+zvwM2TrGtSFlSXsLP9aJgliIhMG5H6ZeygRTWlCnoRkUBkg37fkV6O9mq5YhGRSAb9GTWlAOzYr169iEgkg35RrYJeRGRQJIN+4WwFvYjIoEgGfVEyTl1lsYJeRISIBj1kvpB9U0EvIhLtoN/R1qlVLEUk70U66I/0pDigVSxFJM9FN+g180ZEBIhw0A/Opdc4vYjku8gGfV1lMcm4qUcvInkvskGfiMc4fXYp21o7wy5FRCRUkQ16gMVzyhT0IpL3oh30c8t5q/0oPf0DYZciIhKaSAf94a4+0g7n/t2vWHnbb3jkpd1hlyQiMuUiG/SPvLSbB9btAjJXJt99qJtbH96ksBeRvBPZoL99zVZ6U+nj2rr7B7h9zdaQKhIRCUdkg77l0PDXjB2pXUQkqiIb9PMri8fVLiISVZEN+luuXEJxMn5cW3Eyzi1XLgmpIhGRcCTCLiBXVi2rA+Dv/3MzB7v6qS0v5H9dc+6xdhGRfBHZHj1kwv7Bz70LgK9ec45CXkTyUqSDHmBhTSnJuLF1r34hKyL5KfJBn4zHOLO2jNf2Hgm7FBGRUEQ+6AHOmz+LV1sU9CKSn/Ii6M+fX0FrRy9tHb1hlyIiMuXyIujPmzcLgFf3qFcvIvknv4JewzcikodGDXozu9fMWs3slSFtXzez3Wa2IbhdM+S5W81sm5ltNbMrc1X4eFSUJKmvKmZzy+GwSxERmXJj6dF/D7hqmPZvufvS4PYYgJmdB9wAnB8c8x0ziw9z7JQ7b94sDd2ISF4aNejd/RngwBhf7zrgAXfvdfcdwDZgxSTqy5rz5s9ix/6jdPWlwi5FRGRKTWaM/gtmtjEY2qkK2uqAXUP2aQ7aTmJmq82sycya2traJlHG2Jw/vwJ3eG1vR87fS0RkOplo0N8BnAksBfYA/xS02zD7+nAv4O53uXujuzfW1tZOsIyxO29+5gvZzfpCVkTyzISC3t33ufuAu6eB7/LH4ZlmoGHIrvVAy+RKzI75FUVUlSR5pVlfyIpIfplQ0JvZvCEPrwcGZ+Q8CtxgZoVmtghYDKydXInZYWa8o6GSl5sPhV2KiMiUGnWZYjP7MfAeoMbMmoGvAe8xs6VkhmV2Ap8DcPfNZvYg8CqQAm5294HclD5+76iv5JnX3+Bob4rSwsiu0CwicpxR087dPzpM8z2n2P8bwDcmU1SuLG2oJO2wafdhLj1jdtjliIhMibz4ZeygdzRUAvDyLg3fiEj+yKugry4tYEF1icbpRSSv5FXQQ6ZX//IuzbwRkfyRd0G/tKGS3Ye6ae3oCbsUEZEpkYdBXwGgXr2I5I28C/rz51eQiBkvvX0w7FJERKZE3gV9UTLO+XUVNO1U0ItIfsi7oAdYsbCKDc2H6E1Nm99yiYjkTF4GfePCavpSaTZq3RsRyQN5GfQXL6wGYO2OsS6zLyIyc+Vl0FeXFnDWnDLW7VTQi0j05WXQQ6ZXv37nQQbSwy6XLyISGXkb9CsWVdHRm+K1vboQiYhEW94G/eA4/TqN04tIxOVt0NdXlVBXWczzb7aHXYqISE7lbdAD/OlZNfx+e7vG6UUk0vI66FcurqGjJ8VGLVssIhGW30F/ZuYqU7/btj/kSkREcievg352WSHnzZvFcwp6EYmwvA56gHcvrmH9Wwfp6kuFXYqISE7kfdCvPKuG/gHXcggiEll5H/QrFlVTkIjx3BsavhGRaMr7oC9KxrlkUTVPbW0NuxQRkZzI+6AHuPycOWxvO8rO/UfDLkVEJOsU9MDl584F4MnX1KsXkehR0AMN1SWcPbeMJ7fsC7sUEZGsU9AH3nfOXNbuOMCRnv6wSxERySoFfeDPz51DKu0883pb2KWIiGSVgj6wbEEVVSVJntyicXoRiZZRg97M7jWzVjN7ZUhbtZk9YWZvBPdVQbuZ2b+a2TYz22hmy3NZfDbFY8bl587l11v20ZsaCLscEZGsGUuP/nvAVSe0fQV40t0XA08GjwGuBhYHt9XAHdkpc2pce+E8OnpSWuRMRCJl1KB392eAE9cHuA64L9i+D1g1pP1+z/gDUGlm87JVbK6tPKuGWUUJfr5xT9iliIhkzUTH6Oe6+x6A4H5O0F4H7BqyX3PQdhIzW21mTWbW1NY2Pb4ALUjEuPL803his4ZvRCQ6sv1lrA3TNuzlm9z9LndvdPfG2traLJcxcddeNI+O3hTPvq7hGxGJhokG/b7BIZngfnCqSjPQMGS/eqBl4uVNvZVn1VBRnOQXmzR8IyLRMNGgfxS4Kdi+CfjZkPZPBLNvLgUODw7xzBTJeIyrzj+Nxzfv1Rr1IhIJY5le+WPgeWCJmTWb2WeA24ArzOwN4IrgMcBjwJvANuC7wF/npOocu355HUf7BlizeW/YpYiITFpitB3c/aMjPHX5MPs6cPNkiwrbioXVNFQX89D6Zq5fVh92OSIik6Jfxg4jFjM+tLye329vZ/eh7rDLERGZFAX9CD60vB53+OmLzWGXIiIyKQr6ETRUl3DpGdU8tL6ZzIiUiMjMpKA/hY+8s4Gd7V08/2Z72KWIiEyYgv4Urr1oHpUlSb7//FthlyIiMmEK+lMoSsb5Lxc38Pir+9hzWF/KisjMpKAfxccvOZ20Oz964e2wSxERmRAF/Sgaqkt435I5/HjtLvpS6bDLEREZNwX9GNz4rtPZ39nLLzbNqGV7REQABf2YXLa4lrPnlvHvT7+pqZYiMuMo6McgFjM+d9mZvLa3g99unR5r54uIjJWCfow+sHQ+8yuKuOO328MuRURkXBT0Y5SMx/jsu89g7c4DrH/rxCsriohMXwr6cbhhRQNVJUn+7Tfbwi5FRGTMFPTjUFKQYPVlZ/LbrW007VSvXkRmBgX9ON30J6dTU1bI7Wu2agaOiMwICvpxKilI8IX3nskLOw7w3DZdQFxEpj8F/QR89JIF1FUWc/uaraTT6tWLyPSmoJ+AwkScv7nibDY2H+aRDbvDLkdE5JQU9BP0wWV1vKO+gtt++RpHe1NhlyMiMiIF/QTFYsbXPnA+rR29fPspTbcUkelLQT8JyxdU8cFlddz97A7eaj8adjkiIsNKhF3ATPc/rz6HX2xs4f3feoa+VJr5lcXccuUSVi2rC7s0ERFAQT9pz29vJw30B2vV7z7Uza0PbwJQ2IvItKChm0m6fc1W+geOn2LZ3T/A7Wu2hlSRiMjxFPST1HJo+GvJjtQuIjLVFPSTNL+yeFztIiJTTUE/SbdcuYTiZPyk9hsvXRBCNSIiJ1PQT9KqZXX8wwcvpK6yGANOm1VEaUGcRza00NM/EHZ5IiKTm3VjZjuBDmAASLl7o5lVA/8PWAjsBP7S3Q9OrszpbdWyuuNm2Dz1Wiuf+t46vv7oZv7hgxdiZiFWJyL5Lhs9+ve6+1J3bwwefwV40t0XA08Gj/PKe8+Zw83vPZMH1u3ihy+8HXY5IpLncjF0cx1wX7B9H7AqB+8x7f3tFUt475Javv7oZtbu0EVKRCQ8kw16Bx43s/Vmtjpom+vuewCC+zmTfI8ZKR4z/vmGZSyoLuHzP1ivJRJEJDSTDfqV7r4cuBq42cwuG+uBZrbazJrMrKmtrW2SZUxPFcVJvntTIwPufOLetbR19IZdkojkoUkFvbu3BPetwE+BFcA+M5sHENy3jnDsXe7e6O6NtbW1kyljWjuztox7P3kx+4708OnvraNTSxqLyBSbcNCbWamZlQ9uA+8HXgEeBW4KdrsJ+Nlki5zpli+o4jt/tZxX9xxh9f1NdPdp2qWITJ3J9OjnAs+Z2cvAWuAX7v4r4DbgCjN7A7gieJz33nfOXP7xIxfx/JvtfOa+dQp7EZkyE55H7+5vAu8Ypr0duHwyRUXV9cvqAfjygy/z6e+t455PNlJSoAVERSS39MvYKXb9snq++ZdLeWFHOzfes5aDR/vCLklEIk5BH4JVy+r49seWs2n3YT505+/ZdaAr7JJEJMIU9CG5+sJ5/OAzl9De2cf13/k9G5sPhV2SiESUgj5EKxZV85PPv4vCRIwP3/k8D61vDrskEYkgBX3IzppTzqNfWEnj6VX89/94mb975BX6gssSiohkg4J+GphdVsj9n17B6svO4Pt/eIuP3Pl7duzXkgkikh0K+mkiEY/x1WvO5c6PL2dnexfX/Muz/Hjt27j76AeLiJyCgn6aueqCeaz50mUsP72SWx/exGfva9L1Z0VkUhT009BpFUV8/9OX8L+vPZffbd/PFd98mnue20FqQGP3IjJ+CvppKhYzPvvuM3jib/6MFYuq+T8/f5VV3/kd63ZqbXsRGR8F/TTXUF3CvZ+8mO/81XLaOnr5yJ3P81+/v15f1orImGmhlRnAzLjmwnm8Z0ktdz+7gzuf3s6vt+zjY5cs4K/fcxanVRSFXaKITGM2HWZ1NDY2elNTU9hlzBitHT1864k3eLBpF3EzPtxYz+f/7EwaqkvCLk1EppCZrR9yve6R91PQz1y7DnRxx9PbeaipmbQ7H1g6n0+vXMQFdRVhlyYiU0BBn0f2HO7m359+kwebdtHVN8A7T6/ik3+ykKsuOI1kXF/DiESVgj4PHe7u56H1zdz//E7eau+itryQDy6r40PvrOfsueVhlyciWaagz2PptPPb11v50Qu7eGprKwNp56L6Cj78znquvXAes8sKwy5RRLJAQS8A7O/s5WcbWnhofTNb9hwhZplVM6++YB5Xnn+aZuyIzGAKejnJlj1H+OWmPfzylb280doJwLIFlVx+zhwuO7uWC+ZXEItZyFWKyFgp6OWUtrV28qtX9rBm8z427T4MQHVpAe9eXMNli2v508U1zJ2l3r7IdKaglzHb39nLc2/s5+nX23j2jTb2d2auY7uguoSLF1azYlEVFy+sZlFNKWbq8YtMFwp6mZB02nl1zxFe2HGAtTvaadp5kPbgAuY1ZQUsbajkwrpKLqqv4IK6CmrL9cWuSFgU9JIV7s72tqOs23mAdTsPsLH5MNvbOhn8z2ZeRREX1lVw/vwKlpxWxtlzyzl9dilxjfWL5NxYg15r3cgpmRlnzSnjrDllfHTFAgA6e1Ns3n2YTcFtY/NhHn9137FjChIxzqwtY8ncMhbPLWfxnDIW1ZTSUF1CUTIe1kcRyVsKehm3ssIEl5wxm0vOmH2srasvxbbWTrbu7eCN1k5e39fB2h0HeGRDy7F9zGDerCJOn13KwpqSzP3sEhqqS6irLKaiOKnvAERyQEEvWVFSkOCi+kouqq88rr2jp5/tbUd5q/0oO/d3Ze7bj/L45n3Hxv4HFSfjzK8sYn5lMfMripk3ZPu0iiJqywuZVZTQyUBknBT0klPlRUmWNlSytKHypOeO9PTzdnsXbx/oouVQN3sO99ByqJuWwz28treVto7ek44pSMSoLSuktryQmuD+2K2skNryAipLCqgqKaCiOKnvCkRQ0EuIZhUluaCuYsTVNntTA+w73EvL4W72Hu5hf2cvbR3BrbOX5oNdbNiVmRU00pyCWUUJqkoHwz957ARQVVJAVWmSiuIks4qSlBUlKC9KUFaYoLwoSVlhQicJiQwFvUxbhYk4C2aXsGD2qdfZTw2kOXC0j9aOXvZ39nKoq5+DXX0c7Orn0JD79s4+trV2cqirn87e1KjvX1oQp7womTkBFCWObZcXJigtTFBSEKe4IE5JMk5JQSKzPdhWEDyfzLSVFCQoSsY07CShyFnQm9lVwL8AceBud78tV+8l+S0RjzFnVhFzxvFL3r5UmkPdfRzp7udIT4rOnhQdPSk6e/vp6EkNacucFDp6Uhzu6qP5YBcdPSmO9qbo7h8Y8S+J4ZhxLPiLC+IUJeIUJmMUJuIUJmLBbbBtSPsJ+xQlTzwuTkEiRjJuJOOx4HbCdiJGMpbZjsdMJ5yQPPLSbm5fs5WWQ93MryzmliuXsGpZXc7fNydBb2Zx4NvAFUAzsM7MHnX3V3PxfiLjVZCIMae8iDnlE1/mwd3p6U/T1Zeiq2+A7v4BuvoG6OpL0d2X2e4OHnf1Dxxry7Sn6BtI09ufpjeVpjc1QEdPit7UQOZxf/rYdk//AOks/tzFjGOhn0zESMRiFBzbzpwgCobZTsQz9/Eht8zj49tH2ice47h9EzEjNmT/xDD7DH0Ns8x2zCBmlrnFhmwbwfOGnbh9bJ8Tjjnh+FyeAB95aTe3PryJ7v4BAHYf6ubWhzcB5Dzsc9WjXwFsc/c3AczsAeA6QEEvkWFmFAe989mj7z4pqYHBE0JwAhhyghg8MfSn0/Sn0vQPOKl0mr5gu38gHdwy26mBNH0jbPcPOH0nbB/tTQVtTtqdVNoZSPswj9OkHVLpdOZx2sf1F890YMFJJB6cIGI2+BfQH08cx51oDGKxk7cNjp1kBu9f39dB/8Dx/0C6+we4fc3WGRv0dcCuIY+bgUty9F4ikZeIx0jEY5TOsBUn0mlnwP1Y8A8MZB4fOxmc8uSRObkM+B/b3SEdvF462M7cMu813PaAO+4e1JL5SyzzGgSvecL2cK+RDl7juNcj2DdT10DwWXGOqzUdvOfmliPD/jNqOdSd838PuQr64f7+Oe5UZmargdUACxYsyFEZIhKmWMyIYegH0bDytt+we5hQn19ZnPP3ztUFRZuBhiGP64GWoTu4+13u3ujujbW1tTkqQ0RkerjlyiUUn3DGK07GueXKJTl/71z16NcBi81sEbAbuAH4WI7eS0Rk2hsch4/MrBt3T5nZF4A1ZKZX3uvum3PxXiIiM8WqZXVTEuwnytk8end/DHgsV68vIiJjk6sxehERmSYU9CIiEaegFxGJOAW9iEjEKehFRCJOQS8iEnEKehGRiDOfBsvLmVkb8NYED68B9mexnOko6p9Rn29m0+cLz+nuPuoaMtMi6CfDzJrcvTHsOnIp6p9Rn29m0+eb/jR0IyIScQp6EZGIi0LQ3xV2AVMg6p9Rn29m0+eb5mb8GL2IiJxaFHr0IiJyCjM26M3sXjNrNbNXwq4lF8yswcyeMrMtZrbZzL4Ydk3ZZGZFZrbWzF4OPt/fh11TLphZ3MxeMrOfh11LLpjZTjPbZGYbzKwp7HqyzcwqzewhM3st+H/xXWHXNBEzdujGzC4DOoH73f2CsOvJNjObB8xz9xfNrBxYD6xy91dDLi0rzMyAUnfvNLMk8BzwRXf/Q8ilZZWZ/S3QCMxy978Iu55sM7OdQKO7T9d55pNiZvcBz7r73WZWAJS4+6Gw6xqvGdujd/dngANh15Er7r7H3V8MtjuALcDUX5omRzyjM3iYDG4zs9cxAjOrB64F7g67Fhk/M5sFXAbcA+DufTMx5GEGB30+MbOFwDLghXArya5gWGMD0Ao84e6R+nzAPwP/A0iHXUgOOfC4ma03s9VhF5NlZwBtwP8Nht/uNrPSsIuaCAX9NGdmZcBPgC+5+5Gw68kmdx9w96VAPbDCzCIzBGdmfwG0uvv6sGvJsZXuvhy4Grg5GFKNigSwHLjD3ZcBR4GvhFvSxCjop7Fg7PonwA/d/eGw68mV4M/h3wJXhVxKNq0EPhCMYT8AvM/MfhBuSdnn7i3BfSvwU2BFuBVlVTPQPOQvzYfIBP+Mo6CfpoIvK+8Btrj7N8OuJ9vMrNbMKoPtYuDPgdfCrSp73P1Wd69394XADcBv3P3jIZeVVWZWGkwUIBjSeD8QmVlw7r4X2GVmS4Kmy4EZORkiEXYBE2VmPwbeA9SYWTPwNXe/J9yqsmolcCOwKRjHBviquz8WYk3ZNA+4z8ziZDocD7p7JKcgRthc4KeZPgkJ4Efu/qtwS8q6/wb8MJhx8ybwqZDrmZAZO71SRETGRkM3IiIRp6AXEYk4Bb2ISMQp6EVEIk5BLyIScQp6EZGIU9CLiEScgl5EJOL+P0ZSdzmC3LhuAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHW9JREFUeJzt3Xl0XGeZ5/HvU6WSVLKszVpiSbblJMZZyGLjOAHTaUgAk8CQhKUn0ECGgTZNBw6cYTJDaOYA5wwzORMCPX2gM52QNMlAJxMghEAHTAiBEAix5Wze4tiJN8mbbEebtZbqmT/qypZt2ZalKt/Srd/nHJ1776t7q57K8ruv3nrvvebuiIhIdMXCLkBERHJLQS8iEnEKehGRiFPQi4hEnIJeRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQirijsAgBqa2u9paUl7DJERKaVNWvW7Hf3ulPtlxdB39LSQmtra9hliIhMK2a2fSL7aehGRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYib1kG/aU8Pt698mc6+obBLERHJW9M66LcfOMR3n3yVnQf7wy5FRCRvTeugb6goBWBv90DIlYiI5K9TBr2ZzTGzJ81so5mtN7PPB+1fM7N2M3sh+Ll2zDG3mtkWM9tkZstzVfzhoO9R0IuInMhE7nWTAr7o7s+Z2UxgjZk9Hvzu2+7+zbE7m9kFwI3AhUAj8Bsze4O7j2SzcIDa8mLMYG/3YLZfWkQkMk7Zo3f33e7+XLDeA2wEmk5yyHXAg+4+6O5bgS3A0mwUe6yieIza8hL2aehGROSETmuM3sxagEXAs0HTZ83sJTO718yqg7YmYOeYw9oY58RgZivMrNXMWjs6Ok678FENFSUaoxcROYkJB72ZlQM/Ab7g7t3AncA5wKXAbuCO0V3HOdyPa3C/y92XuPuSurpT3k75hBpmlmroRkTkJCYU9GaWIBPyP3T3hwHcfa+7j7h7GribI8MzbcCcMYc3A7uyV/LR6itK2acvY0VETmgis24MuAfY6O7fGtM+e8xuNwDrgvVHgRvNrMTM5gMLgFXZK/loDRUl7O8dYiiVztVbiIhMaxOZdbMM+Biw1sxeCNq+DHzYzC4lMyyzDfg0gLuvN7OHgA1kZuzcnIsZN6NGp1h29A7SVJXM1duIiExbpwx6d3+a8cfdHzvJMd8AvjGFuibsrDEXTSnoRUSON62vjAWorygB0BRLEZETmPZBf+Q2CJp5IyIynmkf9DVlxRTFTHPpRUROYNoHfSxm1M8sUY9eROQEpn3Qg+bSi4icTCSCXrdBEBE5sYgEfSl7uhT0IiLjiUzQdw+k6BtKhV2KiEjeiUTQj14otatTvXoRkWNFIugbDwe9nh0rInKsiAR95qIpBb2IyPEiEfQNFaWYwS59ISsicpxIBH0iHqNhZql69CIi44hE0ENm+EZBLyJyvMgE/eyqJLs1dCMicpzIBH1TVZL2zn7cj3s8rYhIQYtM0DdWljKUSnPg0FDYpYiI5JXoBH0wl363LpoSETlK5IK+XV/IiogcJXJBr5k3IiJHi0zQV5clKE3E2N2loBcRGSsyQW9mNFYmdWMzEZFjRCboITN806ahGxGRo0Qq6JuqkrS/rqAXERkrUkE/d1YZ+3sH6R8aCbsUEZG8Eamgn1NTBsDO1/tCrkREJH9EK+irM1MsdxxQ0IuIjIpU0M9Vj15E5DiRCvqaGcWUFcfZcVBBLyIy6pRBb2ZzzOxJM9toZuvN7PNBe42ZPW5mm4NlddBuZvaPZrbFzF4ys8W5/hBjamVuTRk7D2rmjYjIqIn06FPAF939fOAK4GYzuwD4EvCEuy8Angi2Aa4BFgQ/K4A7s171STRXl7FTPXoRkcNOGfTuvtvdnwvWe4CNQBNwHXBfsNt9wPXB+nXA/Z7xZ6DKzGZnvfITmFtTxo6DfbovvYhI4LTG6M2sBVgEPAs0uPtuyJwMgPpgtyZg55jD2oK2Y19rhZm1mllrR0fH6Vd+AnNrkvQPj+i+9CIigQkHvZmVAz8BvuDu3SfbdZy247rX7n6Xuy9x9yV1dXUTLeOURufS6wtZEZGMCQW9mSXIhPwP3f3hoHnv6JBMsNwXtLcBc8Yc3gzsyk65p3Z4iqWCXkQEmNisGwPuATa6+7fG/OpR4KZg/SbgZ2PaPx7MvrkC6Bod4jkTmqsV9CIiYxVNYJ9lwMeAtWb2QtD2ZeA24CEz+ySwA/hQ8LvHgGuBLUAf8ImsVnwKyeI4dTNLNMVSRCRwyqB396cZf9wd4Opx9nfg5inWNSVza8rYduBQmCWIiOSNSF0ZO2p+7QwFvYhIILJBv7d7kEODqbBLEREJXSSD/uzaGQBs3a9evYhIJIN+fp2CXkRkVCSDvmWWgl5EZFQkg740EaepKqmgFxEhokEPmS9kX1PQi4hEO+i3dvTqLpYiUvAiHfTdAykO6i6WIlLgohv0mnkjIgJEOOhH59JrnF5ECl1kg76pKkkiburRi0jBi2zQF8VjzJs1gy37esMuRUQkVJENeoAF9eUKehEpeNEO+oaZbD9wiIHhkbBLEREJTaSDvqtviLTD+f/tVyy77bc88nx72CWJiJxxkQ36R55v58HVO4HMk8nbO/u59eG1CnsRKTiRDfrbV25iMJU+qq1/eITbV24KqSIRkXBENuh3dY7/zNgTtYuIRFVkg76xKnla7SIiURXZoL9l+UKSifhRbclEnFuWLwypIhGRcBSFXUCuXL+oCYCv/3w9r/cNUzezhL+/9vzD7SIihSKyPXrIhP1Dn34zAF++9jyFvIgUpEgHPUBL7QwSceOVvbpCVkQKU+SDPhGPcU5dORt3d4ddiohIKCIf9AAXNFawYZeCXkQKU0EE/YWNlezrGaSjZzDsUkREzriCCPoLZlcAsEHDNyJSgAor6DV8IyIF6JRBb2b3mtk+M1s3pu1rZtZuZi8EP9eO+d2tZrbFzDaZ2fJcFX46KssSNFcn1aMXkYI0kR7994F3j9P+bXe/NPh5DMDMLgBuBC4MjvknM4uPc+wZd8HsCtbv6gq7DBGRM+6UQe/uTwEHJ/h61wEPuvugu28FtgBLp1Bf1lzQWMHW/YfoG0qFXYqIyBk1lTH6z5rZS8HQTnXQ1gTsHLNPW9B2HDNbYWatZtba0dExhTIm5sLGStzh5T09OX8vEZF8MtmgvxM4B7gU2A3cEbTbOPv6eC/g7ne5+xJ3X1JXVzfJMibugsbMF7Lr9YWsiBSYSQW9u+919xF3TwN3c2R4pg2YM2bXZmDX1ErMjsbKUqrLEqxr0zi9iBSWSQW9mc0es3kDMDoj51HgRjMrMbP5wAJg1dRKzA4z45I5VbzY1hl2KSIiZ9Qpb1NsZg8AbwNqzawN+CrwNjO7lMywzDbg0wDuvt7MHgI2ACngZncfyU3pp++S5iqeemUzhwZTzCiJ7B2aRUSOcsq0c/cPj9N8z0n2/wbwjakUlSuXzqki7bC2vYsrzp4VdjkiImdEQVwZO+ri5koAXtyp4RsRKRwFFfSzykuYW1OmcXoRKSgFFfRA5gvZnZp5IyKFo/CCvrmS9s5+9vUMhF2KiMgZUXBBv2huFYB69SJSMAou6C9srKQoZjy/4/WwSxEROSMKLuhLE3EubKygdZuCXkQKQ8EFPcBlLTW80NbJYCpvruUSEcmZwgz6+TUMpdK8pPveiEgBKMygb6kBYNXWid5mX0Rk+irIoK+ZUcy59eWs3qagF5HoK8igh0yvfs221xlJj3u7fBGRyCjYoF86v5qewRQv79GDSEQk2go26EfH6VdrnF5EIq5gg765uoymqiTPvHYg7FJERHKqYIMe4K3n1vKnVw9onF5EIq2gg37Zglp6BlKsbdd8ehGJrsIO+nMyT5l6enNHyJWIiOROQQf9rPISLphdwdNb9oddiohIzhR00AP8xYJantveSd9QKuxSRERyouCDftm5tQyNpHU7BBGJrIIP+staaiiOx3h6s4ZvRCSaCj7ok8VxLj+7hic37Qu7FBGRnCj4oAe46rx6Xu04xLb9h8IuRUQk6xT0wNXnNQDwxMvq1YtI9CjogbmzylhQX84TG/eGXYqISNYp6ANXn9/Aqq0H6R4YDrsUEZGsUtAH3nF+Pam089QrukpWRKJFQR9YNLea6rIET2zUOL2IRMspg97M7jWzfWa2bkxbjZk9bmabg2V10G5m9o9mtsXMXjKzxbksPpviMeOq8xr4zca9DKZGwi5HRCRrJtKj/z7w7mPavgQ84e4LgCeCbYBrgAXBzwrgzuyUeWa89+LZ9Ayk+KPufSMiEXLKoHf3p4Bj7w9wHXBfsH4fcP2Y9vs9489AlZnNzlaxubbs3FoqSov4xUu7wy5FRCRrJjtG3+DuuwGCZX3Q3gTsHLNfW9B2HDNbYWatZtba0ZEfX4AWF8VYfuFZPL5ewzciEh3Z/jLWxmkb9/FN7n6Xuy9x9yV1dXVZLmPyrr14Nj2DKf7wioZvRCQaiiZ53F4zm+3uu4OhmdGpKm3AnDH7NQO7plLgmbbsnFqSiRife+B5BoZHaKxKcsvyhVy/aNw/TERE8t5ke/SPAjcF6zcBPxvT/vFg9s0VQNfoEM908dja3QyNOP3DIzjQ3tnPrQ+v5ZHn28MuTURkUiYyvfIB4BlgoZm1mdkngduAd5rZZuCdwTbAY8BrwBbgbuDvclJ1Dt2+ctNxDwvvHx7h9pWbQqpIRGRqTjl04+4fPsGvrh5nXwdunmpRYdrV2X9a7SIi+U5Xxh6jsSp5Wu0iIvlOQX+MW5YvJJmIH9WWTMS5ZfnCkCoSEZmayc66iazR2TW3r9xEe2c/8ZjxP254o2bdiMi0pR79OK5f1MQfv3QVd3zoEkbSTkNladgliYhMmoL+JN5z8WyqyhL832e2h12KiMikKehPojQR598vmcOvN+xld5dm3YjI9KSgP4WPXjGPtDsPPLsj7FJERCZFQX8Kc2rKuGphPf+6aidDqXTY5YiInDYF/QR87M3z2N87yL+tnVa37RERART0E3LlgjoW1Jfzz79/jczFvyIi04eCfgJiMeNv//IcXt7Tw+825ce980VEJkpBP0Hvu7SRxspS7vz9q2GXIiJyWhT0E5SIx/jUX5zNqq0HWbP99bDLERGZMAX9abhx6RyqyhJ857ebwy5FRGTCFPSnoay4iBVXns2TmzpYs/3Y56WLiOQnBf1p+g9vaaG2vIT/9atNmoEjItOCgv40lRUX8dm3n8OzWw/y9BY9QFxE8p+CfhI+fPlcmqqSfHPlJtJp9epFJL8p6CehpCjOF96xgBfbuvjZi3pouIjkNwX9JH1gcTMXN1dy2y9f5tBgKuxyREROSEE/SbGY8dV/dyF7uwf5p99tCbscEZETUtBPwZvmVXPDoibufmor2w8cCrscEZFxKein6EvXnEdxUYy//+k6TbcUkbykoJ+ihopS/us15/H0lv38aE1b2OWIiBxHQZ8Ff710Lktbavjvv9jAvu6BsMsRETmKgj4LYjHjtg9cxEAqzVce0RCOiOQXBX2WnF1Xzn9+1xv49Ya9PLh6Z9jliIgcpqDPok+99Wzeem4tX//5erbs6wm7HBERQEGfVbGY8a2/uoSy4iI+98ALDAyPhF2SiMjUgt7MtpnZWjN7wcxag7YaM3vczDYHy+rslDo91FeU8s0PXczG3d18/efrwy5HRCQrPfq3u/ul7r4k2P4S8IS7LwCeCLYLylXnNfB3bzuHB1bt5IfPbg+7HBEpcLkYurkOuC9Yvw+4Pgfvkfe++K6FvH1hHV97dD2rt+khJSISnqkGvQO/NrM1ZrYiaGtw990AwbJ+iu8xLcVjxj/cuIjm6jI+84M17DjQF3ZJIlKgphr0y9x9MXANcLOZXTnRA81shZm1mllrR0fHFMvIT5XJBHd/fAmptPPxe59lf+9g2CWJSAGaUtC7+65guQ/4KbAU2GtmswGC5b4THHuXuy9x9yV1dXVTKSOvnVtfzj03Xcae7gH+4/dX65bGInLGTTrozWyGmc0cXQfeBawDHgVuCna7CfjZVIuc7t40r5rvfmQx63d18zf3t9I/pGmXInLmTKVH3wA8bWYvAquAf3P3XwG3Ae80s83AO4Ptgnf1+Q3c/sGLeea1A3zyvtUKexE5Y4ome6C7vwZcMk77AeDqqRQVVe9f3AzAF3/0Ip+8bzX33HQZyeJ4yFWJSNTpytgz7P2Lm/nWX13Cn187wEfveZbXDw2FXZKIRJyCPgQ3LGrmOx9ZzNr2Lj7wf/7EzoOaeikiuTPpoRuZmmsvmk1teQmfum8177/zT3z8ink8uHonuzr7aaxKcsvyhVy/qCnsMkUkAtSjD9HS+TX85DNvYTiV5o7HX6G9sx8H2jv7ufXhtTzyfHvYJYpIBCjoQ7agYSal43wh2z88wu0rN4VQkYhEjYI+D+ztGv/xg7s6+89wJSISRQr6PNBYlRy3vTKZ0GMJRWTKFPR54JblC0kmjh6+iRl09g/zN/e3srtLPXsRmTwFfR64flET//P9F9FUlcSApqokd3zwEr7ynvN5est+3nHH7/mXP25lJK3evYicPsuHoYElS5Z4a2tr2GXkpZ0H+/jKI+v4/SsdXNRUydfedwFvmlcTdlkikgfMbM2Yhz6dkHr0eW5OTRnf/8RlfOcji9jbPcAH7nyGz/xgDdv2Hwq7NBGZJnTB1DRgZrz34kauOq+eu5/ayj8/9Sq/2biXv758Hn/7l+dwVmVp2CWKSB7T0M00tK97gG//5hUeam0jbsaHljTzmbedQ3N1WdilicgZNNGhGwX9NLbjQB93/v5VfrxmJ+5w3aVNfGJZC29sqgy7NBE5AxT0BWRXZz93PfUa/2/1TvqHR1gyr5qb3tLCu994Fom4voYRiSoFfQHq6h/mR607uf+Z7ew42Ef9zBJuWNzEBxc3s6BhZtjliUiWKegL2Eja+d2mfTywagdPbupgJO1cMqeKDy5u4j0XN1IzozjsEkUkCxT0AkBHzyA/e6GdH69p4+U9PcQMLp8/i2suOovlF55FQ4Vm7IhMVwp6OYq7s2F3N79cu4dfrtvNqx2ZefhvmlfNVefVc+WCOi5srCAWs5ArFZGJUtDLSW3e28Ov1u1h5YY9rGvvBmDWjGL+YkEtV76hjmXn1qq3L5LnFPQyYR09g/xhcwdPvdLBU5v3czB4ju3cmjKWzq9haUsNl82voWVWGWbq8YvkCwW9TEo67azf1c2zWw+wautBWre/fjj4a8tLuHROFRc3V3JRcyUXNVVSW14ScsUihUtBL1nh7rza0cuqra/Tuu0gL7Z18tr+Q4z+Z9NYWcpFzZVc2FjJGxpmsvCsmcytKSOusX6RnJto0OteN3JSZsa59TM5t34mH7l8LgA9A8Os39XNuvYuXmrr4qW2Tlau33v4mJKiGOfUlbPwrJksaChnQf1M5teW0VxdRmni+McmikhuKejltM0sTXDF2bO44uxZh9sODabYsq+XTXt72Ly3h017e3nm1QP8dMwDzs2gsTLJvFllzJs1g5ZgObemjKaqJBXJIn0HIJIDCnrJihklRVwyp4pL5lQd1d7VP8yrHb3sONDHtgOH2B4sV67fc3js//BrFMeZXZWksSpJY2UpjVVJZleW0lSV5KzKUupmllBeopOByOlS0EtOVSYTLJ5bzeK51cf9rqt/mB0H+thxsI/dXf20d/azu3OAXV39bNjVzf7eweOOKU3EqC0voW5mCXXB8vB2sF5dlqC6rJiKZELfFYigoJcQVSYTmdk7zePfbXNgeIQ9XZng39M1wP7eQTp6BtnfO0RHzyDbD/QdNSvoWGaZ96guKw6WmfWqsmKqyxJUzSimKpmgvLSIitIiZpYmKC8pYmZpETOKi3TxmESGgl7yVmkiTkvtDFpqZ5x0v+GRNAcPZcK/o3eQzr4hXj80nFn2DfN63xBd/cN09A7yyt5eOvuGODQ0ctLXNIPy4iLKSzPBP/YkMHoiKCuOkzy8jFMW/CQTRUfWi+OUBfuUFMU07CShyFnQm9m7gf8NxIHvufttuXovKWyJeIyGitLTupJ3MDVCV98wXf3DdA+k6B1M0TMwTO9Aip6BzHrPYGa9dyBFz2DmhLHzYB/dAykODaboHz75yeJYMYNk4sjJoTQRo6QocwIoGbteFKM0Mdp+pK2kKB7sd2Tf0f2Ki2Ik4jGK4kZxfPz1RLCu4azwPPJ8O7ev3MSuzn4aq5Lcsnwh1y9qyvn75iTozSwOfBd4J9AGrDazR919Qy7eT+R0lRTFqa+IUz+F2zyk085AaoS+oRH6hzLLvqHUkfXhEfqHUkH7kX36hzNtQ6k0A8MjDKbSDAyn6eofZnA4zWAqzWAq0z44nGYgNUI2L3eJGYdDPzHmBDC6XhSPURw3io75fVHMKIob8VhmPWZGUcyIx4NlzIjb2O3YkfbYkX2KYkYsNv4+R+8XIxaDoljm5BQzgqVhY9ZjRrDMHGfB9lHrwe8txpH1Y14j139tPfJ8O7c+vPZwB6G9s59bH14LkPOwz1WPfimwxd1fAzCzB4HrAAW9REYsZsGwTG5HQN2dVNoPnxQyJ4Dj11PpNEMpZ3gksz6ccoZG0qRG0gyPOMNB2/BI+qj1Y48bSnnm+JHMPr2pFEOpNCNpP/yTOrxMM5KGkXT6cNvYfaYTO3zCOHLiiFnm3/PoiWP0hBAfc3IYbY+ZBfuCceQkE4tltl/e083wyNH/TPqHR7h95aZpG/RNwM4x223A5Tl6L5FIM7PDPevp9PgYdyftkEqnSacJTgo+7gnh2BPF4RPJiJP2zEnIHdLBa46kHXdnJNh2z+yfHt1n7Pp422Ne47j18Y457vUy6yPuh+vKvP+R7cxy9J+DHxfyo3Z19uf830Wugn68v4GO+pRmtgJYATB37twclSEiYcn0fCEeG70aurCvil52229pHyfUG6uSOX/vXD1QtA2YM2a7Gdg1dgd3v8vdl7j7krq6uhyVISKSH25ZvpDkMbcASSbi3LJ8Yc7fO1c9+tXAAjObD7QDNwIfydF7iYjkvdFx+MjMunH3lJl9FlhJ5u+1e919fS7eS0Rkurh+UdMZCfZj5Wy6gLs/BjyWq9cXEZGJydUYvYiI5AkFvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4syzef/TyRZh1gFsn+ThtcD+LJaTj6L+GfX5pjd9vvDMc/dT3kMmL4J+Ksys1d2XhF1HLkX9M+rzTW/6fPlPQzciIhGnoBcRibgoBP1dYRdwBkT9M+rzTW/6fHlu2o/Ri4jIyUWhRy8iIicxbYPezO41s31mti7sWnLBzOaY2ZNmttHM1pvZ58OuKZvMrNTMVpnZi8Hn+3rYNeWCmcXN7Hkz+0XYteSCmW0zs7Vm9oKZtYZdT7aZWZWZ/djMXg7+X3xz2DVNxrQdujGzK4Fe4H53f2PY9WSbmc0GZrv7c2Y2E1gDXO/uG0IuLSvMzIAZ7t5rZgngaeDz7v7nkEvLKjP7T8ASoMLd3xt2PdlmZtuAJe6er/PMp8TM7gP+4O7fM7NioMzdO8Ou63RN2x69uz8FHAy7jlxx993u/lyw3gNsBM78o2lyxDN6g81E8DM9ex0nYGbNwHuA74Vdi5w+M6sArgTuAXD3oekY8jCNg76QmFkLsAh4NtxKsisY1ngB2Ac87u6R+nzAPwD/BUiHXUgOOfBrM1tjZivCLibLzgY6gH8Jht++Z2Yzwi5qMhT0ec7MyoGfAF9w9+6w68kmdx9x90uBZmCpmUVmCM7M3gvsc/c1YdeSY8vcfTFwDXBzMKQaFUXAYuBOd18EHAK+FG5Jk6Ogz2PB2PVPgB+6+8Nh15MrwZ/DvwPeHXIp2bQMeF8whv0gcJWZ/SDckrLP3XcFy33AT4Gl4VaUVW1A25i/NH9MJvinHQV9ngq+rLwH2Oju3wq7nmwzszozqwrWk8A7gJfDrSp73P1Wd2929xbgRuC37v7RkMvKKjObEUwUIBjSeBcQmVlw7r4H2GlmC4Omq4FpORmiKOwCJsvMHgDeBtSaWRvwVXe/J9yqsmoZ8DFgbTCODfBld38sxJqyaTZwn5nFyXQ4HnL3SE5BjLAG4KeZPglFwL+6+6/CLSnrPgf8MJhx8xrwiZDrmZRpO71SREQmRkM3IiIRp6AXEYk4Bb2ISMQp6EVEIk5BLyIScQp6EZGIU9CLiEScgl5EJOL+P7IeihVMYkBBAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] @@ -302,11 +314,11 @@ { "data": { "text/plain": [ - "(array([168.62498827, 59.99018107, 0.98484788]),\n", - " array([168.62498827, 60. , 1. ]))" + "(array([168.62498827, 94.03830275, 36.64656385, 0.97773399]),\n", + " array([168.62498827, 94.04820896, 36.66409252, 1. ]))" ] }, - "execution_count": 22, + "execution_count": 106, "metadata": {}, "output_type": "execute_result" } @@ -338,12 +350,12 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 107, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGwdJREFUeJzt3XtwXOWZ5/Hvc/qmiy1ZtuWrjG0SA8GEQOIBzzK7m8XhmhSmapMsuzOJa5ZaqjbMJLM7u5OQVJHdXKbIbCpkUrlskeDgzKZCXIQMDBU2eLjshAwQzP1ijI1NsHyVb7JsS7KkfvaPfiW3pW6pbaQ+7T6/T5Wqz3nP26efli399J73nD7m7oiISPJEcRcgIiLxUACIiCSUAkBEJKEUACIiCaUAEBFJKAWAiEhCKQBERBJKASAiklAKABGRhErHXcB4Zs+e7UuWLIm7DBGRs8pzzz23393bJ+pX0wGwZMkSNm7cGHcZIiJnFTP7fSX9dAhIRCShFAAiIgmlABARSSgFgIhIQikAREQSSgEgIpJQCgARkYSqywDY3d3Ltx7ZzLauo3GXIiJSs+oyAPb3nOA7j23lra5jcZciIlKz6jIAGjKFt9U3MBRzJSIitatOAyAFQK8CQESkrLoMgMZsIQA0AhARKa8uA2B4BKAAEBEprz4DIF14W70n8jFXIiJSu+oyANKpiEzKNAcgIjKOugwAgFw6pUNAIiLjqDgAzCxlZi+Y2UNhfamZPWNmW8zs52aWDe25sL41bF9StI/bQvtmM7tmst9MsVw6UgCIiIzjdEYAnwM2Fa1/A7jT3ZcBh4CbQ/vNwCF3fy9wZ+iHmV0I3AQsB64Fvm9mqXdXfnkKABGR8VUUAGbWAXwU+FFYN+BK4L7QZR1wY1heHdYJ21eF/quBe9293923A1uByybjTZSSTUeaAxARGUelI4BvA38FDJ9WMws47O6DYb0TWBiWFwI7AML27tB/pL3EcyZdIQB0FpCISDkTBoCZfQzY5+7PFTeX6OoTbBvvOcWvd4uZbTSzjV1dXROVV1Y2pUNAIiLjqWQEcAVwg5m9DdxL4dDPt4EZZpYOfTqAXWG5E1gEELa3AgeL20s8Z4S73+XuK9x9RXt7+2m/oWGZdETvCQWAiEg5EwaAu9/m7h3uvoTCJO5j7v7HwOPAx0O3NcADYfnBsE7Y/pi7e2i/KZwltBRYBvxu0t7JKDnNAYiIjCs9cZeyPg/ca2ZfA14A7g7tdwN/Z2ZbKfzlfxOAu79mZuuB14FB4FZ3n7Lf0DoEJCIyvtMKAHd/AngiLG+jxFk87t4HfKLM878OfP10izwTOR0CEhEZV91eCZzVlcAiIuOq4wCI6BvUaaAiIuXUbQDk0hFDeWdgSCEgIlJKXQcA6K5gIiLl1G0AZNO6L7CIyHjqNgCGRwB9uimMiEhJdRsA2bRuDC8iMp46DgAdAhIRGU/dBoAmgUVExqcAEBFJqLoNgGyq8Nb6FQAiIiXVbQBoBCAiMr66DYDhSeDj+kA4EZGS6jYAGjLhNFAFgIhISXUfAMf6FQAiIqXUbQCkIiMdGccHBifuLCKSQHUbAFCYBziuEYCISEn1HQCpiGMnNAIQESmlrgMgk9IIQESknLoOgHTKNAIQESmj7gNA1wGIiJRW1wGQiSKO9WsEICJSSn0HQFoBICJSTn0HQMo4pkNAIiIl1XUAZFMRxzUJLCJSUl0HQCYV0TeQJ5/3uEsREak5dR8AoI+EFhEppc4DwAB0LYCISAl1HQDDdwXT1cAiImPVdQBkwk1hNAIQERmrvgMgpbuCiYiUU+cBEOYAdDGYiMgYdR4AGgGIiJRT1wEwPAmsEYCIyFh1HQAaAYiIlFfnAaDrAEREyqnrAEhFRmQ6BCQiUsqEAWBmDWb2OzN7ycxeM7P/GdqXmtkzZrbFzH5uZtnQngvrW8P2JUX7ui20bzaza6bqTRW9Hrl0iqN9CgARkdEqGQH0A1e6+weAS4BrzWwl8A3gTndfBhwCbg79bwYOuft7gTtDP8zsQuAmYDlwLfB9M0tN5pspJZeO6FEAiIiMMWEAeMHRsJoJXw5cCdwX2tcBN4bl1WGdsH2VmVlov9fd+919O7AVuGxS3sU4MumIHh0CEhEZo6I5ADNLmdmLwD5gA/AWcNjdh3+zdgILw/JCYAdA2N4NzCpuL/GcKZNNRfT0DUz1y4iInHUqCgB3H3L3S4AOCn+1v69Ut/BoZbaVaz+Fmd1iZhvNbGNXV1cl5Y0rm4440qsRgIjIaKd1FpC7HwaeAFYCM8wsHTZ1ALvCciewCCBsbwUOFreXeE7xa9zl7ivcfUV7e/vplFdSNhXR068RgIjIaJWcBdRuZjPCciPwEWAT8Djw8dBtDfBAWH4wrBO2P+buHtpvCmcJLQWWAb+brDdSTjYd6SwgEZES0hN3YT6wLpyxEwHr3f0hM3sduNfMvga8ANwd+t8N/J2ZbaXwl/9NAO7+mpmtB14HBoFb3X3KL9EdPgvI3SnMRYuICFQQAO7+MnBpifZtlDiLx937gE+U2dfXga+ffplnLpuOGMw7/YN5GjJTftapiMhZo66vBIZCAAC6FkBEZJS6D4DcSABoIlhEpFjdB8AfHnuMJ7OfZen3OuDOi+Dl9XGXJCJSEyqZBD5rnb/vYT6y95tko/5CQ/cO+IfPFpYv/mR8hYmI1IC6HgH80TvfJ+v9pzYO9MKjX4mnIBGRGlLXATC9f2/pDd2d1S1ERKQG1XUA9OTmlt7Q2lHdQkREalBdB8CT53yGgajh1MZMI6y6PZ6CRERqSF1PAm+ecx0AH3jzOyywA1hrR+GXvyaARUTqOwCgEAKf33IB1180n298/OK4yxERqRl1fQhomD4RVERkrEQEQC4d0d2rABARKZaMAMikOHRMASAiUiwRAdCQjjh8/ETcZYiI1JREBEAuk9IhIBGRURIRAA2ZiGMnhhgYysddiohIzUhGAKQLN4LRKEBE5KRkBEC4E9jh4woAEZFhCQmAwtvs7tVEsIjIsEQEQE4jABGRMRIRAA3htpAKABGRk5IRAMMjAE0Ci4iMSEQA5NIRBnTrYjARkRGJCAAzoyGT0ghARKRIIgIAoDGT0hyAiEiRxARALhNpBCAiUiQxAZBNRRw+pjkAEZFhiQmAXCbioCaBRURGJCYAmrJpDmkEICIyIjEB0JhNcezEEH0DQ3GXIiJSExITAE3hYrADGgWIiABJCoBsCICj/TFXIiJSGxITAI1ZjQBERIolJwCGDwEdVQCIiECCAqApmwZ0CEhEZFhiAiCTMtKR6RCQiEiQmAAwM5pzafZrBCAiAlQQAGa2yMweN7NNZvaamX0utM80sw1mtiU8toV2M7PvmNlWM3vZzD5YtK81of8WM1szdW+rtMZMSnMAIiJBJSOAQeAv3f19wErgVjO7EPgC8Ki7LwMeDesA1wHLwtctwA+gEBjAl4HLgcuALw+HRrU0ZCKNAEREggkDwN13u/vzYbkH2AQsBFYD60K3dcCNYXk18BMveBqYYWbzgWuADe5+0N0PARuAayf13UygMZtSAIiIBKc1B2BmS4BLgWeAue6+GwohAcwJ3RYCO4qe1hnayrWPfo1bzGyjmW3s6uo6nfIm1JRJc/DYCdx9UvcrInI2qjgAzGwa8AvgL9z9yHhdS7T5OO2nNrjf5e4r3H1Fe3t7peVVpDGbYmDI6ekfnNT9ioicjSoKADPLUPjl/1N3vz807w2HdgiP+0J7J7Co6OkdwK5x2qumOVe4GGzfkb5qvqyISE2q5CwgA+4GNrn7t4o2PQgMn8mzBnigqP3T4WyglUB3OET0a+BqM2sLk79Xh7aqmZYrXAy294jmAURE0hX0uQL4FPCKmb0Y2r4I3AGsN7ObgXeAT4RtvwKuB7YCx4E/BXD3g2b2VeDZ0O8r7n5wUt5FhZpDAOzp1ghARGTCAHD3Jyl9/B5gVYn+DtxaZl9rgbWnU+BkGhkB9CgAREQScyUwQCYV0ZCJ2KsRgIhIsgIACqMAzQGIiCQwAJqyafboLCARkeQFQHMupQAQESGJAZBN09XTTz6vq4FFJNkSFwDTcmmG8q77AohI4iUvABqGLwbTYSARSbbEBUBzVheDiYhAAgNgehgB7DzcG3MlIiLxSlwANGVTpCOj89DxuEsREYlV4gLAzGhtzNB5SCMAEUm2xAUAFCaCd2gEICIJl8gAaGnI0HlQIwARSbaEBkCaw70DHNWdwUQkwZIZAI0ZAHZqHkBEEiyZAdBQCACdCSQiSZbIABi+FkBnAolIkiUyAJqyKTIpY8dBjQBEJLkSGQCFawGyvH1AASAiyZXIAABobUzzVtfRuMsQEYlNYgOgrSnLOwePMziUj7sUEZFYJDoAhvLODk0Ei0hCJTYAZjQVTgXdvl+HgUQkmRIbAG3NWQC2dR2LuRIRkXgkNgAaMykaMym27VcAiEgyJTYAoHAYaLtGACKSUIkPgC37euIuQ0QkFokOgNnTcuw/eoIDR/vjLkVEpOoSHwAAm/doFCAiyZPwACicCbRJASAiCZToAGjKppmWS/PG7iNxlyIiUnWJDgCAmc1ZNikARCSBEh8As6dl2bLvqD4TSEQSRwEwLUf/YJ63D+h6ABFJlsQHQPv0wplAL3d2x1yJiEh1JT4AZjZnyaYjXtxxOO5SRESqKvEBEJkxd3qOF95RAIhIskwYAGa21sz2mdmrRW0zzWyDmW0Jj22h3czsO2a21cxeNrMPFj1nTei/xczWTM3bOTNzWhrYtPsIfQNDcZciIlI1lYwA7gGuHdX2BeBRd18GPBrWAa4DloWvW4AfQCEwgC8DlwOXAV8eDo1aMK+lgcG889ounQ4qIskxYQC4+z8BB0c1rwbWheV1wI1F7T/xgqeBGWY2H7gG2ODuB939ELCBsaESm3mtDQC8pHkAEUmQM50DmOvuuwHC45zQvhDYUdSvM7SVax/DzG4xs41mtrGrq+sMyzs903JpWhrSPPf7Q1V5PRGRWjDZk8BWos3HaR/b6H6Xu69w9xXt7e2TWtx4Fsxo5J/f2o97ybJEROrOmQbA3nBoh/C4L7R3AouK+nUAu8ZprxkdbY0cOj7Am3t1j2ARSYYzDYAHgeEzedYADxS1fzqcDbQS6A6HiH4NXG1mbWHy9+rQVjMWtTUB8NRb+2OuRESkOio5DfRnwFPA+WbWaWY3A3cAV5nZFuCqsA7wK2AbsBX4IfAZAHc/CHwVeDZ8fSW01YyWxgytjRme2nYg7lJERKoiPVEHd//3ZTatKtHXgVvL7GctsPa0qquyhTMaeeqtAwzlnVRUatpCRKR+JP5K4GKLZzVxpG+Q59/R2UAiUv8UAEUWz2oiMvjHTXvjLkVEZMopAIrk0ik62pr4x9cVACJS/xQAoyyd3cxbXcd4e7/uDyAi9U0BMMrS2c0A/Pq1PTFXIiIytRQAo7Q2ZpjX0sDfv7gz7lJERKaUAqCE8+dNZ9PuHrbs7Ym7FBGRKaMAKGHZnGlEhkYBIlLXFAAlNOfSLJrZxP3P72Qorw+HE5H6pAAoY/n8FnZ39/H4G/sm7iwichZSAJRxbvs0pjekWffU23GXIiIyJRQAZaQi46IFrfxmy362dekjokWk/igAxrF8QQupyPjhb7bFXYqIyKRTAIyjOZdm+fwW1m/spPPQ8bjLERGZVAqACaxY0gbA9594K+ZKREQmlwJgAtMbMlw4v4X1z+5guz4fSETqiAKgApcvnUkqMr760OtxlyIiMmkUABVozqX5gyUzeeyNfTyxWdcFiEh9UABU6JJFM5jZnOVLv3yVnr6BuMsREXnXFAAVSkXGqgvmsKu7l689tCnuckRE3jUFwGlYMKORD53Txs837uDhV3bHXY6IyLuiADhNK8+dxfzWBv7r+pfYvEcfFy0iZy8FwGlKRcb1F80nFRn/6Scb2X+0P+6SRETOiALgDExrSHP9++exu7uXT939DN29mhQWkbOPAuAMzW9t5KPvn8+be4/y6bXPcPj4ibhLEhE5LQqAd2HxrGauu2ger+48wr/9wT+z83Bv3CWJiFRMAfAuvad9GjdesoCdh3pZ/d0neXrbgbhLEhGpiAJgEnS0NfHxD3WQd/gPP3ya7z62hcGhfNxliYiMSwEwSWZNy/HvVixi2ZxpfPORN7nxe7/l1Z3dcZclIlKWAmASZdMR1yyfx/UXzWP7gWPc8N0n+eIvX2FPd1/cpYmIjJGOu4B6Y2YsmzudRTObeHrbAX7+7A5+8Vwnf7JyMX96xRI62priLlFEBFAATJmGTIoPnz+HS89p45ltB/jxb7fz499u55rl8/jUysWsPHcWUWRxlykiCaYAmGKtjRmuXj6Ple+Zxcud3TyxuYuHX93D3JYcN3xgAR+7eAHvX9iqMBCRqlMAVElLQ4Y/eu9sLl86k+37j7F5Tw9rf/s2P/zNdmY2Z/nw+e18+Pw5rFw6kzktDXGXKyIJoACoskwq4ry50zlv7nR6B4b4/f5jvH3gOA+/sof7n98JwMIZjaxY0saHFrexfEEL582dzvSGzMQ7f3k9PPoV6O6E1g5YdTtc/MkpfkcicrZSAMSoMZPigvktXDC/hbw7+470s6u7l93dfTy6aR8PvLhrpO+CGQ1cOL+FZXOns3hmE+fMauKcmU3Mb20kFVnhl/8/fBYGwtXI3TsK66AQEJGSqh4AZnYt8LdACviRu99R7RpqUWTGvNYG5rUWDv+4Oz19g+w/2s/+Yyc40NPPizsO89gb+8j7yeelI2NhWyP39X2J9qFRH0Ux0MvAI/+Do++5kZbGTCEoRESCqgaAmaWA7wFXAZ3As2b2oLvrbuujmBktjRlaGjOc236yPZ93evoH6e4dGPk60jvArKGukvtJ9ezi0q9uAGB6Q5oZjRnamrPMaMoyozFDa2OG5lya5myKplyaabkUTdk0zeFxWi5NUzZFcy5NLh2RS6fIpSNNWotMppgO31Z7BHAZsNXdtwGY2b3AakABUKEoMlrDL+5iRzfOpaV/z5j+hzLt/OvF7fQNDBW+BvN09w7Q1dNP38AQ/YN5TgzmGSweVlQgHRm5dER2OBQyEQ3pFNl0REPmZFBkUhHplJGOjHQqIpMyUpGRjqKRtsKjkUlFYZuN2lZ4npmRMiOywvchKrGcstAvKqwXLxf6GFF0sl9khO0W9lPoZwbG8GNhP4XHQjtG2W0WsnF4fbiOkT6m8JQiMR6+rXYALAR2FK13ApdXuYa69OQ5n+Gqt/6aTP7kVccDUQPPLP0zLpkzY8LnD+WdgaE8A0OFQBgYKlofyjMw6Azm8wzlncG8n/JYWM4zmM/T1zfEoeMn2/Pu5L1wSGso77jDkIf2fFjOO6cXP/VhTKBQPkSGw4YSwXSy38mgGXmNUVlzyuqojeWed+oeR287uVBRvxINxc8br97i4By9v0rrHa+93P7H1lRZvZW8qAF3H/oic/JjD9/y6FfqLgBKfX9O+dk3s1uAWwDOOeecM36hFUtm8qHFbWf8/LPPMnhlPl40jEyvup3r3v8Jrou7tArkQ6AUgsQZHHIGh8JyCCd3yBcFyfDySMCEsMmPBE8hZPJF23z0c8LySP+i57uDM/wI7sAp6z7SPrzOyPrJfvl86fbidcrsb+R1x9nmFBqLayhUeqqiTSW2lY7g0c3FUX3K/sb0K79vL7My+s+AcvVWWtPY1zr97814tY/5lpX5HpZ7zWHtB/aX3tDdOe7+JkO1A6ATWFS03gHsKu7g7ncBdwGsWLHiXf1hmLih9sWfPGvP+EmljFQKcqTiLkWkuu7sKBz2Ga21Y8pfutofBvcssMzMlppZFrgJeLDKNYiI1I5Vt0Om8dS2TGOhfYpVdQTg7oNm9mfArymcBrrW3V+rZg0iIjVleNSegLOAcPdfAb+q9uuKiNSsmA7f6n4AIiIJpQAQEUkoBYCISEIpAEREEkoBICKSUAoAEZGEUgCIiCSUAkBEJKGs3IdA1QIz6wJ+H3cdwWygzKc21QTVd+ZquTao7fpquTao7fqmsrbF7t4+UaeaDoBaYmYb3X1F3HWUo/rOXC3XBrVdXy3XBrVdXy3UpkNAIiIJpQAQEUkoBUDl7oq7gAmovjNXy7VBbddXy7VBbdcXe22aAxARSSiNAEREEkoBMAEzW2Rmj5vZJjN7zcw+F3dNo5lZysxeMLOH4q5lNDObYWb3mdkb4Xv4h3HXVMzM/kv4d33VzH5mZg0x1rLWzPaZ2atFbTPNbIOZbQmPsd3oukx9/yv8275sZr80sxm1VF/Rtv9mZm5ms2upNjP7czPbHP4P/k2161IATGwQ+Et3fx+wErjVzC6MuabRPgdsiruIMv4W+L/ufgHwAWqoTjNbCHwWWOHuF1G4S91NMZZ0D3DtqLYvAI+6+zLg0bAel3sYW98G4CJ3vxh4E7it2kUVuYex9WFmi4CrgHeqXVCRexhVm5n9G2A1cLG7Lwe+We2iFAATcPfd7v58WO6h8AtsYbxVnWRmHcBHgR/FXctoZtYC/CvgbgB3P+Huh+Otaow00GhmaaAJ2BVXIe7+T8DBUc2rgXVheR1wY1WLKlKqPnd/xN0Hw+rTwNTfybyMMt8/gDuBvwJim/AsU9t/Bu5w9/7QZ1+161IAnAYzWwJcCjwTbyWn+DaF/9z5uAsp4VygC/hxOET1IzNrjruoYe6+k8JfXe8Au4Fud38k3qrGmOvuu6HwxwgwJ+Z6xvMfgYfjLqKYmd0A7HT3l+KupYTzgH9pZs+Y2f8zsz+odgEKgAqZ2TTgF8BfuPuRuOsBMLOPAfvc/bm4aykjDXwQ+IG7XwocI95DGKcIx9NXA0uBBUCzmf1JvFWdnczsSxQOl/407lqGmVkT8CXg9rhrKSMNtFE4tPzfgfVmZtUsQAFQATPLUPjl/1N3vz/ueopcAdxgZm8D9wJXmtn/ibekU3QCne4+PGK6j0Ig1IqPANvdvcvdB4D7gX8Rc02j7TWz+QDhseqHCSZiZmuAjwF/7LV1Xvl7KIT7S+FnpAN43szmxVrVSZ3A/V7wOwqj+KpOUisAJhAS+W5gk7t/K+56irn7be7e4e5LKExePubuNfMXrLvvAXaY2fmhaRXweowljfYOsNLMmsK/8ypqaJI6eBBYE5bXAA/EWMsYZnYt8HngBnc/Hnc9xdz9FXef4+5Lws9IJ/DB8P+yFvw9cCWAmZ0HZKnyB9cpACZ2BfApCn9dvxi+ro+7qLPInwM/NbOXgUuAv465nhFhZHIf8DzwCoWfh9iuzjSznwFPAeebWaeZ3QzcAVxlZlsonMlyR43V911gOrAh/Gz87xqrryaUqW0tcG44NfReYE21R1C6ElhEJKE0AhARSSgFgIhIQikAREQSSgEgIpJQCgARkYRSAIiIJJQCQEQkoRQAIiIJ9f8BYJviX8a/T5YAAAAASUVORK5CYII=\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG1xJREFUeJzt3X1wHPWd5/H3t+dJD7bkJ/lRfiIxEEwIJF7wHrm7HA6PSWGqNslxt0tce9RRdWGX7N3ebUJSBXck2SN3uZBNEXJFYgdnNxXiImQhVLjgNbAbskAwT+bBGBubYPlRfpJlW5Ilzff+6JY8lmakkZGmx9OfV5Vqun/9m57vyJY++vWve9rcHRERSZ4g7gJERCQeCgARkYRSAIiIJJQCQEQkoRQAIiIJpQAQEUkoBYCISEIpAEREEkoBICKSUOm4CxjJjBkzfNGiRXGXISJyVnnxxRcPuHvLaP2qOgAWLVrExo0b4y5DROSsYma/L6efDgGJiCSUAkBEJKEUACIiCaUAEBFJKAWAiEhCKQBERBJKASAiklA1GQB7Orr49hNb2N5+LO5SRESqVk0GwIHOk3z3yW2803487lJERKpWTQZAXSZ8W929/TFXIiJSvWo0AFIAdCkARERKqskAqM+GAaARgIhIaTUZAAMjAAWAiEhptRkA6fBtdZ3Mx1yJiEj1qskASKcCMinTHICIyAhqMgAAcumUDgGJiIyg7AAws5SZvWxmj0Xri83seTPbamY/M7Ns1J6L1rdF2xcV7OP2qH2LmV093m+mUC4dKABEREYwlhHAF4HNBevfBO5x9yXAYeDmqP1m4LC7fxC4J+qHmV0A3AgsBa4B7jOz1PsrvzQFgIjIyMoKADNrBT4F/DBaN+AK4KGoy1rghmh5ZbROtH1F1H8l8KC797j7DmAbcOl4vIlisulAcwAiIiModwTwHeCvgIHTaqYDR9y9L1pvA+ZFy/OAnQDR9o6o/2B7keeMuzAAdBaQiEgpowaAmX0a2O/uLxY2F+nqo2wb6TmFr3eLmW00s43t7e2jlVdSNqVDQCIiIylnBHA5cL2ZvQs8SHjo5zvAFDNLR31agd3RchswHyDa3gwcKmwv8pxB7n6/uy9z92UtLS1jfkMDMumArpMKABGRUkYNAHe/3d1b3X0R4STuk+7+x8BTwGeibquAR6LlR6N1ou1PurtH7TdGZwktBpYAvxu3dzJETnMAIiIjSo/epaQvAQ+a2deBl4HVUftq4G/NbBvhX/43Arj7G2a2DngT6ANudfcJ+w2tQ0AiIiMbUwC4+9PA09HydoqcxePu3cBnSzz/G8A3xlrkmcjpEJCIyIhq9krgrK4EFhEZUQ0HQEB3n04DFREppWYDIJcO6M87vf0KARGRYmo6AEB3BRMRKaVmAyCb1n2BRURGUrMBMDAC6NZNYUREiqrZAMimdWN4EZGR1HAA6BCQiMhIajYANAksIjIyBYCISELVbABkU+Fb61EAiIgUVbMBoBGAiMjIajYABiaBT+gD4UREiqrZAKjLRKeBKgBERIqq+QA43qMAEBEppmYDIBUY6cA40ds3emcRkQSq2QCAcB7ghEYAIiJF1XYApAKOn9QIQESkmJoOgExKIwARkVJqOgDSKdMIQESkhJoPAF0HICJSXE0HQCYION6jEYCISDG1HQBpBYCISCm1HQAp47gOAYmIFFXTAZBNBZzQJLCISFE1HQCZVEB3b5583uMuRUSk6tR8AIA+ElpEpJgaDwAD0LUAIiJF1HQADNwVTFcDi4gMV9MBkIluCqMRgIjIcLUdACndFUxEpJQaD4BoDkAXg4mIDFPjAaARgIhIKTUdAAOTwBoBiIgMV9MBoBGAiEhpNR4Aug5ARKSUmg6AVGAEpkNAIiLFjBoAZlZnZr8zs1fN7A0z+x9R+2Ize97MtprZz8wsG7XnovVt0fZFBfu6PWrfYmZXT9SbKng9cukUx7oVACIiQ5UzAugBrnD3jwAXA9eY2XLgm8A97r4EOAzcHPW/GTjs7h8E7on6YWYXADcCS4FrgPvMLDWeb6aYXDqgUwEgIjLMqAHgoWPRaib6cuAK4KGofS1wQ7S8Mlon2r7CzCxqf9Dde9x9B7ANuHRc3sUIMumATh0CEhEZpqw5ADNLmdkrwH5gPfAOcMTdB36ztgHzouV5wE6AaHsHML2wvchzJkw2FdDZ3TvRLyMictYpKwDcvd/dLwZaCf9q/1CxbtGjldhWqv00ZnaLmW00s43t7e3llDeibDrgaJdGACIiQ43pLCB3PwI8DSwHpphZOtrUCuyOltuA+QDR9mbgUGF7kecUvsb97r7M3Ze1tLSMpbyisqmAzh6NAEREhirnLKAWM5sSLdcDnwQ2A08Bn4m6rQIeiZYfjdaJtj/p7h613xidJbQYWAL8brzeSCnZdKCzgEREikiP3oU5wNrojJ0AWOfuj5nZm8CDZvZ14GVgddR/NfC3ZraN8C//GwHc/Q0zWwe8CfQBt7r7hF+iO3AWkLsTzkWLiAiUEQDuvgm4pEj7doqcxePu3cBnS+zrG8A3xl7mmcumA/ryTk9fnrrMhJ91KiJy1qjpK4EhDABA1wKIiAxR8wGQGwwATQSLiBSq+QD4w+NP8kz2NhZ/rxXuuRA2rYu7JBGRqlDOJPBZ67z9j/PJfd8iG/SEDR074Ze3hcsXfS6+wkREqkBNjwA+/t59ZL3n9MbeLthwVzwFiYhUkZoOgMk9+4pv6GirbCEiIlWopgOgMzer+Ibm1soWIiJShWo6AJ5Z8AV6g7rTGzP1sOKOeAoSEakiNT0JvGXmtQB85O3vMtcOYs2t4S9/TQCLiNR2AEAYAl/aej7XXTiHb37morjLERGpGjV9CGiAPhFURGS4RARALh3Q0aUAEBEplIwAyKQ4fFwBICJSKBEBUJcOOHLiZNxliIhUlUQEQC6T0iEgEZEhEhEAdZmA4yf76e3Px12KiEjVSEYApMMbwWgUICJySjICILoT2JETCgARkQEJCYDwbXZ0aSJYRGRAIgIgpxGAiMgwiQiAuui2kAoAEZFTkhEAAyMATQKLiAxKRADk0gEGdOhiMBGRQYkIADOjLpPSCEBEpEAiAgCgPpPSHICISIHEBEAuE2gEICJSIDEBkE0FHDmuOQARkQGJCYBcJuCQJoFFRAYlJgAasmkOawQgIjIoMQFQn01x/GQ/3b39cZciIlIVEhMADdHFYAc1ChARAZIUANkoAI71xFyJiEh1SEwA1Gc1AhARKZScABg4BHRMASAiAgkKgIZsGtAhIBGRAYkJgEzKSAemQ0AiIpHEBICZ0ZhLc0AjABERoIwAMLP5ZvaUmW02szfM7ItR+zQzW29mW6PHqVG7mdl3zWybmW0ys48W7GtV1H+rma2auLdVXH0mpTkAEZFIOSOAPuAv3f1DwHLgVjO7APgysMHdlwAbonWAa4El0dctwPchDAzgTuAy4FLgzoHQqJS6TKARgIhIZNQAcPc97v5StNwJbAbmASuBtVG3tcAN0fJK4Mceeg6YYmZzgKuB9e5+yN0PA+uBa8b13YyiPptSAIiIRMY0B2Bmi4BLgOeBWe6+B8KQAGZG3eYBOwue1ha1lWof+hq3mNlGM9vY3t4+lvJG1ZBJc+j4Sdx9XPcrInI2KjsAzGwS8HPgL9z96Ehdi7T5CO2nN7jf7+7L3H1ZS0tLueWVpT6borff6ezpG9f9ioicjcoKADPLEP7y/4m7Pxw174sO7RA97o/a24D5BU9vBXaP0F4xjbnwYrD9R7sr+bIiIlWpnLOADFgNbHb3bxdsehQYOJNnFfBIQfvno7OBlgMd0SGiXwNXmdnUaPL3qqitYiblwovB9h3VPICISLqMPpcDNwGvmdkrUdtXgLuBdWZ2M/Ae8Nlo26+A64BtwAngTwHc/ZCZfQ14Iep3l7sfGpd3UabGKAD2dmgEICIyagC4+zMUP34PsKJIfwduLbGvNcCasRQ4ngZHAJ0KABGRxFwJDJBJBdRlAvZpBCAikqwAgHAUoDkAEZEEBkBDNs1enQUkIpK8AGjMpRQAIiIkMQCyado7e8jndTWwiCRb4gJgUi5Nf951XwARSbzkBUDdwMVgOgwkIsmWuABozOpiMBERSGAATI5GALuOdMVciYhIvBIXAA3ZFOnAaDt8Iu5SRERilbgAMDOa6zO0HdYIQESSLXEBAOFE8E6NAEQk4RIZAE11GdoOaQQgIsmW0ABIc6Srl2O6M5iIJFgiA2BF7z/yTPY2Gv/nDLjnQti0Lu6SREQqrpwbwtSU8/Y/zifb/w/ZIPpE0I6d8MvbwuWLPhdfYSIiFZa4EcDH37uPrA/5OOjeLthwVzwFiYjEJHEBMLlnX/ENHW2VLUREJGaJC4DO3KziG5pbK1uIiEjMEhcAzyz4Ar1B3emNmXpYcUc8BYmIxCRxk8BbZl4LwLJ37mVGfzs2ZR624k5NAItI4iQuACAMgV/0Xc76zft46qZPsHhGY9wliYhUXOIOAQ2Y0pABYMeBYzFXIiISj8QGwNTGLADb24/HXImISDwSGwD1mRT1mRTbDygARCSZEhsAEB4G2qERgIgkVOIDYOv+zrjLEBGJRaIDYMakHAeOneTgsZ7RO4uI1JjEBwDAlr0aBYhI8iQ8AMIzgTYrAEQkgRIdAA3ZNJNyad7aczTuUkREKi7RAQAwrTHLZgWAiCRQ4gNgxqQsW/cfo68/H3cpIiIVpQCYlKOnL8+7B3U9gIgkS+IDoGVyeCbQpraOmCsREamsxAfAtMYs2XTAKzuPxF2KiEhFJT4AAjNmTc7x8nsKABFJllEDwMzWmNl+M3u9oG2ama03s63R49So3czsu2a2zcw2mdlHC56zKuq/1cxWTczbOTMzm+rYvOco3b39cZciIlIx5YwAHgCuGdL2ZWCDuy8BNkTrANcCS6KvW4DvQxgYwJ3AZcClwJ0DoVENZjfV0Zd33tit00FFJDlGDQB3/yfg0JDmlcDaaHktcENB+4899BwwxczmAFcD6939kLsfBtYzPFRiM7s5vEfwq5oHEJEEOdM5gFnuvgcgepwZtc8Ddhb0a4vaSrUPY2a3mNlGM9vY3t5+huWNzaRcmqa6NC/+/nBFXk9EpBqM9ySwFWnzEdqHN7rf7+7L3H1ZS0vLuBY3krlT6vnndw7gXrQsEZGac6YBsC86tEP0uD9qbwPmF/RrBXaP0F41WqfWc/hEL2/v0z2CRSQZzjQAHgUGzuRZBTxS0P756Gyg5UBHdIjo18BVZjY1mvy9KmqrGvOnNgDw7DsHYq5ERKQyyjkN9KfAs8B5ZtZmZjcDdwNXmtlW4MpoHeBXwHZgG/AD4AsA7n4I+BrwQvR1V9RWNZrqMzTXZ3h2+8G4SxERqYj0aB3c/d+V2LSiSF8Hbi2xnzXAmjFVV2HzptTz7DsH6c87qaDYtIWISO1I/JXAhRZOb+Bodx8vvaezgUSk9ikACiyc3kBg8A+b98VdiojIhFMAFMilU7RObeAf3lQAiEjtUwAMsXhGI++0H+fdA7o/gIjUNgXAEItnNALw6zf2xlyJiMjEUgAM0VyfYXZTHX//yq64SxERmVAKgCLOmz2ZzXs62bqvM+5SREQmjAKgiCUzJxEYGgWISE1TABTRmEszf1oDD7+0i/68PhxORGqTAqCEpXOa2NPRzVNv7R+9s4jIWUgBUMI5LZOYXJdm7bPvxl2KiMiEUACUkAqMC+c285utB9jero+IFpHaowAYwdK5TaQC4we/2R53KSIi404BMILGXJqlc5pYt7GNtsMn4i5HRGRcKQBGsWzRVADue/qdmCsRERlfCoBRTK7LcMGcJta9sJMd+nwgEakhCoAyXLZ4GqnA+Npjb8ZdiojIuFEAlKExl+YPFk3jybf28/QWXRcgIrVBAVCmi+dPYVpjlq/+4nU6u3vjLkdE5H1TAJQpFRgrzp/J7o4uvv7Y5rjLERF53xQAYzB3Sj0fWzCVn23cyeOv7Ym7HBGR90UBMEbLz5nOnOY6/su6V9myVx8XLSJnLwXAGKUC47oL55AKjP/4440cONYTd0kiImdEAXAGJtWlue7Ds9nT0cVNq5+no0uTwiJy9lEAnKE5zfV86sNzeHvfMT6/5nmOnDgZd0kiImOiAHgfFk5v5NoLZ/P6rqP80ff/mV1HuuIuSUSkbAqA9+kDLZO44eK57Drcxcp7n+G57QfjLklEpCwKgHHQOrWBz3yslbzDv//Bc9z75Fb6+vNxlyUiMiIFwDiZPinHv102nyUzJ/GtJ97mhu/9ltd3dcRdlohISQqAcZRNB1y9dDbXXTibHQePc/29z/CVX7zG3o7uuEsTERkmHXcBtcbMWDJrMvOnNfDc9oP87IWd/PzFNv5k+UL+9PJFtE5tiLtEERFAATBh6jIpPnHeTC5ZMJXntx/kR7/dwY9+u4Orl87mpuULWX7OdILA4i5TRBJMATDBmuszXLV0Nss/MJ1NbR08vaWdx1/fy6ymHNd/ZC6fvmguH57XrDAQkYpTAFRIU12Gj39wBpctnsaOA8fZsreTNb99lx/8ZgfTGrN84rwWPnHeTJYvnsbMprq4yxWRBFAAVFgmFXDurMmcO2syXb39/P7Acd49eILHX9vLwy/tAmDelHqWLZrKxxZOZencJs6dNZnJdZnRd75pHWy4CzraoLkVVtwBF31ugt+RiJytFAAxqs+kOH9OE+fPaSLvzv6jPezu6GJPRzcbNu/nkVd2D/adO6WOC+Y0sWTWZBZOa2DB9AYWTGtgTnM9qcDCX/6/vA16o6uRO3aG66AQEJGiKh4AZnYN8DdACvihu99d6RqqUWDG7OY6ZjeHh3/cnc7uPg4c6+HA8ZMc7OzhlZ1HePKt/eT91PPSgTFvaj0PdX+Vlv4hH0XR20XvE/+dYx+4gab6TBgUIiKRigaAmaWA7wFXAm3AC2b2qLvrbutDmBlN9Rma6jOc03KqPZ93Onv66OjqHfw62tXL9P72ovtJde7mkq+tB2ByXZop9RmmNmaZ0pBlSn2G5voMjbk0jdkUDbk0k3IpGrJpGqPHSbk0DdkUjbk0uXRALp0ilw40aS0ynmI6fFvpEcClwDZ33w5gZg8CKwEFQJmCwGiOfnEXOrZxFk09e4f1P5xp4V8vbKG7tz/86svT0dVLe2cP3b399PTlOdmXp69wWFGGdGDk0gHZgVDIBNSlU2TTAXWZU0GRSQWkU0Y6MNKpgEzKSAVGOggG28JHI5MKom02ZFv4PDMjZUZg4fchKLKcsqhfEK4XLod9jCA41S8wou0W7SfsZwbGwGO4n/AxbMcouc2ibBxYH6hjsI8pPKVAjIdvKx0A84CdBettwGUVrqEmPbPgC1z5zl+TyZ+66rg3qOP5xX/GxTOnjPr8/rzT25+ntz8MhN7+gvX+PL19Tl8+T3/e6cv7aY/hcp6+fJ7u7n4OnzjVnncn7+Ehrf684w79HrXno+W8M7b4qQ3DAoXSITIQNhQJplP9TgXN4GsMyZrTVodsLPW80/c4dNuphbL6FWkofN5I9RYG59D9lVvvSO2l9j+8pvLqLedFDVh9+CvMzA8/fMuGu2ouAIp9f0772TezW4BbABYsWHDGL7Rs0TQ+tnDqGT//7LMEXpuDFwwj0yvu4NoPf5Zr4y6tDPkoUMIgcfr6nb7+aDkKJ3fIFwTJwPJgwERhkx8MnjBk8gXbfOhzouXB/gXPdwdn4BHcgdPWfbB9YJ3B9VP98vni7YXrlNjf4OuOsM0JGwtrCCs9XcGmItuKR/DQ5sKoPm1/w/qV3reXWBn6Z0Cpesutafhrjf17M1Ltw75lJb6HpV5zQMvBA8U3dLSNuL/xUOkAaAPmF6y3ArsLO7j7/cD9AMuWLXtffxgmbqh90efO2jN+UikjlYIcqbhLEamse1rDwz5DNbdO+EtX+sPgXgCWmNliM8sCNwKPVrgGEZHqseIOyNSf3papD9snWEVHAO7eZ2Z/Bvya8DTQNe7+RiVrEBGpKgOj9gScBYS7/wr4VaVfV0SkasV0+Fb3AxARSSgFgIhIQikAREQSSgEgIpJQCgARkYRSAIiIJJQCQEQkoRQAIiIJZaU+BKoamFk78Pu464jMAEp8alNVUH1nrpprg+qur5prg+qubyJrW+juLaN1quoAqCZmttHdl8VdRymq78xVc21Q3fVVc21Q3fVVQ206BCQiklAKABGRhFIAlO/+uAsYheo7c9VcG1R3fdVcG1R3fbHXpjkAEZGE0ghARCShFACjMLP5ZvaUmW02szfM7Itx1zSUmaXM7GUzeyzuWoYysylm9pCZvRV9D/8w7poKmdl/jv5dXzezn5pZXYy1rDGz/Wb2ekHbNDNbb2Zbo8fYbnRdor7/Hf3bbjKzX5jZlGqqr2DbfzUzN7MZ1VSbmf25mW2J/g/+r0rXpQAYXR/wl+7+IWA5cKuZXRBzTUN9EdgcdxEl/A3w/9z9fOAjVFGdZjYPuA1Y5u4XEt6l7sYYS3oAuGZI25eBDe6+BNgQrcflAYbXtx640N0vAt4Gbq90UQUeYHh9mNl84ErgvUoXVOABhtRmZv8GWAlc5O5LgW9VuigFwCjcfY+7vxQtdxL+ApsXb1WnmFkr8Cngh3HXMpSZNQH/ClgN4O4n3f1IvFUNkwbqzSwNNAC74yrE3f8JODSkeSWwNlpeC9xQ0aIKFKvP3Z9w975o9Tlg4u9kXkKJ7x/APcBfAbFNeJao7T8Bd7t7T9Rnf6XrUgCMgZktAi4Bno+3ktN8h/A/dz7uQoo4B2gHfhQdovqhmTXGXdQAd99F+FfXe8AeoMPdn4i3qmFmufseCP8YAWbGXM9I/gPweNxFFDKz64Fd7v5q3LUUcS7wL83seTP7RzP7g0oXoAAok5lNAn4O/IW7H427HgAz+zSw391fjLuWEtLAR4Hvu/slwHHiPYRxmuh4+kpgMTAXaDSzP4m3qrOTmX2V8HDpT+KuZYCZNQBfBe6Iu5YS0sBUwkPL/w1YZ2ZWyQIUAGUwswzhL/+fuPvDcddT4HLgejN7F3gQuMLM/i7ekk7TBrS5+8CI6SHCQKgWnwR2uHu7u/cCDwP/IuaahtpnZnMAoseKHyYYjZmtAj4N/LFX13nlHyAM91ejn5FW4CUzmx1rVae0AQ976HeEo/iKTlIrAEYRJfJqYLO7fzvuegq5++3u3uruiwgnL59096r5C9bd9wI7zey8qGkF8GaMJQ31HrDczBqif+cVVNEkdeRRYFW0vAp4JMZahjGza4AvAde7+4m46ynk7q+5+0x3XxT9jLQBH43+X1aDvweuADCzc4EsFf7gOgXA6C4HbiL86/qV6Ou6uIs6i/w58BMz2wRcDPx1zPUMikYmDwEvAa8R/jzEdnWmmf0UeBY4z8zazOxm4G7gSjPbSngmy91VVt+9wGRgffSz8X+rrL6qUKK2NcA50amhDwKrKj2C0pXAIiIJpRGAiEhCKQBERBJKASAiklAKABGRhFIAiIgklAJARCShFAAiIgmlABARSaj/D6Gz7+yGqKSPAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] @@ -354,11 +366,11 @@ { "data": { "text/plain": [ - "Feerates:\t1.1539704395225572, 4.139754128892224, 16.2278954349457 \n", - "Times:\t\t2769.889957638353, 60.00000000000002, 1.0000000000000007" + "Feerates:\t1.1539704395225572, 1.4115360845240776, 4.139754128892224, 16.2278954349457 \n", + "Times:\t\t2769.889957638353, 1513.4606486459202, 60.00000000000002, 1.0000000000000007" ] }, - "execution_count": 23, + "execution_count": 107, "metadata": {}, "output_type": "execute_result" } @@ -389,12 +401,12 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 108, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHh5JREFUeJzt3X2QXXWd5/H39z737ecknRC6ExMkgooo2IP4UK5jxAF1DDMrFo6r0aU2NSvjwzhbik6V7DrljFPjDuqOUhsFjbssQiErcRYHM4CFjAQITyEQICFA0klIOnTSeexO973f/eOcTm6nb3cnffve033P51V1657zO79zzvf60J+c33kyd0dEROInEXUBIiISDQWAiEhMKQBERGJKASAiElMKABGRmFIAiIjElAJARCSmJg0AM7vZzPaa2aYyy/6LmbmZzQvnzcy+b2ZbzWyjmV1c0nelmW0JPyun92eIiMiZOp0jgJ8Cl5/aaGaLgMuA7SXNVwDLws8q4Maw7xzgeuAdwCXA9WbWXknhIiJSmdRkHdz9ATNbUmbRDcBXgLtK2lYAP/Pg9uL1ZtZmZguB9wHr3L0PwMzWEYTKrRPte968eb5kSbldi4jIeB577LF97t4xWb9JA6AcM/sosNPdnzKz0kWdwI6S+Z6wbbz2CS1ZsoQNGzZMpUQRkdgys1dOp98ZB4CZ5YG/Bj5YbnGZNp+gvdz2VxEMH7F48eIzLU9ERE7TVK4Cej2wFHjKzF4GuoDHzewsgn/ZLyrp2wXsmqB9DHdf7e7d7t7d0THpEYyIiEzRGQeAuz/t7vPdfYm7LyH4436xu78KrAU+HV4NdCnQ7+67gXuAD5pZe3jy94Nhm4iIROR0LgO9FXgIOM/Meszsmgm63w1sA7YCPwI+BxCe/P0b4NHw882RE8IiIhINm8nvA+ju7nadBBYROTNm9pi7d0/WT3cCi4jElAJARCSm6jIADg0MccO6F3hyx4GoSxERmbHqMgAKRed7927h8Vf2R12KiMiMVZcB0JgN7m87PDgccSUiIjNXXQZAOpkgl04oAEREJlCXAQDBUcChAQWAiMh46jcAMikODQxFXYaIyIxVtwHQkElqCEhEZAJ1GwD5TJLDGgISERlXXQfAQQ0BiYiMq44DQCeBRUQmUscBoHMAIiITqesAODI4zEx+2qmISJTqNgAasymKDkePF6IuRURkRqrbAMhnkoAeByEiMp66DYDGTPA8IJ0IFhEpr24DQEcAIiITq9sAGHkiqB4HISJSXt0GwIkjAA0BiYiUNWkAmNnNZrbXzDaVtP2DmT1nZhvN7P+aWVvJsq+Z2VYze97M/qik/fKwbauZXTf9P2W0/Mg5AA0BiYiUdTpHAD8FLj+lbR1wgbtfCLwAfA3AzN4EXA28OVznh2aWNLMk8APgCuBNwCfCvlWTzwZHADoJLCJS3qQB4O4PAH2ntP3G3Uf+sq4HusLpFcDP3X3Q3V8CtgKXhJ+t7r7N3Y8DPw/7Vs3IVUAaAhIRKW86zgH8R+DX4XQnsKNkWU/YNl571SQTRjppHB7USWARkXIqCgAz+2tgGLhlpKlMN5+gvdw2V5nZBjPb0NvbW0l5ZFN6HpCIyHimHABmthL4CPBJP/nAnR5gUUm3LmDXBO1juPtqd+929+6Ojo6plgdAJpXgoIaARETKmlIAmNnlwFeBj7r70ZJFa4GrzSxrZkuBZcAjwKPAMjNbamYZghPFaysrfXLppOkcgIjIOFKTdTCzW4H3AfPMrAe4nuCqnyywzswA1rv7n7v7M2Z2O/AswdDQte5eCLfzF8A9QBK42d2fqcLvGSWTTHDwmM4BiIiUM2kAuPsnyjTfNEH/bwHfKtN+N3D3GVVXoWw6Sb8CQESkrLq9Exggl0ooAERExlHXAZBNB+8F1kthRETGqu8ASCUYKjgDQ8WoSxERmXHqOgByqeBxEBoGEhEZq64DIJsOfp4CQERkrPoOgJQCQERkPHUdALm0hoBERMZT1wGgIwARkfHVdQDoCEBEZHx1HQAZHQGIiIyrrgMgYUYurecBiYiUU9cBAME7ARQAIiJjxSAA9DwgEZFyFAAiIjFV9wGQSSU4cFQBICJyqroPgJzeCSAiUlbdB0A2leDggAJARORU9R8A6SSDw0UGhgpRlyIiMqPUfQDkwpvBdCmoiMhodR8ADeHjIPbrRLCIyCiTBoCZ3Wxme81sU0nbHDNbZ2Zbwu/2sN3M7PtmttXMNprZxSXrrAz7bzGzldX5OWONPA+o78jxWu1SRGRWOJ0jgJ8Cl5/Sdh1wr7svA+4N5wGuAJaFn1XAjRAEBnA98A7gEuD6kdCotobMyBGAAkBEpNSkAeDuDwB9pzSvANaE02uAK0vaf+aB9UCbmS0E/ghY5+597r4fWMfYUKmKBh0BiIiUNdVzAAvcfTdA+D0/bO8EdpT06wnbxmuvupEhoP0KABGRUab7JLCVafMJ2sduwGyVmW0wsw29vb0VF5RMGNlUgj4NAYmIjDLVANgTDu0Qfu8N23uARSX9uoBdE7SP4e6r3b3b3bs7OjqmWN5o+UxSRwAiIqeYagCsBUau5FkJ3FXS/unwaqBLgf5wiOge4INm1h6e/P1g2FYTuXSSPl0GKiIySmqyDmZ2K/A+YJ6Z9RBczfNt4HYzuwbYDlwVdr8b+BCwFTgKfBbA3fvM7G+AR8N+33T3U08sV002laDvyGCtdiciMitMGgDu/olxFi0v09eBa8fZzs3AzWdU3TRpSCd1FZCIyCnq/k5gCO4F2H9EQ0AiIqViEQC5dJJjQwU9EE5EpEQsAkA3g4mIjBWPAMgoAEREThWLADhxN7BuBhMROSEWAaAhIBGRsWIVALobWETkpFgEQDadwNARgIhIqVgEQMKMfDZJ72EFgIjIiFgEAEA+k2LfYT0OQkRkRCwC4Ly9v+ZXw3/O/9z2AbjhAth4e9QliYhEbtJnAc125+39NZe9+LekfSBo6N8Bv/pCMH3hx6MrTEQkYnV/BPCe7T8kXRwY3Th0DO79ZjQFiYjMEHUfAM2De8ov6O+pbSEiIjNM3QfAoeyC8gtau2pbiIjIDFP3AfDg4s8xlMiNbkw3wPJvRFOQiMgMUfcngZ+ffwUA73z5B7Qe38tAfiH5K/6bTgCLSOzV/REABCFw49vu4pzBW7jtPXfrj7+ICDEJAIBcOkHCoPeQbgYTEYEYBYCZ0ZTV3cAiIiMqCgAz+0sze8bMNpnZrWaWM7OlZvawmW0xs9vMLBP2zYbzW8PlS6bjB5yJhkxSRwAiIqEpB4CZdQJfALrd/QIgCVwN/D1wg7svA/YD14SrXAPsd/dzgRvCfjXVkE6yVwEgIgJUPgSUAhrMLAXkgd3A+4E7wuVrgCvD6RXhPOHy5WZmFe7/jOQzKQWAiEhoygHg7juB7wDbCf7w9wOPAQfcfTjs1gN0htOdwI5w3eGw/9yp7n8qmrIp+g4fZ7hQrOVuRURmpEqGgNoJ/lW/FDgbaASuKNPVR1aZYFnpdleZ2QYz29Db2zvV8spqzCYpuLNP7wUQEaloCOgDwEvu3uvuQ8CdwLuAtnBICKAL2BVO9wCLAMLlrUDfqRt199Xu3u3u3R0dHRWUN1ZTLihrd/+xad2uiMhsVEkAbAcuNbN8OJa/HHgWuB/4WNhnJXBXOL02nCdcfp+7jzkCqKbmbBqAV/sHJukpIlL/KjkH8DDBydzHgafDba0Gvgp82cy2Eozx3xSuchMwN2z/MnBdBXVPSVN25AhAASAiUtGzgNz9euD6U5q3AZeU6TsAXFXJ/iqVSydIJYw9BxUAIiKxuRMYgruBm3MpHQGIiBCzAABozKZ0ElhEhNgGgI4ARERiFwBN2RR7Dg5QLNb0AiQRkRknlgEwVHD6jupmMBGJt1gGAOheABGR+AVATvcCiIhADAOgOTwC2HVAVwKJSLzFLgDymSSphNGz/2jUpYiIRCp2AWBmtDak2dGnIwARibfYBQAE5wG29+kIQETiLZYB0JJLs0NDQCISc7EMgNaGNIcGhuk/NhR1KSIikYllALSEl4LqRLCIxFk8A6AheDGMTgSLSJzFOgB0BCAicRbLAMilEmRTCXr26whAROIrlgFgZsGVQLoUVERiLJYBANCsewFEJOZiGwCt+TTb+47qvQAiElsVBYCZtZnZHWb2nJltNrN3mtkcM1tnZlvC7/awr5nZ981sq5ltNLOLp+cnTE17Q4bB4SK79YJ4EYmpSo8Avgf8i7ufD7wV2AxcB9zr7suAe8N5gCuAZeFnFXBjhfuuSFs+uBLopd4jUZYhIhKZKQeAmbUA7wVuAnD34+5+AFgBrAm7rQGuDKdXAD/zwHqgzcwWTrnyCrU3ZgB4ad/hqEoQEYlUJUcA5wC9wE/M7Akz+7GZNQIL3H03QPg9P+zfCewoWb8nbItEYyZJJpngRR0BiEhMVRIAKeBi4EZ3vwg4wsnhnnKsTNuYM7BmtsrMNpjZht7e3grKm5iZ0d6Y5qV9CgARiadKAqAH6HH3h8P5OwgCYc/I0E74vbek/6KS9buAXadu1N1Xu3u3u3d3dHRUUN7kWnNptvVqCEhE4mnKAeDurwI7zOy8sGk58CywFlgZtq0E7gqn1wKfDq8GuhToHxkqikpbY4adB44xOFyIsgwRkUikKlz/88AtZpYBtgGfJQiV283sGmA7cFXY927gQ8BW4GjYN1Lt+TRFhx19Rzl3fnPU5YiI1FRFAeDuTwLdZRYtL9PXgWsr2d90a8sHVwK92HtEASAisRPbO4EB5oQBsHWvzgOISPzEOgAyqQStDWmee/VQ1KWIiNRcrAMAYE5jhud2H4y6DBGRmot9AMxtzLBt3xGODxejLkVEpKYUAE0ZCkVnmx4JISIxE/sAmNeUBeB5nQcQkZiJfQC05zMkTAEgIvET+wBIJow5jRkFgIjETuwDAIIrgZ7VlUAiEjMKAKCjKcvu/gH2HzkedSkiIjWjAADmt+QAeHpnf8SViIjUjgIAmN8cXAmkABCROFEAALl0kvZ8mqd7FAAiEh8KgFBHc5aneg5EXYaISM0oAEILmnPs7h9g3+HBqEsREakJBUBofovOA4hIvCgAQh3NWQx4aoeGgUQkHhQAoWwqybymDI+9sj/qUkREakIBUGJhawOPvbKf4YIeDS0i9U8BUOLstgaOHi/oDWEiEgsVB4CZJc3sCTP753B+qZk9bGZbzOw2M8uE7dlwfmu4fEml+55uZ7cFdwQ/+nJfxJWIiFTfdBwBfBHYXDL/98AN7r4M2A9cE7ZfA+x393OBG8J+M0pzLk1rQ5oNL+s8gIjUv4oCwMy6gA8DPw7nDXg/cEfYZQ1wZTi9IpwnXL487D+jnNWS45GX+nD3qEsREamqSo8Avgt8BRg5azoXOODuw+F8D9AZTncCOwDC5f1h/xnl7LYcvYcHefm1o1GXIiJSVVMOADP7CLDX3R8rbS7T1U9jWel2V5nZBjPb0NvbO9XypmzRnDwAD26p/b5FRGqpkiOAdwMfNbOXgZ8TDP18F2gzs1TYpwvYFU73AIsAwuWtwJizre6+2t273b27o6OjgvKmpq0hOA/wwJZ9Nd+3iEgtTTkA3P1r7t7l7kuAq4H73P2TwP3Ax8JuK4G7wum14Tzh8vt8Bg60mxmL2ht46MXXGNL9ACJSx6pxH8BXgS+b2VaCMf6bwvabgLlh+5eB66qw72mxeE6ew4PDeiyEiNS11ORdJufuvwV+G05vAy4p02cAuGo69ldti+bkMYMHtuyje8mcqMsREakK3QlcRi6d5KyWHL99fm/UpYiIVI0CYBxL5jaysaefV/sHoi5FRKQqFADjeH1HIwDrNu+JuBIRkepQAIxjTmOGOfk092x6NepSRESqQgEwDjNjaUcTD217jf5jQ1GXIyIy7RQAE3h9RyOFonPfcxoGEpH6owCYwFktOVpyKe56YtfknUVEZhkFwATMjDcsaOZ3W/bRe2gw6nJERKaVAmAS55/VTMGdXz2lowARqS8KgEnMbcqyoCXLnY/3RF2KiMi0UgCchjcsaGbTroM8r3cFi0gdUQCchjee1UIqYfyv9S9HXYqIyLRRAJyGhkySZfObuPPxnRwa0D0BIlIfFACn6cKuNo4eL3Dn4zujLkVEZFooAE7TWa05zmrJseb3L1Mszrj32IiInDEFwBl466JWtu07ogfEiUhdUACcgTfMb6Ytn+af7tvKDHybpYjIGVEAnIFEwnj74nae3tnP7/TSeBGZ5RQAZ+j8hc0051J8919f0FGAiMxqCoAzlEok+IMlc3h8+wHueUbnAkRk9ppyAJjZIjO738w2m9kzZvbFsH2Oma0zsy3hd3vYbmb2fTPbamYbzezi6foRtfbmhS3Mbczwd7/ezFChGHU5IiJTUskRwDDwV+7+RuBS4FozexNwHXCvuy8D7g3nAa4AloWfVcCNFew7UomE8a5z5/LKa0e5Zf0rUZcjIjIlUw4Ad9/t7o+H04eAzUAnsAJYE3ZbA1wZTq8AfuaB9UCbmS2ccuURWzq3kcVz8nznNy+w56BeHC8is8+0nAMwsyXARcDDwAJ33w1BSADzw26dwI6S1XrCtlnJzPjD8zoYGCrwX9c+E3U5IiJnrOIAMLMm4BfAl9z94ERdy7SNuYzGzFaZ2QYz29Db21tpeVXVls9wydI5/HrTq9zzjF4eLyKzS0UBYGZpgj/+t7j7nWHznpGhnfB7b9jeAywqWb0LGPOWFXdf7e7d7t7d0dFRSXk1cfHidjqas1z3i43s1VCQiMwilVwFZMBNwGZ3/8eSRWuBleH0SuCukvZPh1cDXQr0jwwVzWbJhHH5m8/i8OAwX7rtST0nSERmjUqOAN4NfAp4v5k9GX4+BHwbuMzMtgCXhfMAdwPbgK3Aj4DPVbDvGWVOY4b3Luvg9y++xg/u3xp1OSIipyU11RXd/UHKj+sDLC/T34Frp7q/me7NZ7ew88Ax/vu6F1i2oInLL5i1FziJSEzoTuBpYmYsP38+C1tzfOm2J9m0sz/qkkREJqQAmEapZIIPv2UhmVSCT9/8CC/2Ho66JBGRcSkAplljNsWVb+1kcKjAn/1oPdtfOxp1SSIiZSkAqqC9McOVF3Vy8NgwH1/9EFv3Hoq6JBGRMRQAVTKvKcufXNTJoYEh/v2ND/HYK/ujLklEZBQFQBV1NGe56u2LSBj82Y/W88sn9EJ5EZk5FABV1tqQ5mNv72JeU5Yv3fYk19+1iePDeoS0iERPAVAD+UyKP7mok4sWt7HmoVf40x/+G8+/qvMCIhItBUCNJBPGe5d18OG3LGTbviN85H/8jh/cv1UvlBGRyCgAauzc+U188h2LWTK3kX+453ku/+4D/Pb5vZOvKCIyzRQAEchnUnzoLQv54wsX0nfkOJ/5yaN85uZH2NhzIOrSRCRGpvwsIKncOR1NLJ6b56kd/ax/6TU++k//xh+e18Hnly/j4sXtUZcnInVOARCxVCLB21/XzgWdLTzV08/DL/Vx/w9/z1u7WvnUO5fwkQsXkksng84bb4d7vwn9PdDaBcu/ARd+PNofICKzlgUP6ZyZuru7fcOGDVNa94U9h/h/G2ff6waODxd5dvdBNu3s57Ujx2ltSLPibWfzmeZHWPrQ17GhYyc7pxvgj7+vEBCRUczsMXfvnqyfzgHMMJlUgrctauOT71jMn17UyfyWLP/n4e1kfvut0X/8AYaOBUcEIiJToCGgGcrMWDQnz6I5eQaHC3Q+/FrZft7fw64Dx+hsa6hxhSIy2ykAZoFsKsmh7AJaBse+eH5ncS7v+fZ9LGzN8Y6lc/iDpXO4eHE7585vIp3UAZ6IjE8BMEs8uPhzXPbi35Iunnzx/FAix4OL/jP/zjrYeeAY/7p5L798chcAmWSC885q5oLOVt58dgtvXNjCuR1NtObTUf0EEZlhFACzxPPzrwDgPdt/SPPgHg5lF/Dg4s+xe/4VvA1426I23J0Dx4bYc3CA3kOD9B4e5JdP7OTWR7af2E57Ps2585t4fUfwed3cPJ3tDXS15WlpSGE23ls+RaTeKABmkefnX3EiCMoxM9rzGdrzGc4/K2hzdw4NDLPv8CD7jw6x/+hxdh8Y4NldBzlyvDBq/XwmSWdbA13tDZzd1sDC1hzzmrJ0NAefeU3BJ5PS0JJIPah5AJjZ5cD3gCTwY3f/dq1riBMzo6UhTUvD2KGfgaEC/ceGODgwxKGBYQ4dG+bQ4BDP7j7I+m19HBsqlNkitORSdDRnmduUpbUhTVtDmtbw05YP9tWWz5xoa86laMykyKUTOsIQmUFqGgBmlgR+AFwG9ACPmtlad3+2lnVIIJdOkksnWdCSK7t8uFDk6PFC+Bnm6PECR8Lvo4MFdu4/xkv7jjA4VODYUIGhwsT3lCQMGjJJGjMpGrMpGrNJmrJBOOSzKZqySfKZFA3pJNlUglw6STadIJcKvrOp0fMnvkv6p5NGOpEgkVDQyCwS0U2etT4CuATY6u7bAMzs58AKQAEwA6WSCVoaEmWPHsopFJ2BoQKDw8VR38cLRYaGiwwVPJguFDk+XOTwwDD7jwwxXAzmhwoefhep9PbEpBmpZPBJJxKkk4lgOpkIQiKZKDudKmlLJRIkE5Ac+TYjkTBSieA7aUYyEX7CZcmR5eGykf4nl4fbs2A6EdaZsJMfMzALlhmQSBhBngXfI31O9MVIJE72Nwv6jXwnwqOuke0YJ5ePbGek3Uq2U66WkSO4YDr4z1pHdRXaeDv86gvBfT0A/TuCeah6CNQ6ADqBHSXzPcA7alyDVEkyYeG/7CvbjrtTdBguFikUneGCB99FZ7hYHDtfdAqFYL7gTrHoFN0pFhk1XyhpOz5c5NhQ4cS+gj5h/xPrQNEdH+d7ZHrm3ktfWyMxMBJKIw1W2kZpcIwsHwmykxsaaTu5vo3a/omuJ4Lp5LZKdn1iv3ZqLSUbObW2U4OOMstL1z1Zto1tm2w5cPOBrzO/OM5NnnUWAOX+qTDq/z9mtgpYBbB48eIp7+iceY38p/eeM+X1RU5XsSR4ToRMsUjBTy4rFINPsWS6NJAKRcdLQ4aTQXgyeIL2oK00iABGB9aJdUdtx8esGwTYqe3BMji5/ZFlwZ5Kpwmngwb30rbRfQlrGb08mPETy8dun7CtdPulfRlVy6nbOrkzH7O+U1L6if2Wrs+Y33ryx5T+4Sr3RJ1yfUf9ZxF+d/TtG7syBMNBVVbrAOgBFpXMdwG7Sju4+2pgNQTPAprqjlLJBE26EUpEZrobuoJhn1O1dlV917X+C/kosMzMlppZBrgaWFvjGkREZo7l3wge7Fgq3RC0V1lNjwDcfdjM/gK4h+Ay0Jvd/Zla1iAiMqOMjPPH4Cog3P1u4O5a71dEZMa68OORPNZdg+QiIjGlABARiSkFgIhITCkARERiSgEgIhJTCgARkZhSAIiIxJQCQEQkpszLPcVohjCzXuCVqOuYgnnAOE94qjtx+q2g31vP6um3vs7dOybrNKMDYLYysw3u3h11HbUQp98K+r31LE6/dYSGgEREYkoBICISUwqA6lgddQE1FKffCvq99SxOvxXQOQARkdjSEYCISEwpAKaJmS0ys/vNbLOZPWNmX4y6pmozs6SZPWFm/xx1LdVmZm1mdoeZPRf+d/zOqGuqJjP7y/B/x5vM7FYzy0Vd03Qys5vNbK+ZbSppm2Nm68xsS/jdHmWNtaAAmD7DwF+5+xuBS4FrzexNEddUbV8ENkddRI18D/gXdz8feCt1/LvNrBP4AtDt7hcQvL3v6mirmnY/BS4/pe064F53XwbcG87XNQXANHH33e7+eDh9iOAPRGe0VVWPmXUBHwZ+HHUt1WZmLcB7gZsA3P24ux+ItqqqSwENZpYC8sCuiOuZVu7+ANB3SvMKYE04vQa4sqZFRUABUAVmtgS4CHg42kqq6rvAV4Bi1IXUwDlAL/CTcMjrx2bWGHVR1eLuO4HvANuB3UC/u/8m2qpqYoG774bgH3TA/IjrqToFwDQzsybgF8CX3P1g1PVUg5l9BNjr7o9FXUuNpICLgRvd/SLgCHU8PBCOfa8AlgJnA41m9h+irUqqQQEwjcwsTfDH/xZ3vzPqeqro3cBHzexl4OfA+83sf0dbUlX1AD3uPnJEdwdBINSrDwAvuXuvuw8BdwLvirimWthjZgsBwu+9EddTdQqAaWJmRjBGvNnd/zHqeqrJ3b/m7l3uvoTg5OB97l63/0J091eBHWZ2Xti0HHg2wpKqbTtwqZnlw/9dL6eOT3qXWAusDKdXAndFWEtNpKIuoI68G/gU8LSZPRm2fd3d746wJpk+nwduMbMMsA34bMT1VI27P2xmdwCPE1zd9gR1dpesmd0KvA+YZ2Y9wPXAt4HbzewaghC8KroKa0N3AouIxJSGgEREYkoBICISUwoAEZGYUgCIiMSUAkBEJKYUACIiMaUAEBGJKQWAiEhM/X8k2vFG5y1P8wAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHmVJREFUeJzt3X2QXXWd5/H39z72U/op6TzQnZggEVREwR7Ah3IdIw5RxzCzYuG4Gl1qU7MwPoyzpehUyYxTM+PUuAO6A9REQeMui7DImjgLg5mAhSgBwlMMhJAQIOkkJB066Tx2p/ve7/5xTie307e7k7597+m+5/OqunXP+Z3fOed7fehPfufR3B0REYmfRNQFiIhINBQAIiIxpQAQEYkpBYCISEwpAEREYkoBICISUwoAEZGYGjcAzOwOM9tnZpuKLPtvZuZmNiucNzP7vpltM7ONZnZJQd/lZrY1/Cyf3J8hIiJn60xGAD8Grjy90czmA1cAOwqalwKLw88K4LawbytwI3AZcClwo5m1lFK4iIiUJjVeB3d/xMwWFll0E/A1YHVB2zLgJx7cXrzezJrNbB7wQWCtu/cAmNlaglC5a6x9z5o1yxcuLLZrEREZzVNPPbXf3dvG6zduABRjZp8Adrn7c2ZWuKgd2Fkw3xW2jdY+poULF7Jhw4aJlCgiEltm9tqZ9DvrADCzOuAvgY8UW1ykzcdoL7b9FQSHj1iwYMHZliciImdoIlcBvRlYBDxnZq8CHcDTZjaX4F/28wv6dgC7x2gfwd1Xununu3e2tY07ghERkQk66wBw99+5+2x3X+juCwn+uF/i7q8Da4DPhVcDXQ70uvse4EHgI2bWEp78/UjYJiIiETmTy0DvAh4DzjezLjO7dozu9wPbgW3AD4DrAMKTv38DPBl+vj10QlhERKJhU/l9AJ2dna6TwCIiZ8fMnnL3zvH66U5gEZGYUgCIiMRUVQbA4b4Bblr7Es/uPBh1KSIiU1ZVBkAu73xv3Vaefu1A1KWIiExZVRkA9dng/rYj/YMRVyIiMnVVZQCkkwlq0gkFgIjIGKoyACAYBRzuUwCIiIymegMgk+Jw30DUZYiITFlVGwC1maQOAYmIjKFqA6Auk+SIDgGJiIyqqgPgkA4BiYiMqooDQCeBRUTGUsUBoHMAIiJjqeoAONo/yFR+2qmISJSqNgDqsynyDsdO5KIuRURkSqraAKjLJAE9DkJEZDRVGwD1meB5QDoRLCJSXNUGgEYAIiJjq9oAGHoiqB4HISJSXNUGwMkRgA4BiYgUNW4AmNkdZrbPzDYVtP2jmb1oZhvN7P+aWXPBsm+Y2TYz22Jmf1DQfmXYts3Mbpj8nzJc3dA5AB0CEhEp6kxGAD8GrjytbS1wobtfBLwEfAPAzN4GXAO8PVznVjNLmlkSuAVYCrwN+HTYt2zqssEIQCeBRUSKGzcA3P0RoOe0tl+6+9Bf1vVARzi9DPipu/e7+yvANuDS8LPN3be7+wngp2Hfshm6CkiHgEREipuMcwD/GXggnG4HdhYs6wrbRmsvm2TCSCeNI/06CSwiUkxJAWBmfwkMAncONRXp5mO0F9vmCjPbYGYburu7SymPbErPAxIRGc2EA8DMlgMfBz7jpx640wXML+jWAeweo30Ed1/p7p3u3tnW1jbR8gDIpBIc0iEgEZGiJhQAZnYl8HXgE+5+rGDRGuAaM8ua2SJgMfAE8CSw2MwWmVmG4ETxmtJKH186aToHICIyitR4HczsLuCDwCwz6wJuJLjqJwusNTOA9e7+p+7+vJndA7xAcGjoenfPhdv5M+BBIAnc4e7Pl+H3DJNJJjh0XOcARESKGTcA3P3TRZpvH6P/3wJ/W6T9fuD+s6quRNl0kl4FgIhIUVV7JzBATSqhABARGUVVB0A2HbwXWC+FEREZqboDIJVgIOf0DeSjLkVEZMqp6gCoSQWPg9BhIBGRkao6ALLp4OcpAERERqruAEgpAERERlPVAVCT1iEgEZHRVHUAaAQgIjK6qg4AjQBEREZX1QGQ0QhARGRUVR0ACTNq0noekIhIMVUdABC8E0ABICIyUgwCQM8DEhEpRgEgIhJTVR8AmVSCg8cUACIip6v6AKjROwFERIqq+gDIphIc6lMAiIicrvoDIJ2kfzBP30Au6lJERKaUqg+AmvBmMF0KKiIyXNUHQG34OIgDOhEsIjLMuAFgZneY2T4z21TQ1mpma81sa/jdErabmX3fzLaZ2UYzu6RgneVh/61mtrw8P2ekoecB9Rw9UaldiohMC2cyAvgxcOVpbTcA69x9MbAunAdYCiwOPyuA2yAIDOBG4DLgUuDGodAot9rM0AhAASAiUmjcAHD3R4Ce05qXAavC6VXAVQXtP/HAeqDZzOYBfwCsdfcedz8ArGVkqJRFrUYAIiJFTfQcwBx33wMQfs8O29uBnQX9usK20drLbugQ0AEFgIjIMJN9EtiKtPkY7SM3YLbCzDaY2Ybu7u6SC0omjGwqQY8OAYmIDDPRANgbHtoh/N4XtncB8wv6dQC7x2gfwd1Xununu3e2tbVNsLzh6jJJjQBERE4z0QBYAwxdybMcWF3Q/rnwaqDLgd7wENGDwEfMrCU8+fuRsK0iatJJenQZqIjIMKnxOpjZXcAHgVlm1kVwNc93gHvM7FpgB3B12P1+4KPANuAY8AUAd+8xs78Bngz7fdvdTz+xXDbZVIKeo/2V2p2IyLQwbgC4+6dHWbSkSF8Hrh9lO3cAd5xVdZOkNp3UVUAiIqep+juBIbgX4MBRHQISESkUiwCoSSc5PpDTA+FERArEIgB0M5iIyEjxCICMAkBE5HSxCICTdwPrZjARkZNiEQA6BCQiMlKsAkB3A4uInBKLAMimExgaAYiIFIpFACTMqMsm6T6iABARGRKLAACoy6TYf0SPgxARGRKLADh/3wP8YvBP+ZftH4abLoSN90RdkohI5MZ9FtB0d/6+B7ji5b8j7X1BQ+9O+MWXgumLPhVdYSIiEav6EcD7d9xKOt83vHHgOKz7djQFiYhMEVUfADP69xZf0NtV2UJERKaYqg+Aw9k5xRc0dVS2EBGRKabqA+DRBdcxkKgZ3piuhSXfiqYgEZEpoupPAm+ZvRSA97x6C00n9tFXN4+6pX+tE8AiEntVPwKAIARue9dqzu2/k7vff7/++IuIEJMAAKhJJ0gYdB/WzWAiIhCjADAzGrK6G1hEZEhJAWBmf25mz5vZJjO7y8xqzGyRmT1uZlvN7G4zy4R9s+H8tnD5wsn4AWejNpPUCEBEJDThADCzduBLQKe7XwgkgWuAfwBucvfFwAHg2nCVa4ED7n4ecFPYr6Jq00n2KQBERIDSDwGlgFozSwF1wB7gQ8C94fJVwFXh9LJwnnD5EjOzEvd/VuoyKQWAiEhowgHg7ruA7wI7CP7w9wJPAQfdfTDs1gW0h9PtwM5w3cGw/8yJ7n8iGrIpeo6cYDCXr+RuRUSmpFIOAbUQ/Kt+EXAOUA8sLdLVh1YZY1nhdleY2QYz29Dd3T3R8oqqzybJubNf7wUQESnpENCHgVfcvdvdB4D7gPcCzeEhIYAOYHc43QXMBwiXNwE9p2/U3Ve6e6e7d7a1tZVQ3kgNNUFZe3qPT+p2RUSmo1ICYAdwuZnVhcfylwAvAA8Dnwz7LAdWh9NrwnnC5Q+5+4gRQDnNyKYBeL23b5yeIiLVr5RzAI8TnMx9GvhduK2VwNeBr5rZNoJj/LeHq9wOzAzbvwrcUELdE9KQHRoBKABEREp6FpC73wjceFrzduDSIn37gKtL2V+patIJUglj7yEFgIhIbO4EhuBu4Bk1KY0ARESIWQAA1GdTOgksIkJsA0AjABGR2AVAQzbF3kN95PMVvQBJRGTKiWUADOScnmO6GUxE4i2WAQC6F0BEJH4BUKN7AUREIIYBMCMcAew+qCuBRCTeYhcAdZkkqYTRdeBY1KWIiEQqdgFgZjTVptnZoxGAiMRb7AIAgvMAO3o0AhCReItlADTWpNmpQ0AiEnOxDICm2jSH+wbpPT4QdSkiIpGJZQA0hpeC6kSwiMRZPAOgNngxjE4Ei0icxToANAIQkTiLZQDUpBJkUwm6DmgEICLxFcsAMLPgSiBdCioiMRbLAACYoXsBRCTmYhsATXVpdvQc03sBRCS2SgoAM2s2s3vN7EUz22xm7zGzVjNba2Zbw++WsK+Z2ffNbJuZbTSzSybnJ0xMS22G/sE8e/SCeBGJqVJHAN8D/s3dLwDeCWwGbgDWuftiYF04D7AUWBx+VgC3lbjvkjTXBVcCvdJ9NMoyREQiM+EAMLNG4APA7QDufsLdDwLLgFVht1XAVeH0MuAnHlgPNJvZvAlXXqKW+gwAr+w/ElUJIiKRKmUEcC7QDfzIzJ4xsx+aWT0wx933AITfs8P+7cDOgvW7wrZI1GeSZJIJXtYIQERiqpQASAGXALe5+8XAUU4d7inGirSNOANrZivMbIOZbeju7i6hvLGZGS31aV7ZrwAQkXgqJQC6gC53fzycv5cgEPYOHdoJv/cV9J9fsH4HsPv0jbr7SnfvdPfOtra2EsobX1NNmu3dOgQkIvE04QBw99eBnWZ2fti0BHgBWAMsD9uWA6vD6TXA58KrgS4HeocOFUWluT7DroPH6R/MRVmGiEgkUiWu/0XgTjPLANuBLxCEyj1mdi2wA7g67Hs/8FFgG3As7Buplro0eYedPcc4b/aMqMsREamokgLA3Z8FOossWlKkrwPXl7K/ydZcF1wJ9HL3UQWAiMRObO8EBmgNA2DbPp0HEJH4iXUAZFIJmmrTvPj64ahLERGpuFgHAEBrfYYX9xyKugwRkYqLfQDMrM+wff9RTgzmoy5FRKSiFAANGXJ5Z7seCSEiMRP7AJjVkAVgi84DiEjMxD4AWuoyJEwBICLxE/sASCaM1vqMAkBEYif2AQDBlUAv6EogEYkZBQCwzH7D/zm+Av+rZrjpQth4T9QliYiUXanPApr2zt/3AB/uvZlMoj9o6N0Jv/hSMH3Rp6IrTESkzGI/Anj/jlvJeP/wxoHjsO7b0RQkIlIhsQ+AGf17iy/o7apsISIiFRb7ADicnVN8QVNHZQsREamw2AfAowuuYyBRM7wxXQtLvhVNQSIiFRL7k8BbZi8F4PJXbqF5YB/99fOovfKvdQJYRKpe7EcAEITAynev5s39d/Ivl6zWH38RiQUFQCibSjKrIcNTrx2IuhQRkYpQABSY11TLU68dYDCnR0OLSPVTABQ4p7mWYydyekOYiMRCyQFgZkkze8bM/jWcX2Rmj5vZVjO728wyYXs2nN8WLl9Y6r4n2znNwdVAT77aE3ElIiLlNxkjgC8Dmwvm/wG4yd0XAweAa8P2a4ED7n4ecFPYb0qZUZOmqTbNhld1HkBEql9JAWBmHcDHgB+G8wZ8CLg37LIKuCqcXhbOEy5fEvafUuY21vDEKz24e9SliIiUVakjgJuBrwFDZ01nAgfdfTCc7wLaw+l2YCdAuLw37D+lnNNcQ/eRfl5941jUpYiIlNWEA8DMPg7sc/enCpuLdPUzWFa43RVmtsHMNnR3d0+0vAmb31oHwKNbK79vEZFKKmUE8D7gE2b2KvBTgkM/NwPNZjZ0h3EHsDuc7gLmA4TLm4ARZ1vdfaW7d7p7Z1tbWwnlTUxzbXAe4JGt+yu+bxGRSppwALj7N9y9w90XAtcAD7n7Z4CHgU+G3ZYDq8PpNeE84fKHfAoeaDcz5rfU8tjLbzCg+wFEpIqV4z6ArwNfNbNtBMf4bw/bbwdmhu1fBW4ow74nxYLWOo70D/LczoNRlyIiUjaT8jA4d/8V8KtwejtwaZE+fcDVk7G/cpvfWocZPLJ1P50LW6MuR0SkLHQncBE16SRzG2v41ZZ9UZciIlI2CoBRLJxZz8auXl7v7Yu6FBGRslAAjOLNbfUArN08yisjRUSmOQXAKFrrM7TWpXlw0+tRlyIiUhYKgFGYGYvaGnhs+xv0Hh+IuhwRkUmnABjDm9vqyeWdh17UYSARqT4KgDHMbayhsSbF6md2j99ZRGSaUQCMwcx4y5wZ/HrrfroP90ddjojIpFIAjOOCuTPIufOL5zQKEJHqogAYx8yGLHMas9z3dFfUpYiITCoFwBl4y5wZbNp9iC16V7CIVBEFwBl469xGUgnjf65/NepSREQmjQLgDNRmkiye3cB9T+/icJ/uCRCR6qAAOEMXdTRz7ESO+57eFXUpIiKTQgFwhuY21TC3sYZVv32VfH7KvcdGROSsKQDOwjvnN7F9/1E9IE5EqoIC4Cy8ZfYMmuvS/PND25iCb7MUETkrCoCzkEgY717Qwu929fJrvTReRKY5BcBZumDeDGbUpLj531/SKEBEpjUFwFlKJRL83sJWnt5xkAef17kAEZm+JhwAZjbfzB42s81m9ryZfTlsbzWztWa2NfxuCdvNzL5vZtvMbKOZXTJZP6LS3j6vkZn1Gf7+gc0M5PJRlyMiMiGljAAGgb9w97cClwPXm9nbgBuAde6+GFgXzgMsBRaHnxXAbSXsO1KJhPHe82by2hvHuHP9a1GXIyIyIRMOAHff4+5Ph9OHgc1AO7AMWBV2WwVcFU4vA37igfVAs5nNm3DlEVs0s54FrXV895cvsfeQXhwvItPPpJwDMLOFwMXA48Acd98DQUgAs8Nu7cDOgtW6wrZpycz4/fPb6BvI8Vdrno+6HBGRs1ZyAJhZA/Az4CvufmisrkXaRlxGY2YrzGyDmW3o7u4utbyyaq7LcOmiVh7Y9DoPPq+Xx4vI9FJSAJhZmuCP/53ufl/YvHfo0E74vS9s7wLmF6zeAYx4y4q7r3T3TnfvbGtrK6W8irhkQQttM7Lc8LON7NOhIBGZRkq5CsiA24HN7v5PBYvWAMvD6eXA6oL2z4VXA10O9A4dKprOkgnjyrfP5Uj/IF+5+1k9J0hEpo1SRgDvAz4LfMjMng0/HwW+A1xhZluBK8J5gPuB7cA24AfAdSXse0pprc/wgcVt/PblN7jl4W1RlyMickZSE13R3R+l+HF9gCVF+jtw/UT3N9W9/ZxGdh08zn9f+xKL5zRw5YXT9gInEYkJ3Qk8ScyMJRfMZl5TDV+5+1k27eqNuiQRkTEpACZRKpngY++YRyaV4HN3PMHL3UeiLklEZFQKgElWn01x1Tvb6R/I8Sc/WM+ON45FXZKISFEKgDJoqc9w1cXtHDo+yKdWPsa2fYejLklEZAQFQJnMasjyRxe3c7hvgP9422M89dqBqEsSERlGAVBGbTOyXP3u+SQM/uQH6/n5M3qhvIhMHQqAMmuqTfPJd3cwqyHLV+5+lhtXb+LEoB4hLSLRUwBUQF0mxR9d3M7FC5pZ9dhr/PGtv2HL6zovICLRUgBUSDJhfGBxGx97xzy27z/Kx//Hr7nl4W16oYyIREYBUGHnzW7gM5ctYOHMev7xwS1cefMj/GrLvvFXFBGZZAqACNRlUnz0HfP4w4vm0XP0BJ//0ZN8/o4n2Nh1MOrSRCRGJvwsICnduW0NLJhZx3M7e1n/yht84p9/w++f38YXlyzmkgUtUZcnIlVOARCxVCLBu9/UwoXtjTzX1cvjr/Tw8K2/5Z0dTXz2PQv5+EXzqEkng84b74F134beLmjqgCXfgos+Fe0PEJFpy4KHdE5NnZ2dvmHDhgmt+9Lew/y/jdPvdQMnBvO8sOcQm3b18sbREzTVpln2rnP4/IwnWPTYN7GB46c6p2vhD7+vEBCRYczsKXfvHK+fzgFMMZlUgnfNb+Yzly3gjy9uZ3Zjlv/9+A4yv/rb4X/8AQaOByMCEZEJ0CGgKcrMmN9ax/zWOvoHc7Q//kbRft7bxe6Dx2lvrq1whSIy3SkApoFsKsnh7Bwa+0e+eH5Xfibv/85DzGuq4bJFrfzeolYuWdDCebMbSCc1wBOR0SkApolHF1zHFS//Hen8qRfPDyRqeHT+f+U/WBu7Dh7n3zfv4+fP7gYgk0xw/twZXNjexNvPaeSt8xo5r62Bprp0VD9BRKYYBcA0sWX2UgDev+NWZvTv5XB2Do8uuI49s5fyLuBd85txdw4eH2DvoT66D/fTfaSfnz+zi7ue2HFyOy11ac6b3cCb24LPm2bW0d5SS0dzHY21KcxGe8uniFQbBcA0smX20pNBUIyZ0VKXoaUuwwVzgzZ353DfIPuP9HPg2AAHjp1gz8E+Xth9iKMncsPWr8skaW+upaOllnOaa5nXVMOshixtM4LPrIbgk0np0JJINah4AJjZlcD3gCTwQ3f/TqVriBMzo7E2TWPtyEM/fQM5eo8PcKhvgMN9gxw+Psjh/gFe2HOI9dt7OD6QK7JFaKxJ0TYjy8yGLE21aZpr0zSFn+a6YF/NdZmTbTNqUtRnUtSkExphiEwhFQ0AM0sCtwBXAF3Ak2a2xt1fqGQdEqhJJ6lJJ5nTWFN0+WAuz7ETufAzyLETOY6G38f6c+w6cJxX9h+lfyDH8YEcA7mx7ylJGNRmktRnUtRnU9RnkzRkg3Coy6ZoyCapy6SoTSfJphLUpJNk0wlqUsF3NjV8/uR3Qf900kgnEiQSChqZRiK6ybPSI4BLgW3uvh3AzH4KLAMUAFNQKpmgsTZRdPRQTC7v9A3k6B/MD/s+kcszMJhnIOfBdC7PicE8R/oGOXB0gMF8MD+Q8/A7T6m3JybNSCWDTzqRIJ1MBNPJRBASyUTR6VRBWyqRIJmA5NC3GYmEkUoE30kzkonwEy5LDi0Plw31P7U83J4F04mwzoSd+piBWbDMgETCCPIs+B7qc7IvRiJxqr9Z0G/oOxGOuoa2Y5xaPrSdoXYr2E6xWoZGcMF08J+1RnUl2ngP/OJLwX09AL07g3koewhUOgDagZ0F813AZRWuQcokmbDwX/albcfdyTsM5vPk8s5gzoPvvDOYz4+czzu5XDCfcyefd/Lu5PMMm88VtJ0YzHN8IHdyX0GfsP/JdSDvjo/yPTQ9de+lr6yhGBgKpaEGK2yjMDiGlg8F2akNDbWdWt+Gbf9k15PBdGpbBbs+uV87vZaCjZxe2+lBR5HlheueKttGto23HLjj4DeZnR/lJs8qC4Bi/1QY9v8fM1sBrABYsGDBhHd07qx6/ssHzp3w+iJnKl8QPCdDJp8n56eW5fLBJ18wXRhIubzjhSHDqSA8FTxBe9BWGEQAwwPr5LrDtuMj1g0C7PT2YBmc2v7QsmBPhdOE00GDe2Hb8L6EtQxfHsz4yeUjt0/YVrj9wr4Mq+X0bZ3amY9Y3yko/eR+C9dnxG899WMK/3AVe6JOsb7D/rMIv9t69o9cGYLDQWVW6QDoAuYXzHcAuws7uPtKYCUEzwKa6I5SyQQNuhFKRKa6mzqCwz6na+oo+64r/RfySWCxmS0yswxwDbCmwjWIiEwdS74VPNixULo2aC+zio4A3H3QzP4MeJDgMtA73P35StYgIjKlDB3nj8FVQLj7/cD9ld6viMiUddGnInmsuw6Si4jElAJARCSmFAAiIjGlABARiSkFgIhITCkARERiSgEgIhJTCgARkZgyL/YUoynCzLqB16KuYwJmAaM84anqxOm3gn5vNaum3/omd28br9OUDoDpysw2uHtn1HVUQpx+K+j3VrM4/dYhOgQkIhJTCgARkZhSAJTHyqgLqKA4/VbQ761mcfqtgM4BiIjElkYAIiIxpQCYJGY238weNrPNZva8mX056prKzcySZvaMmf1r1LWUm5k1m9m9ZvZi+N/xe6KuqZzM7M/D/x1vMrO7zKwm6pomk5ndYWb7zGxTQVurma01s63hd0uUNVaCAmDyDAJ/4e5vBS4Hrjezt0VcU7l9GdgcdREV8j3g39z9AuCdVPHvNrN24EtAp7tfSPD2vmuirWrS/Ri48rS2G4B17r4YWBfOVzUFwCRx9z3u/nQ4fZjgD0R7tFWVj5l1AB8Dfhh1LeVmZo3AB4DbAdz9hLsfjLaqsksBtWaWAuqA3RHXM6nc/RGg57TmZcCqcHoVcFVFi4qAAqAMzGwhcDHweLSVlNXNwNeAfNSFVMC5QDfwo/CQ1w/NrD7qosrF3XcB3wV2AHuAXnf/ZbRVVcQcd98DwT/ogNkR11N2CoBJZmYNwM+Ar7j7oajrKQcz+ziwz92firqWCkkBlwC3ufvFwFGq+PBAeOx7GbAIOAeoN7P/FG1VUg4KgElkZmmCP/53uvt9UddTRu8DPmFmrwI/BT5kZv8r2pLKqgvocvehEd29BIFQrT4MvOLu3e4+ANwHvDfimiphr5nNAwi/90VcT9kpACaJmRnBMeLN7v5PUddTTu7+DXfvcPeFBCcHH3L3qv0Xoru/Duw0s/PDpiXACxGWVG47gMvNrC783/USqvikd4E1wPJwejmwOsJaKiIVdQFV5H3AZ4HfmdmzYds33f3+CGuSyfNF4E4zywDbgS9EXE/ZuPvjZnYv8DTB1W3PUGV3yZrZXcAHgVlm1gXcCHwHuMfMriUIwaujq7AydCewiEhM6RCQiEhMKQBERGJKASAiElMKABGRmFIAiIjElAJARCSmFAAiIjGlABARian/D+hfAG5Faoo0AAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] @@ -405,11 +417,11 @@ { "data": { "text/plain": [ - "Feerates:\t1.1531420689155165, 2.816548045571761, 11.10120050773006 \n", - "Times:\t\t874.010579873836, 60.00000000000001, 1.0000000000000004" + "Feerates:\t1.1531420689155165, 1.4085104512204296, 2.816548045571761, 11.10120050773006 \n", + "Times:\t\t874.010579873836, 479.615551452334, 60.00000000000001, 1.0000000000000004" ] }, - "execution_count": 24, + "execution_count": 108, "metadata": {}, "output_type": "execute_result" } diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs index 4dedf711b..5ef3579a5 100644 --- a/mining/src/feerate/mod.rs +++ b/mining/src/feerate/mod.rs @@ -136,8 +136,10 @@ impl FeerateEstimator { let high = self.time_to_feerate(1f64).max(min); // Choose `low` feerate such that it provides sub-hour waiting time AND it covers (at least) the 0.25 quantile let low = self.time_to_feerate(3600f64).max(self.quantile(min, high, 0.25)); - // Choose `mid` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.5 quantile between low and high. - let mid = self.time_to_feerate(60f64).max(self.quantile(low, high, 0.5)); + // Choose `normal` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.66 quantile between low and high. + let normal = self.time_to_feerate(60f64).max(self.quantile(low, high, 0.66)); + // Choose an additional point between normal and low + let mid = self.time_to_feerate(1800f64).max(self.quantile(min, high, 0.5)); /* Intuition for the above: 1. The quantile calculations make sure that we return interesting points on the `feerate_to_time` curve. 2. They also ensure that the times don't diminish too high if small increments to feerate would suffice @@ -145,7 +147,10 @@ impl FeerateEstimator { */ FeerateEstimations { priority_bucket: FeerateBucket { feerate: high, estimated_seconds: self.feerate_to_time(high) }, - normal_buckets: vec![FeerateBucket { feerate: mid, estimated_seconds: self.feerate_to_time(mid) }], + normal_buckets: vec![ + FeerateBucket { feerate: normal, estimated_seconds: self.feerate_to_time(normal) }, + FeerateBucket { feerate: mid, estimated_seconds: self.feerate_to_time(mid) }, + ], low_buckets: vec![FeerateBucket { feerate: low, estimated_seconds: self.feerate_to_time(low) }], } } @@ -185,6 +190,7 @@ mod tests { let estimator = FeerateEstimator { total_weight: 0.00659, inclusion_interval: 0.004f64 }; let minimum_feerate = 0.755; let estimations = estimator.calc_estimations(minimum_feerate); + println!("{estimations}"); let buckets = estimations.ordered_buckets(); assert!(buckets.last().unwrap().feerate >= minimum_feerate); for (i, j) in buckets.into_iter().tuple_windows() { From c7e60cb7f8499faa8486f756da84911069a35e6c Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 15 Aug 2024 23:28:31 +0000 Subject: [PATCH 72/74] enum order --- rpc/wrpc/client/src/client.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rpc/wrpc/client/src/client.rs b/rpc/wrpc/client/src/client.rs index 052c51dd9..c65d582f0 100644 --- a/rpc/wrpc/client/src/client.rs +++ b/rpc/wrpc/client/src/client.rs @@ -596,9 +596,11 @@ impl RpcApi for KaspaRpcClient { GetBlockTemplate, GetCoinSupply, GetConnectedPeerInfo, + GetCurrentNetwork, GetDaaScoreTimestampEstimate, GetServerInfo, - GetCurrentNetwork, + GetFeeEstimate, + GetFeeEstimateExperimental, GetHeaders, GetInfo, GetMempoolEntries, @@ -619,8 +621,6 @@ impl RpcApi for KaspaRpcClient { SubmitTransaction, SubmitTransactionReplacement, Unban, - GetFeeEstimate, - GetFeeEstimateExperimental ] ); From ddc127199c74e51850e2f8e2bea45241f9e04314 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Fri, 16 Aug 2024 12:49:44 +0000 Subject: [PATCH 73/74] with 1 sec there are rare cases where mempool size does not change and we exit early --- testing/integration/src/tasks/tx/sender.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration/src/tasks/tx/sender.rs b/testing/integration/src/tasks/tx/sender.rs index 26a334a76..d29e74373 100644 --- a/testing/integration/src/tasks/tx/sender.rs +++ b/testing/integration/src/tasks/tx/sender.rs @@ -114,7 +114,7 @@ impl Task for TransactionSenderTask { break; } prev_mempool_size = mempool_size; - sleep(Duration::from_secs(1)).await; + sleep(Duration::from_secs(2)).await; } if stopper == Stopper::Signal { warn!("Tx sender task signaling to stop"); From adbaf181df296d8a1f19128b08f3e18debcc4089 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Fri, 16 Aug 2024 12:56:18 +0000 Subject: [PATCH 74/74] final stuff --- mining/src/feerate/fee_estimation.ipynb | 6 +++--- mining/src/mempool/model/frontier.rs | 2 +- rpc/wrpc/client/src/client.rs | 8 ++++---- rpc/wrpc/server/src/router.rs | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mining/src/feerate/fee_estimation.ipynb b/mining/src/feerate/fee_estimation.ipynb index 610d8fcd0..694f47450 100644 --- a/mining/src/feerate/fee_estimation.ipynb +++ b/mining/src/feerate/fee_estimation.ipynb @@ -9,7 +9,7 @@ "The feerate value represents the fee/mass ratio of a transaction in `sompi/gram` units.\n", "Given a feerate value recommendation, one should calculate the required fee by taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)`. \n", "\n", - "This notebook makes an effort to implement and illustrate the feerate estimator method we used. The corresponding Rust implementation is more comprehencive and addresses some additional edge cases, but the code in this notebook highly reflects it." + "This notebook makes an effort to implement and illustrate the feerate estimator method we used. The corresponding Rust implementation is more comprehensive and addresses some additional edge cases, but the code in this notebook highly reflects it." ] }, { @@ -97,7 +97,7 @@ }, { "cell_type": "code", - "execution_count": 103, + "execution_count": 109, "metadata": {}, "outputs": [], "source": [ @@ -200,7 +200,7 @@ " # Choose `low` feerate such that it provides sub-hour waiting time AND it covers (at least) the 0.25 quantile\n", " low = max(self.time_to_feerate(3600.0), self.quantile(1.0, high, 0.25))\n", " \n", - " # Choose `normal` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.75\n", + " # Choose `normal` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.66\n", " # quantile between low and high\n", " normal = max(self.time_to_feerate(60.0), self.quantile(low, high, 0.66))\n", " \n", diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs index b8fbde74a..e8d2b54ab 100644 --- a/mining/src/mempool/model/frontier.rs +++ b/mining/src/mempool/model/frontier.rs @@ -91,7 +91,7 @@ impl Frontier { /// 3. It remains to deal with the case where the weight distribution is highly biased. The process implemented below /// keeps track of the top-weight element. If the distribution is highly biased, this element will be sampled with /// sufficient probability (in constant time). Following each sampling collision we search for a consecutive range of - /// top elements which were already sampled and narrow the sampling space to exclude them all . We do this by computing + /// top elements which were already sampled and narrow the sampling space to exclude them all. We do this by computing /// the prefix weight up to the top most item which wasn't sampled yet (inclusive) and then continue the sampling process /// over the narrowed space. This process is repeated until acquiring the desired mass. /// 4. Numerical stability. Naively, one would simply subtract `total_weight -= top.weight` in order to narrow the sampling diff --git a/rpc/wrpc/client/src/client.rs b/rpc/wrpc/client/src/client.rs index c65d582f0..e57024c4b 100644 --- a/rpc/wrpc/client/src/client.rs +++ b/rpc/wrpc/client/src/client.rs @@ -598,7 +598,6 @@ impl RpcApi for KaspaRpcClient { GetConnectedPeerInfo, GetCurrentNetwork, GetDaaScoreTimestampEstimate, - GetServerInfo, GetFeeEstimate, GetFeeEstimateExperimental, GetHeaders, @@ -606,13 +605,14 @@ impl RpcApi for KaspaRpcClient { GetMempoolEntries, GetMempoolEntriesByAddresses, GetMempoolEntry, - GetPeerAddresses, GetMetrics, + GetPeerAddresses, + GetServerInfo, GetSink, - GetSyncStatus, + GetSinkBlueScore, GetSubnetwork, + GetSyncStatus, GetUtxosByAddresses, - GetSinkBlueScore, GetVirtualChainFromBlock, Ping, ResolveFinalityConflict, diff --git a/rpc/wrpc/server/src/router.rs b/rpc/wrpc/server/src/router.rs index ff72cbdb3..09330eb49 100644 --- a/rpc/wrpc/server/src/router.rs +++ b/rpc/wrpc/server/src/router.rs @@ -46,7 +46,6 @@ impl Router { GetConnectedPeerInfo, GetCurrentNetwork, GetDaaScoreTimestampEstimate, - GetServerInfo, GetFeeEstimate, GetFeeEstimateExperimental, GetHeaders, @@ -55,13 +54,14 @@ impl Router { GetMempoolEntries, GetMempoolEntriesByAddresses, GetMempoolEntry, - GetPeerAddresses, GetMetrics, + GetPeerAddresses, + GetServerInfo, GetSink, + GetSinkBlueScore, GetSubnetwork, GetSyncStatus, GetUtxosByAddresses, - GetSinkBlueScore, GetVirtualChainFromBlock, Ping, ResolveFinalityConflict,