diff --git a/Cargo.lock b/Cargo.lock index 0d4703b8a..bcd372013 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3047,6 +3047,7 @@ dependencies = [ "secp256k1", "serde", "smallvec", + "sweep-bptree", "thiserror", "tokio", ] @@ -3406,6 +3407,7 @@ dependencies = [ name = "kaspa-utils" version = "0.14.1" dependencies = [ + "arc-swap", "async-channel 2.2.1", "async-trait", "bincode", @@ -5779,6 +5781,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "sweep-bptree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea7b1b7c5eaabc40bab84ec98b2f12523d97e91c9bfc430fe5d2a1ea15c9960" + [[package]] name = "syn" version = "1.0.109" diff --git a/cli/src/modules/rpc.rs b/cli/src/modules/rpc.rs index c84915480..53478c174 100644 --- a/cli/src/modules/rpc.rs +++ b/cli/src/modules/rpc.rs @@ -229,6 +229,15 @@ impl Rpc { } } } + RpcApiOps::GetFeeEstimate => { + let result = rpc.get_fee_estimate_call(GetFeeEstimateRequest {}).await?; + self.println(&ctx, result); + } + RpcApiOps::GetFeeEstimateExperimental => { + let verbose = if argv.is_empty() { false } else { argv.remove(0).parse().unwrap_or(false) }; + let result = rpc.get_fee_estimate_experimental_call(GetFeeEstimateExperimentalRequest { verbose }).await?; + self.println(&ctx, result); + } _ => { tprintln!(ctx, "rpc method exists but is not supported by the cli: '{op_str}'\r\n"); return Ok(()); diff --git a/consensus/core/src/config/params.rs b/consensus/core/src/config/params.rs index e2f2639a1..f512cf1e1 100644 --- a/consensus/core/src/config/params.rs +++ b/consensus/core/src/config/params.rs @@ -501,7 +501,8 @@ pub const SIMNET_PARAMS: Params = Params { target_time_per_block: Testnet11Bps::target_time_per_block(), past_median_time_sample_rate: Testnet11Bps::past_median_time_sample_rate(), difficulty_sample_rate: Testnet11Bps::difficulty_adjustment_sample_rate(), - max_block_parents: Testnet11Bps::max_block_parents(), + // For simnet, we deviate from TN11 configuration and allow at least 64 parents in order to support mempool benchmarks out of the box + max_block_parents: if Testnet11Bps::max_block_parents() > 64 { Testnet11Bps::max_block_parents() } else { 64 }, mergeset_size_limit: Testnet11Bps::mergeset_size_limit(), merge_depth: Testnet11Bps::merge_depth_bound(), finality_depth: Testnet11Bps::finality_depth(), diff --git a/kaspad/src/args.rs b/kaspad/src/args.rs index 2774269d3..56dd7c1de 100644 --- a/kaspad/src/args.rs +++ b/kaspad/src/args.rs @@ -134,7 +134,7 @@ impl Default for Args { #[cfg(feature = "devnet-prealloc")] prealloc_address: None, #[cfg(feature = "devnet-prealloc")] - prealloc_amount: 1_000_000, + prealloc_amount: 10_000_000_000, disable_upnp: false, disable_dns_seeding: false, diff --git a/kaspad/src/daemon.rs b/kaspad/src/daemon.rs index 0950ad8fa..ce4c19033 100644 --- a/kaspad/src/daemon.rs +++ b/kaspad/src/daemon.rs @@ -419,15 +419,16 @@ do you confirm? (answer y/n or pass --yes to the Kaspad command line to confirm let (address_manager, port_mapping_extender_svc) = AddressManager::new(config.clone(), meta_db, tick_service.clone()); - let mining_monitor = Arc::new(MiningMonitor::new(mining_counters.clone(), tx_script_cache_counters.clone(), tick_service.clone())); let mining_manager = MiningManagerProxy::new(Arc::new(MiningManager::new_with_extended_config( config.target_time_per_block, false, config.max_block_mass, config.ram_scale, config.block_template_cache_lifetime, - mining_counters, + mining_counters.clone(), ))); + let mining_monitor = + Arc::new(MiningMonitor::new(mining_manager.clone(), mining_counters, tx_script_cache_counters.clone(), tick_service.clone())); let flow_context = Arc::new(FlowContext::new( consensus_manager.clone(), diff --git a/mining/Cargo.toml b/mining/Cargo.toml index facd45d6a..0c7eb2525 100644 --- a/mining/Cargo.toml +++ b/mining/Cargo.toml @@ -27,8 +27,9 @@ parking_lot.workspace = true rand.workspace = true serde.workspace = true smallvec.workspace = true +sweep-bptree = "0.4.1" thiserror.workspace = true -tokio = { workspace = true, features = [ "rt-multi-thread", "macros", "signal" ] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "signal"] } [dev-dependencies] kaspa-txscript.workspace = true diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index 59ff685dd..16cfcc234 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -1,6 +1,16 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use kaspa_mining::model::topological_index::TopologicalIndex; -use std::collections::{hash_set::Iter, HashMap, HashSet}; +use itertools::Itertools; +use kaspa_consensus_core::{ + subnets::SUBNETWORK_ID_NATIVE, + tx::{Transaction, TransactionInput, TransactionOutpoint}, +}; +use kaspa_hashes::{HasherBase, TransactionID}; +use kaspa_mining::{model::topological_index::TopologicalIndex, FeerateTransactionKey, Frontier, Policy}; +use rand::{thread_rng, Rng}; +use std::{ + collections::{hash_set::Iter, HashMap, HashSet}, + sync::Arc, +}; #[derive(Default)] pub struct Dag @@ -68,5 +78,211 @@ pub fn bench_compare_topological_index_fns(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_compare_topological_index_fns); +fn generate_unique_tx(i: u64) -> Arc { + let mut hasher = TransactionID::new(); + let prev = hasher.update(i.to_le_bytes()).clone().finalize(); + let input = TransactionInput::new(TransactionOutpoint::new(prev, 0), vec![], 0, 0); + Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) +} + +fn build_feerate_key(fee: u64, mass: u64, id: u64) -> FeerateTransactionKey { + FeerateTransactionKey::new(fee, mass, generate_unique_tx(id)) +} + +pub fn bench_mempool_sampling(c: &mut Criterion) { + let mut rng = thread_rng(); + let mut group = c.benchmark_group("mempool sampling"); + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = if i % (cap as u64 / 100000) == 0 { 1000000 } else { rng.gen_range(1..10000) }; + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let len = cap; + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + group.bench_function("mempool one-shot sample", |b| { + b.iter(|| { + black_box({ + let selected = frontier.sample_inplace(&mut rng, &Policy::new(500_000), &mut 0); + selected.iter().map(|k| k.mass).sum::() + }) + }) + }); + + // Benchmark frontier insertions and removals (see comparisons below) + let remove = map.values().take(map.len() / 10).cloned().collect_vec(); + group.bench_function("frontier remove/add", |b| { + b.iter(|| { + black_box({ + for r in remove.iter() { + frontier.remove(r).then_some(()).unwrap(); + } + for r in remove.iter().cloned() { + frontier.insert(r).then_some(()).unwrap(); + } + 0 + }) + }) + }); + + // Benchmark hashmap insertions and removals for comparison + let remove = map.iter().take(map.len() / 10).map(|(&k, v)| (k, v.clone())).collect_vec(); + group.bench_function("map remove/add", |b| { + b.iter(|| { + black_box({ + for r in remove.iter() { + map.remove(&r.0).unwrap(); + } + for r in remove.iter().cloned() { + map.insert(r.0, r.1.clone()); + } + 0 + }) + }) + }); + + // Benchmark std btree set insertions and removals for comparison + // Results show that frontier (sweep bptree) and std btree set are roughly the same. + // The slightly higher cost for sweep bptree should be attributed to subtree weight + // maintenance (see FeerateWeight) + #[allow(clippy::mutable_key_type)] + let mut std_btree = std::collections::BTreeSet::from_iter(map.values().cloned()); + let remove = map.iter().take(map.len() / 10).map(|(&k, v)| (k, v.clone())).collect_vec(); + group.bench_function("std btree remove/add", |b| { + b.iter(|| { + black_box({ + for (_, key) in remove.iter() { + std_btree.remove(key).then_some(()).unwrap(); + } + for (_, key) in remove.iter() { + std_btree.insert(key.clone()); + } + 0 + }) + }) + }); + group.finish(); +} + +pub fn bench_mempool_selectors(c: &mut Criterion) { + let mut rng = thread_rng(); + let mut group = c.benchmark_group("mempool selectors"); + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = rng.gen_range(1..1000000); + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + for len in [100, 300, 350, 500, 1000, 2000, 5000, 10_000, 100_000, 500_000, 1_000_000].into_iter().rev() { + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + group.bench_function(format!("rebalancing selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_rebalancing_selector(); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + let mut collisions = 0; + let mut n = 0; + + group.bench_function(format!("sample inplace selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_sample_inplace(&mut collisions); + n += 1; + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + if n > 0 { + println!("---------------------- \n Avg collisions: {}", collisions / n); + } + + if frontier.total_mass() <= 500_000 { + group.bench_function(format!("take all selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_take_all(); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + } + + group.bench_function(format!("dynamic selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector(&Policy::new(500_000)); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + } + + group.finish(); +} + +pub fn bench_inplace_sampling_worst_case(c: &mut Criterion) { + let mut group = c.benchmark_group("mempool inplace sampling"); + let max_fee = u64::MAX; + let fee_steps = (0..10).map(|i| max_fee / 100u64.pow(i)).collect_vec(); + for subgroup_size in [300, 200, 100, 80, 50, 30] { + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = if i < 300 { fee_steps[i as usize / subgroup_size] } else { 1 }; + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let mut frontier = Frontier::default(); + for item in map.values().cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let mut collisions = 0; + let mut n = 0; + + group.bench_function(format!("inplace sampling worst case (subgroup size: {})", subgroup_size), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_sample_inplace(&mut collisions); + n += 1; + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + if n > 0 { + println!("---------------------- \n Avg collisions: {}", collisions / n); + } + } + + group.finish(); +} + +criterion_group!( + benches, + bench_mempool_sampling, + bench_mempool_selectors, + bench_inplace_sampling_worst_case, + bench_compare_topological_index_fns +); criterion_main!(benches); diff --git a/mining/src/block_template/builder.rs b/mining/src/block_template/builder.rs index de3428a74..e111b2562 100644 --- a/mining/src/block_template/builder.rs +++ b/mining/src/block_template/builder.rs @@ -1,25 +1,18 @@ -use super::{errors::BuilderResult, policy::Policy}; -use crate::{block_template::selector::TransactionsSelector, model::candidate_tx::CandidateTransaction}; +use super::errors::BuilderResult; use kaspa_consensus_core::{ api::ConsensusApi, - block::{BlockTemplate, TemplateBuildMode}, + block::{BlockTemplate, TemplateBuildMode, TemplateTransactionSelector}, coinbase::MinerData, merkle::calc_hash_merkle_root, tx::COINBASE_TRANSACTION_INDEX, }; -use kaspa_core::{ - debug, - time::{unix_now, Stopwatch}, -}; +use kaspa_core::time::{unix_now, Stopwatch}; -pub(crate) struct BlockTemplateBuilder { - policy: Policy, -} +pub(crate) struct BlockTemplateBuilder {} impl BlockTemplateBuilder { - pub(crate) fn new(max_block_mass: u64) -> Self { - let policy = Policy::new(max_block_mass); - Self { policy } + pub(crate) fn new() -> Self { + Self {} } /// BuildBlockTemplate creates a block template for a miner to consume @@ -89,12 +82,10 @@ impl BlockTemplateBuilder { &self, consensus: &dyn ConsensusApi, miner_data: &MinerData, - transactions: Vec, + selector: Box, build_mode: TemplateBuildMode, ) -> BuilderResult { let _sw = Stopwatch::<20>::with_threshold("build_block_template op"); - debug!("Considering {} transactions for a new block template", transactions.len()); - let selector = Box::new(TransactionsSelector::new(self.policy.clone(), transactions)); Ok(consensus.build_block_template(miner_data.clone(), selector, build_mode)?) } diff --git a/mining/src/block_template/policy.rs b/mining/src/block_template/policy.rs index ff5197255..12ee98e28 100644 --- a/mining/src/block_template/policy.rs +++ b/mining/src/block_template/policy.rs @@ -1,14 +1,14 @@ /// Policy houses the policy (configuration parameters) which is used to control /// the generation of block templates. See the documentation for -/// NewBlockTemplate for more details on each of these parameters are used. +/// NewBlockTemplate for more details on how each of these parameters are used. #[derive(Clone)] -pub(crate) struct Policy { +pub struct Policy { /// max_block_mass is the maximum block mass to be used when generating a block template. pub(crate) max_block_mass: u64, } impl Policy { - pub(crate) fn new(max_block_mass: u64) -> Self { + pub fn new(max_block_mass: u64) -> Self { Self { max_block_mass } } } diff --git a/mining/src/block_template/selector.rs b/mining/src/block_template/selector.rs index a55ecb93d..6acacb22d 100644 --- a/mining/src/block_template/selector.rs +++ b/mining/src/block_template/selector.rs @@ -18,7 +18,7 @@ use kaspa_consensus_core::{ /// candidate transactions should be. A smaller alpha makes the distribution /// more uniform. ALPHA is used when determining a candidate transaction's /// initial p value. -const ALPHA: i32 = 3; +pub(crate) const ALPHA: i32 = 3; /// REBALANCE_THRESHOLD is the percentage of candidate transactions under which /// we don't rebalance. Rebalancing is a heavy operation so we prefer to avoid @@ -28,7 +28,7 @@ const ALPHA: i32 = 3; /// if REBALANCE_THRESHOLD is 0.95, there's a 1-in-20 chance of collision. const REBALANCE_THRESHOLD: f64 = 0.95; -pub(crate) struct TransactionsSelector { +pub struct RebalancingWeightedTransactionSelector { policy: Policy, /// Transaction store transactions: Vec, @@ -52,8 +52,8 @@ pub(crate) struct TransactionsSelector { gas_usage_map: HashMap, } -impl TransactionsSelector { - pub(crate) fn new(policy: Policy, mut transactions: Vec) -> Self { +impl RebalancingWeightedTransactionSelector { + pub fn new(policy: Policy, mut transactions: Vec) -> Self { let _sw = Stopwatch::<100>::with_threshold("TransactionsSelector::new op"); // Sort the transactions by subnetwork_id. transactions.sort_by(|a, b| a.tx.subnetwork_id.cmp(&b.tx.subnetwork_id)); @@ -103,7 +103,7 @@ impl TransactionsSelector { /// select_transactions loops over the candidate transactions /// and appends the ones that will be included in the next block into /// selected_txs. - pub(crate) fn select_transactions(&mut self) -> Vec { + pub fn select_transactions(&mut self) -> Vec { let _sw = Stopwatch::<15>::with_threshold("select_transaction op"); let mut rng = rand::thread_rng(); @@ -225,7 +225,7 @@ impl TransactionsSelector { } } -impl TemplateTransactionSelector for TransactionsSelector { +impl TemplateTransactionSelector for RebalancingWeightedTransactionSelector { fn select_transactions(&mut self) -> Vec { self.select_transactions() } @@ -269,7 +269,13 @@ mod tests { use kaspa_txscript::{pay_to_script_hash_signature_script, test_helpers::op_true_script}; use std::{collections::HashSet, sync::Arc}; - use crate::{mempool::config::DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, model::candidate_tx::CandidateTransaction}; + use crate::{ + mempool::{ + config::DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, + model::frontier::selectors::{SequenceSelector, SequenceSelectorInput, SequenceSelectorTransaction}, + }, + model::candidate_tx::CandidateTransaction, + }; #[test] fn test_reject_transaction() { @@ -277,29 +283,43 @@ mod tests { // Create a vector of transactions differing by output value so they have unique ids let transactions = (0..TX_INITIAL_COUNT).map(|i| create_transaction(SOMPI_PER_KASPA * (i + 1) as u64)).collect_vec(); + let masses: HashMap<_, _> = transactions.iter().map(|tx| (tx.tx.id(), tx.calculated_mass)).collect(); + let sequence: SequenceSelectorInput = + transactions.iter().map(|tx| SequenceSelectorTransaction::new(tx.tx.clone(), tx.calculated_mass)).collect(); + let policy = Policy::new(100_000); - let mut selector = TransactionsSelector::new(policy, transactions); - let (mut kept, mut rejected) = (HashSet::new(), HashSet::new()); - let mut reject_count = 32; - for i in 0..10 { - let selected_txs = selector.select_transactions(); - if i > 0 { - assert_eq!( - selected_txs.len(), - reject_count, - "subsequent select calls are expected to only refill the previous rejections" - ); - reject_count /= 2; - } - for tx in selected_txs.iter() { - kept.insert(tx.id()).then_some(()).expect("selected txs should never repeat themselves"); - assert!(!rejected.contains(&tx.id()), "selected txs should never repeat themselves"); + let selectors: [Box; 2] = [ + Box::new(RebalancingWeightedTransactionSelector::new(policy.clone(), transactions)), + Box::new(SequenceSelector::new(sequence, policy.clone())), + ]; + + for mut selector in selectors { + let (mut kept, mut rejected) = (HashSet::new(), HashSet::new()); + let mut reject_count = 32; + let mut total_mass = 0; + for i in 0..10 { + let selected_txs = selector.select_transactions(); + if i > 0 { + assert_eq!( + selected_txs.len(), + reject_count, + "subsequent select calls are expected to only refill the previous rejections" + ); + reject_count /= 2; + } + for tx in selected_txs.iter() { + total_mass += masses[&tx.id()]; + kept.insert(tx.id()).then_some(()).expect("selected txs should never repeat themselves"); + assert!(!rejected.contains(&tx.id()), "selected txs should never repeat themselves"); + } + assert!(total_mass <= policy.max_block_mass); + selected_txs.iter().take(reject_count).for_each(|x| { + total_mass -= masses[&x.id()]; + selector.reject_selection(x.id()); + kept.remove(&x.id()).then_some(()).expect("was just inserted"); + rejected.insert(x.id()).then_some(()).expect("was just verified"); + }); } - selected_txs.iter().take(reject_count).for_each(|x| { - selector.reject_selection(x.id()); - kept.remove(&x.id()).then_some(()).expect("was just inserted"); - rejected.insert(x.id()).then_some(()).expect("was just verified"); - }); } } diff --git a/mining/src/feerate/fee_estimation.ipynb b/mining/src/feerate/fee_estimation.ipynb new file mode 100644 index 000000000..694f47450 --- /dev/null +++ b/mining/src/feerate/fee_estimation.ipynb @@ -0,0 +1,496 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Feerates\n", + "\n", + "The feerate value represents the fee/mass ratio of a transaction in `sompi/gram` units.\n", + "Given a feerate value recommendation, one should calculate the required fee by taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)`. \n", + "\n", + "This notebook makes an effort to implement and illustrate the feerate estimator method we used. The corresponding Rust implementation is more comprehensive and addresses some additional edge cases, but the code in this notebook highly reflects it." + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "metadata": {}, + "outputs": [], + "source": [ + "feerates = [1.0, 1.1, 1.2]*10 + [1.5]*3000 + [2]*3000 + [2.1]*3000 + [3, 4, 5]*10\n", + "# feerates = [1.0, 1.1, 1.2] + [1.1]*100 + [1.2]*100 + [1.3]*100 # + [3, 4, 5, 100]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We compute the probability weight of each transaction by raising `feerate` to the power of `alpha` (currently set to `3`). Essentially, alpha represents the amount of bias we want towards higher feerate transactions. " + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "metadata": {}, + "outputs": [], + "source": [ + "ALPHA = 3.0" + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total mempool weight: 64108.589999995806\n" + ] + } + ], + "source": [ + "total_weight = sum(np.array(feerates)**ALPHA)\n", + "print('Total mempool weight: ', total_weight)" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "metadata": {}, + "outputs": [], + "source": [ + "avg_mass = 2000\n", + "bps = 1\n", + "block_mass_limit = 500_000\n", + "network_mass_rate = bps * block_mass_limit" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inclusion interval: 0.004\n" + ] + } + ], + "source": [ + "print('Inclusion interval: ', avg_mass/network_mass_rate)" + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "metadata": {}, + "outputs": [], + "source": [ + "class FeerateBucket:\n", + " def __init__(self, feerate, estimated_seconds):\n", + " self.feerate = feerate\n", + " self.estimated_seconds = estimated_seconds\n", + " \n", + "\n", + "class FeerateEstimations:\n", + " def __init__(self, low_bucket, mid_bucket, normal_bucket, priority_bucket):\n", + " self.low_bucket = low_bucket \n", + " self.mid_bucket = mid_bucket \n", + " self.normal_bucket = normal_bucket\n", + " self.priority_bucket = priority_bucket\n", + " \n", + " def __repr__(self):\n", + " return 'Feerates:\\t{}, {}, {}, {} \\nTimes:\\t\\t{}, {}, {}, {}'.format(\n", + " self.low_bucket.feerate, \n", + " self.mid_bucket.feerate, \n", + " self.normal_bucket.feerate,\n", + " self.priority_bucket.feerate, \n", + " self.low_bucket.estimated_seconds, \n", + " self.mid_bucket.estimated_seconds, \n", + " self.normal_bucket.estimated_seconds, \n", + " self.priority_bucket.estimated_seconds)\n", + " def feerates(self):\n", + " return np.array([\n", + " self.low_bucket.feerate, \n", + " self.mid_bucket.feerate, \n", + " self.normal_bucket.feerate,\n", + " self.priority_bucket.feerate\n", + " ])\n", + " \n", + " def times(self):\n", + " return np.array([\n", + " self.low_bucket.estimated_seconds, \n", + " self.mid_bucket.estimated_seconds, \n", + " self.normal_bucket.estimated_seconds,\n", + " self.priority_bucket.estimated_seconds\n", + " ])\n", + " \n", + "class FeerateEstimator:\n", + " \"\"\"\n", + " `total_weight`: The total probability weight of all current mempool ready \n", + " transactions, i.e., Σ_{tx in mempool}(tx.fee/tx.mass)^ALPHA\n", + " \n", + " 'inclusion_interval': The amortized time between transactions given the current \n", + " transaction masses present in the mempool, i.e., the inverse \n", + " of the transaction inclusion rate. For instance, if the average \n", + " transaction mass is 2500 grams, the block mass limit is 500,000\n", + " and the network has 10 BPS, then this number would be 1/2000 seconds.\n", + " \"\"\"\n", + " def __init__(self, total_weight, inclusion_interval):\n", + " self.total_weight = total_weight\n", + " self.inclusion_interval = inclusion_interval\n", + "\n", + " \"\"\"\n", + " Feerate to time function: f(feerate) = inclusion_interval * (1/p(feerate))\n", + " where p(feerate) = feerate^ALPHA/(total_weight + feerate^ALPHA) represents \n", + " the probability function for drawing `feerate` from the mempool\n", + " in a single trial. The inverse 1/p is the expected number of trials until\n", + " success (with repetition), thus multiplied by inclusion_interval it provides an\n", + " approximation to the overall expected waiting time\n", + " \"\"\"\n", + " def feerate_to_time(self, feerate):\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", + " return c1 * c2 / feerate**ALPHA + c1\n", + "\n", + " \"\"\"\n", + " The inverse function of `feerate_to_time`\n", + " \"\"\"\n", + " def time_to_feerate(self, time):\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", + " return ((c1 * c2 / time) / (1 - c1 / time))**(1 / ALPHA)\n", + " \n", + " \"\"\"\n", + " The antiderivative function of \n", + " feerate_to_time excluding the constant shift `+ c1`\n", + " \"\"\"\n", + " def feerate_to_time_antiderivative(self, feerate):\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", + " return c1 * c2 / (-2.0 * feerate**(ALPHA - 1))\n", + " \n", + " \"\"\"\n", + " Returns the feerate value for which the integral area is `frac` of the total area.\n", + " See figures below for illustration\n", + " \"\"\"\n", + " def quantile(self, lower, upper, frac):\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", + " z1 = self.feerate_to_time_antiderivative(lower)\n", + " z2 = self.feerate_to_time_antiderivative(upper)\n", + " z = frac * z2 + (1.0 - frac) * z1\n", + " return ((c1 * c2) / (-2 * z))**(1.0 / (ALPHA - 1.0))\n", + " \n", + " def calc_estimations(self):\n", + " # Choose `high` such that it provides sub-second waiting time\n", + " high = self.time_to_feerate(1.0)\n", + " \n", + " # Choose `low` feerate such that it provides sub-hour waiting time AND it covers (at least) the 0.25 quantile\n", + " low = max(self.time_to_feerate(3600.0), self.quantile(1.0, high, 0.25))\n", + " \n", + " # Choose `normal` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.66\n", + " # quantile between low and high\n", + " normal = max(self.time_to_feerate(60.0), self.quantile(low, high, 0.66))\n", + " \n", + " # Choose an additional point between normal and low\n", + " mid = max(self.time_to_feerate(1800.0), self.quantile(1.0, high, 0.5))\n", + " \n", + " return FeerateEstimations(\n", + " FeerateBucket(low, self.feerate_to_time(low)),\n", + " FeerateBucket(mid, self.feerate_to_time(mid)),\n", + " FeerateBucket(normal, self.feerate_to_time(normal)),\n", + " FeerateBucket(high, self.feerate_to_time(high)))" + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0" + ] + }, + "execution_count": 104, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "estimator = FeerateEstimator(total_weight=0, inclusion_interval=1/100)\n", + "# estimator.quantile(2, 3, 0.5)\n", + "estimator.time_to_feerate(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Feerate estimation\n", + "\n", + "The figure below illustrates the estimator selection. We first estimate the `feerate_to_time` function and then select 3 meaningfull points by analyzing the curve and its integral (see `calc_estimations`). " + ] + }, + { + "cell_type": "code", + "execution_count": 105, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHVlJREFUeJzt3XuQnHW95/H3t+8990lmMplkQhIwoIAJlxj14PFwRNR4OcA5LoVb60HLLU4VeNTaU7Wl7JaHtWTX2rPKrq5yFgTFEnVTikcUvLABBY5ASBASyIUk5DKT20xuc0vm0tPf/aOfCZNkkpnMdOeZfvrzqup6nufXTz/9bS6f3zO//j1Pm7sjIiLRFQu7ABERKS0FvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4hT0IiIRp6AXEYm4RNgFADQ1NfmiRYvCLkNEpKysW7fuoLs3T7TfjAj6RYsWsXbt2rDLEBEpK2a2azL7aehGRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYgr66Dfsr+Xf/rtZo4eGwq7FBGRGausg37XoX6+/dR22g8fD7sUEZEZa8KgN7MFZvaUmW0ys9fM7PNB+11mtsfMXg4eHx7zmi+Z2TYz22JmHyxV8S11GQAO9AyU6i1ERMreZG6BkAP+wd1fMrNaYJ2ZPRE8d4+7/4+xO5vZpcAtwGXAPOD/mdnF7j5SzMJhTND3KuhFRM5kwjN6d9/n7i8F673AJmD+WV5yA/ATdx909x3ANmBFMYo9VVNNCgMO9AyW4vAiIpFwTmP0ZrYIuBJ4IWj6rJmtN7MHzawxaJsPtI95WQdn7ximLBGP0VCVpFNDNyIiZzTpoDezGuBnwBfcvQe4F7gIuALYB3x9dNdxXu7jHO82M1trZmu7urrOufBR9dmkxuhFRM5iUkFvZkkKIf+wuz8C4O4H3H3E3fPA/bw5PNMBLBjz8jZg76nHdPf73H25uy9vbp7wdspnVJ9Nsl9BLyJyRpOZdWPAA8Amd//GmPbWMbvdBLwarD8K3GJmaTNbDCwB1hSv5JM1VCU1Ri8ichaTmXVzDfBJYIOZvRy03Ql8wsyuoDAssxP4OwB3f83MVgEbKczYuaMUM25GNVSlONw/xFAuTypR1pcFiIiUxIRB7+7PMv64++Nnec3dwN3TqGvSGqqSAHT1DTK/IXs+3lJEpKyU/SlwYxD0+kJWRGR8ZR/0DdkUgKZYioicQfkH/Ykzen0hKyIynrIP+ppMgphp6EZE5EzKPuhjZtSkEzqjFxE5g7IPeoDqdIJO3dhMRGRckQj6qlScfUcV9CIi44lE0FenE7oNgojIGUQi6GvSCfoGcxwbyoVdiojIjBOJoK/NFC7w3avhGxGR00Qj6NOFufR7j+q3Y0VEThWNoD9xRq+gFxE5VSSCvjqdwFDQi4iMJxJBH48ZtZkEe7s1Ri8icqpIBD0UboWgM3oRkdNFJ+hTCfYcUdCLiJwqOkGfSbCvewD3036HXESkokUm6GszSYZG8hzqHwq7FBGRGSVCQa8pliIi44lO0Kd1dayIyHiiE/QZXR0rIjKeyAR9JhkjGTcFvYjIKSIT9GZGXSbJPl00JSJyksgEPUB1Ok7HkWNhlyEiMqNEKuhrM0k6dNGUiMhJIhX0ddkkh/qH9AMkIiJjRCro64OZNzqrFxF5U7SCPlsI+t2HNE4vIjIqUkFfly1cNNWuL2RFRE6IVNBnk3FSiRi7DyvoRURGTRj0ZrbAzJ4ys01m9pqZfT5on2VmT5jZ1mDZGLSbmX3TzLaZ2Xozu6rUH2JMrdRnk7Qr6EVETpjMGX0O+Ad3fxvwLuAOM7sU+CKw2t2XAKuDbYCVwJLgcRtwb9GrPovadEJn9CIiY0wY9O6+z91fCtZ7gU3AfOAG4KFgt4eAG4P1G4AfeMHzQIOZtRa98jOoyybZffiY7ksvIhI4pzF6M1sEXAm8ALS4+z4odAbAnGC3+UD7mJd1BG2nHus2M1trZmu7urrOvfIzqM8mGRjWfelFREZNOujNrAb4GfAFd+85267jtJ12eu3u97n7cndf3tzcPNkyJjQ680bDNyIiBZMKejNLUgj5h939kaD5wOiQTLDsDNo7gAVjXt4G7C1OuRMbvWhKX8iKiBRMZtaNAQ8Am9z9G2OeehS4NVi/FfjFmPa/DWbfvAvoHh3iOR/qsgp6EZGxEpPY5xrgk8AGM3s5aLsT+Bqwysw+A+wG/k3w3OPAh4FtwDHg00WteALJeIwazbwRETlhwqB392cZf9wd4Lpx9nfgjmnWNS11mQS7dBsEEREgYlfGjqqvSvLGwf6wyxARmREiGfSNVSm6egfpH9TtikVEIhn0DVWFL2R36KxeRCSaQd9YlQIU9CIiENGgb8jqjF5EZFQkgz4Rj1GfTSroRUSIaNBD4Z43b3T1hV2GiEjoIhv0DdnCFEvdxVJEKl10g74qSe9AjsO6i6WIVLjIBr1m3oiIFEQ26Efn0usKWRGpdJEN+rpMkriZzuhFpOJFNuhjMaOhKsm2Ts28EZHKFtmgB2isTrFlf2/YZYiIhCrSQT+7OkX74WMMDI+EXYqISGgiHfQfHHmaZ1KfI333bLjncli/KuySRETOu8n8wlRZuqTz17y/6+ukYoOFhu52+OXnCutLbw6vMBGR8yyyZ/Tv2f0dUj54cuPwcVj9lXAKEhEJSWSDvnbwwPhPdHec30JEREIW2aDvTbeM/0R92/ktREQkZJEN+mcvuJ3hWObkxmQWrvtyOAWJiIQksl/GbpmzEoB37vjfNA53kaudR+oDd+mLWBGpOJENeiiE/R+r3scPX9jNPTcu46alGrYRkcoT2aGbUQ1VKeJmbNmvWyGISGWKfNDHY8bsmhSb9/eEXYqISCgiH/QAs2tSvLqnO+wyRERCURFB31yT5mDfEJ29A2GXIiJy3lVG0NemAdi0T3eyFJHKUxlBX1MI+tf2avhGRCrPhEFvZg+aWaeZvTqm7S4z22NmLwePD4957ktmts3MtpjZB0tV+LlIJ+M0ZJNs3KsvZEWk8kzmjP77wIfGab/H3a8IHo8DmNmlwC3AZcFrvmNm8WIVOx2za1K8pqAXkQo0YdC7+9PA4Uke7wbgJ+4+6O47gG3AimnUVzTNNWl2HuynfzAXdikiIufVdMboP2tm64OhncagbT7QPmafjqDtNGZ2m5mtNbO1XV1d0yhjcppr0ziwWT8tKCIVZqpBfy9wEXAFsA/4etBu4+zr4x3A3e9z9+Xuvry5uXmKZUze6Mybjfs0fCMilWVKQe/uB9x9xN3zwP28OTzTASwYs2sbsHd6JRZHTTpBVSrOho6jYZciInJeTSnozax1zOZNwOiMnEeBW8wsbWaLgSXAmumVWBxmxpzaNC+3K+hFpLJMePdKM/sxcC3QZGYdwD8C15rZFRSGZXYCfwfg7q+Z2SpgI5AD7nD3kdKUfu5a6jKs2XGYvsEcNelI37hTROSECdPO3T8xTvMDZ9n/buDu6RRVKnPrMjjw6p5u3nXh7LDLERE5LyriythRLXWFX5x6RcM3IlJBKiros6k4jVVJjdOLSEWpqKCHwjTLPynoRaSCVFzQz63LsL97gM4e3bJYRCpDxQX96Di9hm9EpFJUXNDPqU0TMwW9iFSOigv6RDzGnNoML+6c7H3aRETKW8UFPcC8hgwvtx9lYHjGXMslIlIyFRr0WYZHnPUd+sUpEYm+ig16QMM3IlIRKjLos8k4TTUp1uxQ0ItI9FVk0APMrc+wbtcRRvLj3i5fRCQyKjbo5zdk6RvMsUk/RCIiEVexQa9xehGpFBUb9HWZJPXZJM9tPxR2KSIiJVWxQQ/Q1pjlj9sPkRvJh12KiEjJVHTQXzCrir7BHBv2aD69iERXRQd9W2NhnP7ZrQdDrkREpHQqOuirUgnm1KV5dpuCXkSiq6KDHmBBQxXrdh3h2FAu7FJEREpCQT8rSy7vvKCrZEUkoio+6Oc3ZEnETOP0IhJZFR/0iXiM+Q1ZntzcGXYpIiIlUfFBD7CoqZodB/vZcbA/7FJERIpOQQ9c2FQNwOpNB0KuRESk+BT0QF02SXNNmtWbNHwjItGjoA8snF3Fmp2H6T4+HHYpIiJFpaAPLG6qZiTvPP16V9iliIgUlYI+MLc+Q1UqrnF6EYmcCYPezB40s04ze3VM2ywze8LMtgbLxqDdzOybZrbNzNab2VWlLL6YYmYsml3NE5sOMJgbCbscEZGimcwZ/feBD53S9kVgtbsvAVYH2wArgSXB4zbg3uKUeX4saamhf3CEZ17XxVMiEh0TBr27Pw2cen+AG4CHgvWHgBvHtP/AC54HGsystVjFltqCxiqyyTiPbdgXdikiIkUz1TH6FnffBxAs5wTt84H2Mft1BG2nMbPbzGytma3t6poZX4DGY8bipmqe2HiAgWEN34hINBT7y1gbp83H29Hd73P35e6+vLm5uchlTN3FLTX0DeZ4Rve+EZGImGrQHxgdkgmWo1cadQALxuzXBuydennnX1tjFR9P/ZGrH/lzuKsB7rkc1q8KuywRkSmbatA/CtwarN8K/GJM+98Gs2/eBXSPDvGUi0sP/oavxu5nVu4A4NDdDr/8nMJeRMrWZKZX/hh4DrjEzDrM7DPA14DrzWwrcH2wDfA48AawDbgfuL0kVZfQe3Z/hwyDJzcOH4fVXwmnIBGRaUpMtIO7f+IMT103zr4O3DHdosJUO3iGC6a6O85vISIiRaIrY0/Rm24Z/4n6tvNbiIhIkSjoT/HsBbczHMuc3JjMwnVfDqcgEZFpmnDoptJsmbMSKIzV1wweoNOaaPnYf8WW3hxyZSIiU6OgH8eWOSvZMmclm/b18LuNB3i46p1cE3ZRIiJTpKGbs1gyp4aqVJwfPLcz7FJERKZMQX8WiXiMt7XW8cTGA+w9ejzsckREpkRBP4Gl8+txhx+9sDvsUkREpkRBP4G6bJLFTdX8aM1u3adeRMqSgn4SlrbVc7h/iF+9UlZ3cxARART0k3LBrCqaalL88x+2k8+PezNOEZEZS0E/CWbG1Rc0srWzj6e2dE78AhGRGURBP0lLWmqpzya59/fbwy5FROScKOgnKR4zrljQwNpdR3hx56m/rCgiMnMp6M/BZfPqqErF+dbqrWGXIiIyaQr6c5CMx7jqgkae3nqQNTt0Vi8i5UFBf46WttVTk07wT7/dTOH2+yIiM5uC/hwl4zGWL2rkxZ1HeFo/IC4iZUBBPwWXz6unPpvkv/9ms+bVi8iMp6CfgnjMeOfiWby2t4dH/rQn7HJERM5KQT9Fb51bS2t9hq/9ehN9g7mwyxEROSMF/RSZGe9d0szBviG+9aSmW4rIzKWgn4a59Rkuba3lgWd2sONgf9jliIiMS0E/TX92URPxmPGln63XdEsRmZEU9NNUnU7wnrc08fyOw/zkxfawyxEROY2Cvggum1fHgsYsdz+2if3dA2GXIyJyEgV9EZgZ73vrHAZzI9z58w0awhGRGUVBXyQNVSnefeFsntzcyQ/1+7IiMoMo6IvoigUNLJpdxVd/tZEt+3vDLkdEBFDQF5WZ8f63tZCIG3//45cYGNaPiYtI+KYV9Ga208w2mNnLZrY2aJtlZk+Y2dZg2VicUstDdTrB9W9r4fUDffznf3lV4/UiErpinNH/pbtf4e7Lg+0vAqvdfQmwOtiuKAtnV7Ni8Sx+uq6DHzy3K+xyRKTClWLo5gbgoWD9IeDGErzHjPeuxbO4sKmar/xyI89tPxR2OSJSwaYb9A78zszWmdltQVuLu+8DCJZzpvkeZcnM+MBlLTRUJbn94XW6RYKIhGa6QX+Nu18FrATuMLP3TvaFZnabma01s7VdXV3TLGNmSififGRpK4O5PJ984AU6e3UxlYicf9MKenffGyw7gZ8DK4ADZtYKECw7z/Da+9x9ubsvb25unk4ZM1pjVYqPLZtHZ88gtz64ht6B4bBLEpEKM+WgN7NqM6sdXQc+ALwKPArcGux2K/CL6RZZ7ubWZfjw2+eyZX8v//6htRwb0v3rReT8mc4ZfQvwrJm9AqwBHnP33wBfA643s63A9cF2xVs4u5oPXDqXNTsP86kHX6RfP1YiIudJYqovdPc3gGXjtB8CrptOUVF1ydxaAH67cT+f+t4avv/pFVSnp/yvQERkUnRl7Hl2ydxaPnTZXNbtOsK/vf95DvUNhl2SiEScgj4EF7fU8uG3t/La3h7++jt/ZNchTb0UkdJR0IfkouYa/vqq+XT1DXLjt/+VHU99D+65HO5qKCzXrwq7RBGJCA0Qh6i1PsvHr2pj+OX/S8vv/xlsqPBEdzv88nOF9aU3h1egiESCzuhD1lid4oupVVSNhvyo4eOw+ivhFCUikaKgnwHqhg6M/0R3x/ktREQiSUE/A/SmW8Zt78vM1W2ORWTaFPQzwLMX3M5wLHNS2wBp7uy5iU9970XaDx8LqTIRiQIF/QywZc5KnrjoTnrSc3GMnvRcVi/5T/S85Sae236I6+/5A//nD9sZHsmHXaqIlCHNupkhtsxZyZY5K09qWwZc2FzNH17v4r/9ejOP/GkPd33sMt590exwihSRsqQz+hmuNpPko0vn8dGlrew7epxP3P88n3noRbZ16sfHRWRydEZfJi5qrmHhrCpebj/Ks1sP8sHNz3DzOxZwx19eRFtjVdjlicgMpqAvI4l4jOWLZnHpvDrW7DjMqhfbWbW2nb+5aj63X/sWFjVVh12iiMxACvoyVJVKcO0lc7h6YSPrdh3hkZf28NN1HXx06Tw+fc0irrygMewSRWQGUdCXsdpMkmsvmcM7Fs3ipd1H+O1r+3n0lb0snV/Pp65ZxEeWtpJOxMMuU0RCZjPhgpzly5f72rVrp/TaJzcf4JX27iJXVJ6Gcnk27eth/Z5uDvcP0ViV5KYr2/j41W1cOq8u7PJEpMjMbJ27L59oP53RR0gqEWPZggaWttWz+/AxXt3bw0PP7eTBf93BW+fW8vGr2/irZfOYU5eZ8FgiEh0K+ggyMxbOrmbh7GqOD4/w+v5eNu/v5auPbeLuxzZx1cJGVl4+lw9eNpcFszRjRyTqFPQRl03GWbaggWULGjjcP8S2zj62d/Xx1cc28dXHNnH5vDre97YW/uLiZpa11ZOI69IKkahR0FeQWdUpViyexYrFszh6bIjtXf1s7+rjW09u5Zurt1KbSfDnS5r4i4ub+bOLmmhrzGJmYZctItOkoK9QDVUprl6Y4uqFjQwMj7D78DF2HTrGM68f5PEN+wGYW5dhxeJZvGPxLFYsmsWSOTXEYgp+kXKjoBcyyTgXt9RycUst7s6h/iH2HDnOnqPHeWpzJ4++sheA+mySZW31LG1r4O1t9Sxtq2duXUZn/SIznIJeTmJmNNWkaapJs2xBA+5Oz0COPUePs/focTbv7+XZbQfJB7NyZ1WnWNZWz+Xz61nSUsslLbUsbqomldBYv8hMoaCXszIz6rNJ6rNJLm0tzMXPjeTp6huks2eQA70DbNjTzR9e7zoR/vGYsbipmkuCvxLeMqeGRU1VLJxdTU1a/8mJnG/6v07OWSIeo7U+S2t99kRbbiTPkWPDHOof5HD/EIf6hvjj9oM8vmEfYy/Jm12dYnFTYernotlVLGyqZuGsKlobMjRVp/UdgEgJKOilKBLxGM21aZpr0ye1D4/kOXpsmKPHhjh6fJju48Ps7xng9QO99AzkTto3GTda67PMa8gwryHLvPos8xqytDZkaK3P0FyTprEqpc5A5Bwp6KWkkmfoAKDQCXQH4d83kKN3MEfvwDDth4+xcW8PfYO5E8NBo+JmzK5JnThmc01h2TRm2VidpLEqRUNVUvf6EUFBLyFKxmMnvvgdTz7v9A/l6B3I0TeY49jQCMeGcvQPjtA/mONQ3xAvDR0Zt0MYlU3GaagqBH9jdZKGqhSNwXZ9trBdm0kUHukktZkENcG2OgmJCgW9zFixmFGbSVKbSZ51P3dnYDhf6ASGRhgYHn3kT6wfG8px5NgQg7leBoZHOD40wkS380vFY1Sn49RmktRlT+4IatIJsqk41akEVak42VS8sEwWtt9sG/N8Mq4rjyUUJQt6M/sQ8L+AOPBdd/9aqd5LKpuZkQ3CdLK/puvuDOYKHcFQLs9gLs/QSJ6hXP7N7VyewZHC830DOQ73DzGc8zf3G8kzcqY/Jc4gGTeyyUKt6UScdCJGJhknm4yTTsZIJ2In2gvbwXoiRjo5Zj1x+v7JRIxEzEjGY6TGW0/ESMUL6/GY6fqHMKxfBau/At0dUN8G130Zlt5c8rctSdCbWRz4NnA90AG8aGaPuvvGUryfyLkyMzLJOJnk9IZnRvJOLp9neMQZHsmTC5aFh5MLlsP509ty+UJH0TMwzJFjQ+TdGcl7cExnZKSwHD1+MRmFobNEvNAZJONGIhYjmbBCZxAvdAqjncToMh6LEY9BIhYjHjMSMSMWLE/ffrNTiY95/uT12ATHKNQVixW+n4nFjJgZMYOYjXZYhSm9Y9tPPGKMux43w4LtuBWOMXq8mFGaTnD9Kvjl52D4eGG7u72wDSUP+1Kd0a8Atrn7GwBm9hPgBkBBL5FSCK04pb48wN3JOyc6h9xohzDiJzqbvBc6nrw7+bwz4k4+T7D0U5Zvtp/oYIL3yOed4VyeweGRE/vmvbCfO4UHwb7BMdwhz+n75oNjlqOxHcBJHcaJtjc7hbi9uR4zTnRIxpttP+y9kxY/fvKbDB8vnOGXadDPB9rHbHcA7yzFG82uTrNYv5UqMmONdlLuflrHMdqxjAQdw9jOZ+x+I2M6mRMdDn7K9pj1Me95cnuwzuT2OdPxTnpfd/LB8cZ2hKM1jb6m2Q+O/w+ou6Pk/w5KFfTj/d1zUr9uZrcBtwFccMEFU36j0VvwiojMaPe0FYZrTlXfVvK3LtUUgA5gwZjtNmDv2B3c/T53X+7uy5ubm0tUhojIDHHdlyGZPbktmS20l1ipgv5FYImZLTazFHAL8GiJ3ktEZOZbejN87JtQvwCwwvJj3yzfWTfunjOzzwK/pTC98kF3f60U7yUiUjaW3nxegv1UJZsr4O6PA4+X6vgiIjI5ukxPRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYgz9/BvLWdmXcCuKb68CTjD3YIiI+qfUZ+vvOnzhWehu094D5kZEfTTYWZr3X152HWUUtQ/oz5fedPnm/k0dCMiEnEKehGRiItC0N8XdgHnQdQ/oz5fedPnm+HKfoxeRETOLgpn9CIichZlG/Rm9qCZdZrZq2HXUgpmtsDMnjKzTWb2mpl9PuyaisnMMma2xsxeCT7ffwm7plIws7iZ/cnMfhV2LaVgZjvNbIOZvWxma8Oup9jMrMHMfmpmm4P/F98ddk1TUbZDN2b2XqAP+IG7Xx52PcVmZq1Aq7u/ZGa1wDrgRnffGHJpRWFmBlS7e5+ZJYFngc+7+/Mhl1ZUZvYfgOVAnbt/NOx6is3MdgLL3c/0y9flzcweAp5x9+8Gv5ZX5e5Hw67rXJXtGb27Pw0cDruOUnH3fe7+UrDeC2wC5odbVfF4QV+wmQwe5XnWcQZm1gZ8BPhu2LXIuTOzOuC9wAMA7j5UjiEPZRz0lcTMFgFXAi+EW0lxBcMaLwOdwBPuHqnPB/xP4D8C+bALKSEHfmdm68zstrCLKbILgS7ge8Hw23fNrDrsoqZCQT/DmVkN8DPgC+7eE3Y9xeTuI+5+BdAGrDCzyAzBmdlHgU53Xxd2LSV2jbtfBawE7giGVKMiAVwF3OvuVwL9wBfDLWlqFPQzWDB2/TPgYXd/JOx6SiX4c/j3wIdCLqWYrgH+KhjD/gnwPjP7YbglFZ+77w2WncDPgRXhVlRUHUDHmL80f0oh+MuOgn6GCr6sfADY5O7fCLueYjOzZjNrCNazwPuBzeFWVTzu/iV3b3P3RcAtwJPu/u9CLquozKw6mChAMKTxASAys+DcfT/QbmaXBE3XAWU5GSIRdgFTZWY/Bq4FmsysA/hHd38g3KqK6hrgk8CGYBwb4E53fzzEmoqpFXjIzOIUTjhWuXskpyBGWAvw88I5CQngR+7+m3BLKrq/Bx4OZty8AXw65HqmpGynV4qIyORo6EZEJOIU9CIiEaegFxGJOAW9iEjEKehFRCJOQS8iEnEKehGRiFPQi4hE3P8H1DStq24uP4EAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Feerates:\t1.1499744606513134, 1.3970589103224236, 1.9124681884207781, 6.361686926992798 \n", + "Times:\t\t168.62498827393395, 94.04820895845543, 36.664092522353194, 1.0000000000000004" + ] + }, + "execution_count": 105, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "estimator = FeerateEstimator(total_weight=total_weight, \n", + " inclusion_interval=avg_mass/network_mass_rate)\n", + "\n", + "pred = estimator.calc_estimations()\n", + "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", + "y = estimator.feerate_to_time(x)\n", + "plt.figure()\n", + "plt.plot(x, y)\n", + "plt.fill_between(x, estimator.inclusion_interval, y2=y, alpha=0.5)\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "pred" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Interpolating the original function using two of the points\n", + "\n", + "The code below reverse engineers the original curve using only 2 of the estimated points" + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHW9JREFUeJzt3Xl0XGeZ5/HvU6WSVLKszVpiSbblJMZZyGLjOAHTaUgAk8CQhKUn0ECGgTZNBw6cYTJDaOYA5wwzORMCPX2gM52QNMlAJxMghEAHTAiBEAix5Wze4tiJN8mbbEebtZbqmT/qypZt2ZalKt/Srd/nHJ1776t7q57K8ruv3nrvvebuiIhIdMXCLkBERHJLQS8iEnEKehGRiFPQi4hEnIJeRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQirijsAgBqa2u9paUl7DJERKaVNWvW7Hf3ulPtlxdB39LSQmtra9hliIhMK2a2fSL7aehGRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYib1kG/aU8Pt698mc6+obBLERHJW9M66LcfOMR3n3yVnQf7wy5FRCRvTeugb6goBWBv90DIlYiI5K9TBr2ZzTGzJ81so5mtN7PPB+1fM7N2M3sh+Ll2zDG3mtkWM9tkZstzVfzhoO9R0IuInMhE7nWTAr7o7s+Z2UxgjZk9Hvzu2+7+zbE7m9kFwI3AhUAj8Bsze4O7j2SzcIDa8mLMYG/3YLZfWkQkMk7Zo3f33e7+XLDeA2wEmk5yyHXAg+4+6O5bgS3A0mwUe6yieIza8hL2aehGROSETmuM3sxagEXAs0HTZ83sJTO718yqg7YmYOeYw9oY58RgZivMrNXMWjs6Ok678FENFSUaoxcROYkJB72ZlQM/Ab7g7t3AncA5wKXAbuCO0V3HOdyPa3C/y92XuPuSurpT3k75hBpmlmroRkTkJCYU9GaWIBPyP3T3hwHcfa+7j7h7GribI8MzbcCcMYc3A7uyV/LR6itK2acvY0VETmgis24MuAfY6O7fGtM+e8xuNwDrgvVHgRvNrMTM5gMLgFXZK/loDRUl7O8dYiiVztVbiIhMaxOZdbMM+Biw1sxeCNq+DHzYzC4lMyyzDfg0gLuvN7OHgA1kZuzcnIsZN6NGp1h29A7SVJXM1duIiExbpwx6d3+a8cfdHzvJMd8AvjGFuibsrDEXTSnoRUSON62vjAWorygB0BRLEZETmPZBf+Q2CJp5IyIynmkf9DVlxRTFTHPpRUROYNoHfSxm1M8sUY9eROQEpn3Qg+bSi4icTCSCXrdBEBE5sYgEfSl7uhT0IiLjiUzQdw+k6BtKhV2KiEjeiUTQj14otatTvXoRkWNFIugbDwe9nh0rInKsiAR95qIpBb2IyPEiEfQNFaWYwS59ISsicpxIBH0iHqNhZql69CIi44hE0ENm+EZBLyJyvMgE/eyqJLs1dCMicpzIBH1TVZL2zn7cj3s8rYhIQYtM0DdWljKUSnPg0FDYpYiI5JXoBH0wl363LpoSETlK5IK+XV/IiogcJXJBr5k3IiJHi0zQV5clKE3E2N2loBcRGSsyQW9mNFYmdWMzEZFjRCboITN806ahGxGRo0Qq6JuqkrS/rqAXERkrUkE/d1YZ+3sH6R8aCbsUEZG8Eamgn1NTBsDO1/tCrkREJH9EK+irM1MsdxxQ0IuIjIpU0M9Vj15E5DiRCvqaGcWUFcfZcVBBLyIy6pRBb2ZzzOxJM9toZuvN7PNBe42ZPW5mm4NlddBuZvaPZrbFzF4ys8W5/hBjamVuTRk7D2rmjYjIqIn06FPAF939fOAK4GYzuwD4EvCEuy8Angi2Aa4BFgQ/K4A7s171STRXl7FTPXoRkcNOGfTuvtvdnwvWe4CNQBNwHXBfsNt9wPXB+nXA/Z7xZ6DKzGZnvfITmFtTxo6DfbovvYhI4LTG6M2sBVgEPAs0uPtuyJwMgPpgtyZg55jD2oK2Y19rhZm1mllrR0fH6Vd+AnNrkvQPj+i+9CIigQkHvZmVAz8BvuDu3SfbdZy247rX7n6Xuy9x9yV1dXUTLeOURufS6wtZEZGMCQW9mSXIhPwP3f3hoHnv6JBMsNwXtLcBc8Yc3gzsyk65p3Z4iqWCXkQEmNisGwPuATa6+7fG/OpR4KZg/SbgZ2PaPx7MvrkC6Bod4jkTmqsV9CIiYxVNYJ9lwMeAtWb2QtD2ZeA24CEz+ySwA/hQ8LvHgGuBLUAf8ImsVnwKyeI4dTNLNMVSRCRwyqB396cZf9wd4Opx9nfg5inWNSVza8rYduBQmCWIiOSNSF0ZO2p+7QwFvYhIILJBv7d7kEODqbBLEREJXSSD/uzaGQBs3a9evYhIJIN+fp2CXkRkVCSDvmWWgl5EZFQkg740EaepKqmgFxEhokEPmS9kX1PQi4hEO+i3dvTqLpYiUvAiHfTdAykO6i6WIlLgohv0mnkjIgJEOOhH59JrnF5ECl1kg76pKkkiburRi0jBi2zQF8VjzJs1gy37esMuRUQkVJENeoAF9eUKehEpeNEO+oaZbD9wiIHhkbBLEREJTaSDvqtviLTD+f/tVyy77bc88nx72CWJiJxxkQ36R55v58HVO4HMk8nbO/u59eG1CnsRKTiRDfrbV25iMJU+qq1/eITbV24KqSIRkXBENuh3dY7/zNgTtYuIRFVkg76xKnla7SIiURXZoL9l+UKSifhRbclEnFuWLwypIhGRcBSFXUCuXL+oCYCv/3w9r/cNUzezhL+/9vzD7SIihSKyPXrIhP1Dn34zAF++9jyFvIgUpEgHPUBL7QwSceOVvbpCVkQKU+SDPhGPcU5dORt3d4ddiohIKCIf9AAXNFawYZeCXkQKU0EE/YWNlezrGaSjZzDsUkREzriCCPoLZlcAsEHDNyJSgAor6DV8IyIF6JRBb2b3mtk+M1s3pu1rZtZuZi8EP9eO+d2tZrbFzDaZ2fJcFX46KssSNFcn1aMXkYI0kR7994F3j9P+bXe/NPh5DMDMLgBuBC4MjvknM4uPc+wZd8HsCtbv6gq7DBGRM+6UQe/uTwEHJ/h61wEPuvugu28FtgBLp1Bf1lzQWMHW/YfoG0qFXYqIyBk1lTH6z5rZS8HQTnXQ1gTsHLNPW9B2HDNbYWatZtba0dExhTIm5sLGStzh5T09OX8vEZF8MtmgvxM4B7gU2A3cEbTbOPv6eC/g7ne5+xJ3X1JXVzfJMibugsbMF7Lr9YWsiBSYSQW9u+919xF3TwN3c2R4pg2YM2bXZmDX1ErMjsbKUqrLEqxr0zi9iBSWSQW9mc0es3kDMDoj51HgRjMrMbP5wAJg1dRKzA4z45I5VbzY1hl2KSIiZ9Qpb1NsZg8AbwNqzawN+CrwNjO7lMywzDbg0wDuvt7MHgI2ACngZncfyU3pp++S5iqeemUzhwZTzCiJ7B2aRUSOcsq0c/cPj9N8z0n2/wbwjakUlSuXzqki7bC2vYsrzp4VdjkiImdEQVwZO+ri5koAXtyp4RsRKRwFFfSzykuYW1OmcXoRKSgFFfRA5gvZnZp5IyKFo/CCvrmS9s5+9vUMhF2KiMgZUXBBv2huFYB69SJSMAou6C9srKQoZjy/4/WwSxEROSMKLuhLE3EubKygdZuCXkQKQ8EFPcBlLTW80NbJYCpvruUSEcmZwgz6+TUMpdK8pPveiEgBKMygb6kBYNXWid5mX0Rk+irIoK+ZUcy59eWs3qagF5HoK8igh0yvfs221xlJj3u7fBGRyCjYoF86v5qewRQv79GDSEQk2go26EfH6VdrnF5EIq5gg765uoymqiTPvHYg7FJERHKqYIMe4K3n1vKnVw9onF5EIq2gg37Zglp6BlKsbdd8ehGJrsIO+nMyT5l6enNHyJWIiOROQQf9rPISLphdwdNb9oddiohIzhR00AP8xYJantveSd9QKuxSRERyouCDftm5tQyNpHU7BBGJrIIP+staaiiOx3h6s4ZvRCSaCj7ok8VxLj+7hic37Qu7FBGRnCj4oAe46rx6Xu04xLb9h8IuRUQk6xT0wNXnNQDwxMvq1YtI9CjogbmzylhQX84TG/eGXYqISNYp6ANXn9/Aqq0H6R4YDrsUEZGsUtAH3nF+Pam089QrukpWRKJFQR9YNLea6rIET2zUOL2IRMspg97M7jWzfWa2bkxbjZk9bmabg2V10G5m9o9mtsXMXjKzxbksPpviMeOq8xr4zca9DKZGwi5HRCRrJtKj/z7w7mPavgQ84e4LgCeCbYBrgAXBzwrgzuyUeWa89+LZ9Ayk+KPufSMiEXLKoHf3p4Bj7w9wHXBfsH4fcP2Y9vs9489AlZnNzlaxubbs3FoqSov4xUu7wy5FRCRrJjtG3+DuuwGCZX3Q3gTsHLNfW9B2HDNbYWatZtba0ZEfX4AWF8VYfuFZPL5ewzciEh3Z/jLWxmkb9/FN7n6Xuy9x9yV1dXVZLmPyrr14Nj2DKf7wioZvRCQaiiZ53F4zm+3uu4OhmdGpKm3AnDH7NQO7plLgmbbsnFqSiRife+B5BoZHaKxKcsvyhVy/aNw/TERE8t5ke/SPAjcF6zcBPxvT/vFg9s0VQNfoEM908dja3QyNOP3DIzjQ3tnPrQ+v5ZHn28MuTURkUiYyvfIB4BlgoZm1mdkngduAd5rZZuCdwTbAY8BrwBbgbuDvclJ1Dt2+ctNxDwvvHx7h9pWbQqpIRGRqTjl04+4fPsGvrh5nXwdunmpRYdrV2X9a7SIi+U5Xxh6jsSp5Wu0iIvlOQX+MW5YvJJmIH9WWTMS5ZfnCkCoSEZmayc66iazR2TW3r9xEe2c/8ZjxP254o2bdiMi0pR79OK5f1MQfv3QVd3zoEkbSTkNladgliYhMmoL+JN5z8WyqyhL832e2h12KiMikKehPojQR598vmcOvN+xld5dm3YjI9KSgP4WPXjGPtDsPPLsj7FJERCZFQX8Kc2rKuGphPf+6aidDqXTY5YiInDYF/QR87M3z2N87yL+tnVa37RERART0E3LlgjoW1Jfzz79/jczFvyIi04eCfgJiMeNv//IcXt7Tw+825ce980VEJkpBP0Hvu7SRxspS7vz9q2GXIiJyWhT0E5SIx/jUX5zNqq0HWbP99bDLERGZMAX9abhx6RyqyhJ857ebwy5FRGTCFPSnoay4iBVXns2TmzpYs/3Y56WLiOQnBf1p+g9vaaG2vIT/9atNmoEjItOCgv40lRUX8dm3n8OzWw/y9BY9QFxE8p+CfhI+fPlcmqqSfHPlJtJp9epFJL8p6CehpCjOF96xgBfbuvjZi3pouIjkNwX9JH1gcTMXN1dy2y9f5tBgKuxyREROSEE/SbGY8dV/dyF7uwf5p99tCbscEZETUtBPwZvmVXPDoibufmor2w8cCrscEZFxKein6EvXnEdxUYy//+k6TbcUkbykoJ+ihopS/us15/H0lv38aE1b2OWIiBxHQZ8Ff710Lktbavjvv9jAvu6BsMsRETmKgj4LYjHjtg9cxEAqzVce0RCOiOQXBX2WnF1Xzn9+1xv49Ya9PLh6Z9jliIgcpqDPok+99Wzeem4tX//5erbs6wm7HBERQEGfVbGY8a2/uoSy4iI+98ALDAyPhF2SiMjUgt7MtpnZWjN7wcxag7YaM3vczDYHy+rslDo91FeU8s0PXczG3d18/efrwy5HRCQrPfq3u/ul7r4k2P4S8IS7LwCeCLYLylXnNfB3bzuHB1bt5IfPbg+7HBEpcLkYurkOuC9Yvw+4Pgfvkfe++K6FvH1hHV97dD2rt+khJSISnqkGvQO/NrM1ZrYiaGtw990AwbJ+iu8xLcVjxj/cuIjm6jI+84M17DjQF3ZJIlKgphr0y9x9MXANcLOZXTnRA81shZm1mllrR0fHFMvIT5XJBHd/fAmptPPxe59lf+9g2CWJSAGaUtC7+65guQ/4KbAU2GtmswGC5b4THHuXuy9x9yV1dXVTKSOvnVtfzj03Xcae7gH+4/dX65bGInLGTTrozWyGmc0cXQfeBawDHgVuCna7CfjZVIuc7t40r5rvfmQx63d18zf3t9I/pGmXInLmTKVH3wA8bWYvAquAf3P3XwG3Ae80s83AO4Ptgnf1+Q3c/sGLeea1A3zyvtUKexE5Y4ome6C7vwZcMk77AeDqqRQVVe9f3AzAF3/0Ip+8bzX33HQZyeJ4yFWJSNTpytgz7P2Lm/nWX13Cn187wEfveZbXDw2FXZKIRJyCPgQ3LGrmOx9ZzNr2Lj7wf/7EzoOaeikiuTPpoRuZmmsvmk1teQmfum8177/zT3z8ink8uHonuzr7aaxKcsvyhVy/qCnsMkUkAtSjD9HS+TX85DNvYTiV5o7HX6G9sx8H2jv7ufXhtTzyfHvYJYpIBCjoQ7agYSal43wh2z88wu0rN4VQkYhEjYI+D+ztGv/xg7s6+89wJSISRQr6PNBYlRy3vTKZ0GMJRWTKFPR54JblC0kmjh6+iRl09g/zN/e3srtLPXsRmTwFfR64flET//P9F9FUlcSApqokd3zwEr7ynvN5est+3nHH7/mXP25lJK3evYicPsuHoYElS5Z4a2tr2GXkpZ0H+/jKI+v4/SsdXNRUydfedwFvmlcTdlkikgfMbM2Yhz6dkHr0eW5OTRnf/8RlfOcji9jbPcAH7nyGz/xgDdv2Hwq7NBGZJnTB1DRgZrz34kauOq+eu5/ayj8/9Sq/2biXv758Hn/7l+dwVmVp2CWKSB7T0M00tK97gG//5hUeam0jbsaHljTzmbedQ3N1WdilicgZNNGhGwX9NLbjQB93/v5VfrxmJ+5w3aVNfGJZC29sqgy7NBE5AxT0BWRXZz93PfUa/2/1TvqHR1gyr5qb3tLCu994Fom4voYRiSoFfQHq6h/mR607uf+Z7ew42Ef9zBJuWNzEBxc3s6BhZtjliUiWKegL2Eja+d2mfTywagdPbupgJO1cMqeKDy5u4j0XN1IzozjsEkUkCxT0AkBHzyA/e6GdH69p4+U9PcQMLp8/i2suOovlF55FQ4Vm7IhMVwp6OYq7s2F3N79cu4dfrtvNqx2ZefhvmlfNVefVc+WCOi5srCAWs5ArFZGJUtDLSW3e28Ov1u1h5YY9rGvvBmDWjGL+YkEtV76hjmXn1qq3L5LnFPQyYR09g/xhcwdPvdLBU5v3czB4ju3cmjKWzq9haUsNl82voWVWGWbq8YvkCwW9TEo67azf1c2zWw+wautBWre/fjj4a8tLuHROFRc3V3JRcyUXNVVSW14ScsUihUtBL1nh7rza0cuqra/Tuu0gL7Z18tr+Q4z+Z9NYWcpFzZVc2FjJGxpmsvCsmcytKSOusX6RnJto0OteN3JSZsa59TM5t34mH7l8LgA9A8Os39XNuvYuXmrr4qW2Tlau33v4mJKiGOfUlbPwrJksaChnQf1M5teW0VxdRmni+McmikhuKejltM0sTXDF2bO44uxZh9sODabYsq+XTXt72Ly3h017e3nm1QP8dMwDzs2gsTLJvFllzJs1g5ZgObemjKaqJBXJIn0HIJIDCnrJihklRVwyp4pL5lQd1d7VP8yrHb3sONDHtgOH2B4sV67fc3js//BrFMeZXZWksSpJY2UpjVVJZleW0lSV5KzKUupmllBeopOByOlS0EtOVSYTLJ5bzeK51cf9rqt/mB0H+thxsI/dXf20d/azu3OAXV39bNjVzf7eweOOKU3EqC0voW5mCXXB8vB2sF5dlqC6rJiKZELfFYigoJcQVSYTmdk7zePfbXNgeIQ9XZng39M1wP7eQTp6BtnfO0RHzyDbD/QdNSvoWGaZ96guKw6WmfWqsmKqyxJUzSimKpmgvLSIitIiZpYmKC8pYmZpETOKi3TxmESGgl7yVmkiTkvtDFpqZ5x0v+GRNAcPZcK/o3eQzr4hXj80nFn2DfN63xBd/cN09A7yyt5eOvuGODQ0ctLXNIPy4iLKSzPBP/YkMHoiKCuOkzy8jFMW/CQTRUfWi+OUBfuUFMU07CShyFnQm9m7gf8NxIHvufttuXovKWyJeIyGitLTupJ3MDVCV98wXf3DdA+k6B1M0TMwTO9Aip6BzHrPYGa9dyBFz2DmhLHzYB/dAykODaboHz75yeJYMYNk4sjJoTQRo6QocwIoGbteFKM0Mdp+pK2kKB7sd2Tf0f2Ki2Ik4jGK4kZxfPz1RLCu4azwPPJ8O7ev3MSuzn4aq5Lcsnwh1y9qyvn75iTozSwOfBd4J9AGrDazR919Qy7eT+R0lRTFqa+IUz+F2zyk085AaoS+oRH6hzLLvqHUkfXhEfqHUkH7kX36hzNtQ6k0A8MjDKbSDAyn6eofZnA4zWAqzWAq0z44nGYgNUI2L3eJGYdDPzHmBDC6XhSPURw3io75fVHMKIob8VhmPWZGUcyIx4NlzIjb2O3YkfbYkX2KYkYsNv4+R+8XIxaDoljm5BQzgqVhY9ZjRrDMHGfB9lHrwe8txpH1Y14j139tPfJ8O7c+vPZwB6G9s59bH14LkPOwz1WPfimwxd1fAzCzB4HrAAW9REYsZsGwTG5HQN2dVNoPnxQyJ4Dj11PpNEMpZ3gksz6ccoZG0qRG0gyPOMNB2/BI+qj1Y48bSnnm+JHMPr2pFEOpNCNpP/yTOrxMM5KGkXT6cNvYfaYTO3zCOHLiiFnm3/PoiWP0hBAfc3IYbY+ZBfuCceQkE4tltl/e083wyNH/TPqHR7h95aZpG/RNwM4x223A5Tl6L5FIM7PDPevp9PgYdyftkEqnSacJTgo+7gnh2BPF4RPJiJP2zEnIHdLBa46kHXdnJNh2z+yfHt1n7Pp422Ne47j18Y457vUy6yPuh+vKvP+R7cxy9J+DHxfyo3Z19uf830Wugn68v4GO+pRmtgJYATB37twclSEiYcn0fCEeG70aurCvil52229pHyfUG6uSOX/vXD1QtA2YM2a7Gdg1dgd3v8vdl7j7krq6uhyVISKSH25ZvpDkMbcASSbi3LJ8Yc7fO1c9+tXAAjObD7QDNwIfydF7iYjkvdFx+MjMunH3lJl9FlhJ5u+1e919fS7eS0Rkurh+UdMZCfZj5Wy6gLs/BjyWq9cXEZGJydUYvYiI5AkFvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4syzef/TyRZh1gFsn+ThtcD+LJaTj6L+GfX5pjd9vvDMc/dT3kMmL4J+Ksys1d2XhF1HLkX9M+rzTW/6fPlPQzciIhGnoBcRibgoBP1dYRdwBkT9M+rzTW/6fHlu2o/Ri4jIyUWhRy8iIicxbYPezO41s31mti7sWnLBzOaY2ZNmttHM1pvZ58OuKZvMrNTMVpnZi8Hn+3rYNeWCmcXN7Hkz+0XYteSCmW0zs7Vm9oKZtYZdT7aZWZWZ/djMXg7+X3xz2DVNxrQdujGzK4Fe4H53f2PY9WSbmc0GZrv7c2Y2E1gDXO/uG0IuLSvMzIAZ7t5rZgngaeDz7v7nkEvLKjP7T8ASoMLd3xt2PdlmZtuAJe6er/PMp8TM7gP+4O7fM7NioMzdO8Ou63RN2x69uz8FHAy7jlxx993u/lyw3gNsBM78o2lyxDN6g81E8DM9ex0nYGbNwHuA74Vdi5w+M6sArgTuAXD3oekY8jCNg76QmFkLsAh4NtxKsisY1ngB2Ac87u6R+nzAPwD/BUiHXUgOOfBrM1tjZivCLibLzgY6gH8Jht++Z2Yzwi5qMhT0ec7MyoGfAF9w9+6w68kmdx9x90uBZmCpmUVmCM7M3gvsc/c1YdeSY8vcfTFwDXBzMKQaFUXAYuBOd18EHAK+FG5Jk6Ogz2PB2PVPgB+6+8Nh15MrwZ/DvwPeHXIp2bQMeF8whv0gcJWZ/SDckrLP3XcFy33AT4Gl4VaUVW1A25i/NH9MJvinHQV9ngq+rLwH2Oju3wq7nmwzszozqwrWk8A7gJfDrSp73P1Wd2929xbgRuC37v7RkMvKKjObEUwUIBjSeBcQmVlw7r4H2GlmC4Omq4FpORmiKOwCJsvMHgDeBtSaWRvwVXe/J9yqsmoZ8DFgbTCODfBld38sxJqyaTZwn5nFyXQ4HnL3SE5BjLAG4KeZPglFwL+6+6/CLSnrPgf8MJhx8xrwiZDrmZRpO71SREQmRkM3IiIRp6AXEYk4Bb2ISMQp6EVEIk5BLyIScQp6EZGIU9CLiEScgl5EJOL+P7IeihVMYkBBAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(array([168.62498827, 94.03830275, 36.64656385, 0.97773399]),\n", + " array([168.62498827, 94.04820896, 36.66409252, 1. ]))" + ] + }, + "execution_count": 106, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x1, x2 = pred.low_bucket.feerate**ALPHA, pred.normal_bucket.feerate**ALPHA\n", + "y1, y2 = pred.low_bucket.estimated_seconds, pred.normal_bucket.estimated_seconds\n", + "b2 = (y1 - y2*x2/x1) / (1 - x1/x2)\n", + "b1 = (y1 - b2) * x1\n", + "def p(ff):\n", + " return b1/ff**ALPHA + b2\n", + "\n", + "plt.figure()\n", + "plt.plot(x, p(x))\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "p(pred.feerates()), pred.times()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Challenge: outliers\n", + "\n", + "The segment below illustrates a challenge in the current approach. It is sufficient to add a single outlier \n", + "to the total weight (with `feerate=100`), and the `feerate_to_time` function is notably influenced. In truth, this tx should not affect our prediction because it only captures the first slot of each block, however because we sample with repetition it has a significant impact on the function. The following figure shows the `feerate_to_time` function with such an outlier " + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG1xJREFUeJzt3X1wHPWd5/H3t+dJD7bkJ/lRfiIxEEwIJF7wHrm7HA6PSWGqNslxt0tce9RRdWGX7N3ebUJSBXck2SN3uZBNEXJFYgdnNxXiImQhVLjgNbAbskAwT+bBGBubYPlRfpJlW5Ilzff+6JY8lmakkZGmx9OfV5Vqun/9m57vyJY++vWve9rcHRERSZ4g7gJERCQeCgARkYRSAIiIJJQCQEQkoRQAIiIJpQAQEUkoBYCISEIpAEREEkoBICKSUOm4CxjJjBkzfNGiRXGXISJyVnnxxRcPuHvLaP2qOgAWLVrExo0b4y5DROSsYma/L6efDgGJiCSUAkBEJKEUACIiCaUAEBFJKAWAiEhCKQBERBJKASAiklA1GQB7Orr49hNb2N5+LO5SRESqVk0GwIHOk3z3yW2803487lJERKpWTQZAXSZ8W929/TFXIiJSvWo0AFIAdCkARERKqskAqM+GAaARgIhIaTUZAAMjAAWAiEhptRkA6fBtdZ3Mx1yJiEj1qskASKcCMinTHICIyAhqMgAAcumUDgGJiIyg7AAws5SZvWxmj0Xri83seTPbamY/M7Ns1J6L1rdF2xcV7OP2qH2LmV093m+mUC4dKABEREYwlhHAF4HNBevfBO5x9yXAYeDmqP1m4LC7fxC4J+qHmV0A3AgsBa4B7jOz1PsrvzQFgIjIyMoKADNrBT4F/DBaN+AK4KGoy1rghmh5ZbROtH1F1H8l8KC797j7DmAbcOl4vIlisulAcwAiIiModwTwHeCvgIHTaqYDR9y9L1pvA+ZFy/OAnQDR9o6o/2B7keeMuzAAdBaQiEgpowaAmX0a2O/uLxY2F+nqo2wb6TmFr3eLmW00s43t7e2jlVdSNqVDQCIiIylnBHA5cL2ZvQs8SHjo5zvAFDNLR31agd3RchswHyDa3gwcKmwv8pxB7n6/uy9z92UtLS1jfkMDMumArpMKABGRUkYNAHe/3d1b3X0R4STuk+7+x8BTwGeibquAR6LlR6N1ou1PurtH7TdGZwktBpYAvxu3dzJETnMAIiIjSo/epaQvAQ+a2deBl4HVUftq4G/NbBvhX/43Arj7G2a2DngT6ANudfcJ+w2tQ0AiIiMbUwC4+9PA09HydoqcxePu3cBnSzz/G8A3xlrkmcjpEJCIyIhq9krgrK4EFhEZUQ0HQEB3n04DFREppWYDIJcO6M87vf0KARGRYmo6AEB3BRMRKaVmAyCb1n2BRURGUrMBMDAC6NZNYUREiqrZAMimdWN4EZGR1HAA6BCQiMhIajYANAksIjIyBYCISELVbABkU+Fb61EAiIgUVbMBoBGAiMjIajYABiaBT+gD4UREiqrZAKjLRKeBKgBERIqq+QA43qMAEBEppmYDIBUY6cA40ds3emcRkQSq2QCAcB7ghEYAIiJF1XYApAKOn9QIQESkmJoOgExKIwARkVJqOgDSKdMIQESkhJoPAF0HICJSXE0HQCYION6jEYCISDG1HQBpBYCISCm1HQAp47gOAYmIFFXTAZBNBZzQJLCISFE1HQCZVEB3b5583uMuRUSk6tR8AIA+ElpEpJgaDwAD0LUAIiJF1HQADNwVTFcDi4gMV9MBkIluCqMRgIjIcLUdACndFUxEpJQaD4BoDkAXg4mIDFPjAaARgIhIKTUdAAOTwBoBiIgMV9MBoBGAiEhpNR4Aug5ARKSUmg6AVGAEpkNAIiLFjBoAZlZnZr8zs1fN7A0z+x9R+2Ize97MtprZz8wsG7XnovVt0fZFBfu6PWrfYmZXT9SbKng9cukUx7oVACIiQ5UzAugBrnD3jwAXA9eY2XLgm8A97r4EOAzcHPW/GTjs7h8E7on6YWYXADcCS4FrgPvMLDWeb6aYXDqgUwEgIjLMqAHgoWPRaib6cuAK4KGofS1wQ7S8Mlon2r7CzCxqf9Dde9x9B7ANuHRc3sUIMumATh0CEhEZpqw5ADNLmdkrwH5gPfAOcMTdB36ztgHzouV5wE6AaHsHML2wvchzJkw2FdDZ3TvRLyMictYpKwDcvd/dLwZaCf9q/1CxbtGjldhWqv00ZnaLmW00s43t7e3llDeibDrgaJdGACIiQ43pLCB3PwI8DSwHpphZOtrUCuyOltuA+QDR9mbgUGF7kecUvsb97r7M3Ze1tLSMpbyisqmAzh6NAEREhirnLKAWM5sSLdcDnwQ2A08Bn4m6rQIeiZYfjdaJtj/p7h613xidJbQYWAL8brzeSCnZdKCzgEREikiP3oU5wNrojJ0AWOfuj5nZm8CDZvZ14GVgddR/NfC3ZraN8C//GwHc/Q0zWwe8CfQBt7r7hF+iO3AWkLsTzkWLiAiUEQDuvgm4pEj7doqcxePu3cBnS+zrG8A3xl7mmcumA/ryTk9fnrrMhJ91KiJy1qjpK4EhDABA1wKIiAxR8wGQGwwATQSLiBSq+QD4w+NP8kz2NhZ/rxXuuRA2rYu7JBGRqlDOJPBZ67z9j/PJfd8iG/SEDR074Ze3hcsXfS6+wkREqkBNjwA+/t59ZL3n9MbeLthwVzwFiYhUkZoOgMk9+4pv6GirbCEiIlWopgOgMzer+Ibm1soWIiJShWo6AJ5Z8AV6g7rTGzP1sOKOeAoSEakiNT0JvGXmtQB85O3vMtcOYs2t4S9/TQCLiNR2AEAYAl/aej7XXTiHb37morjLERGpGjV9CGiAPhFURGS4RARALh3Q0aUAEBEplIwAyKQ4fFwBICJSKBEBUJcOOHLiZNxliIhUlUQEQC6T0iEgEZEhEhEAdZmA4yf76e3Px12KiEjVSEYApMMbwWgUICJySjICILoT2JETCgARkQEJCYDwbXZ0aSJYRGRAIgIgpxGAiMgwiQiAuui2kAoAEZFTkhEAAyMATQKLiAxKRADk0gEGdOhiMBGRQYkIADOjLpPSCEBEpEAiAgCgPpPSHICISIHEBEAuE2gEICJSIDEBkE0FHDmuOQARkQGJCYBcJuCQJoFFRAYlJgAasmkOawQgIjIoMQFQn01x/GQ/3b39cZciIlIVEhMADdHFYAc1ChARAZIUANkoAI71xFyJiEh1SEwA1Gc1AhARKZScABg4BHRMASAiAgkKgIZsGtAhIBGRAYkJgEzKSAemQ0AiIpHEBICZ0ZhLc0AjABERoIwAMLP5ZvaUmW02szfM7ItR+zQzW29mW6PHqVG7mdl3zWybmW0ys48W7GtV1H+rma2auLdVXH0mpTkAEZFIOSOAPuAv3f1DwHLgVjO7APgysMHdlwAbonWAa4El0dctwPchDAzgTuAy4FLgzoHQqJS6TKARgIhIZNQAcPc97v5StNwJbAbmASuBtVG3tcAN0fJK4Mceeg6YYmZzgKuB9e5+yN0PA+uBa8b13YyiPptSAIiIRMY0B2Bmi4BLgOeBWe6+B8KQAGZG3eYBOwue1ha1lWof+hq3mNlGM9vY3t4+lvJG1ZBJc+j4Sdx9XPcrInI2KjsAzGwS8HPgL9z96Ehdi7T5CO2nN7jf7+7L3H1ZS0tLueWVpT6borff6ezpG9f9ioicjcoKADPLEP7y/4m7Pxw174sO7RA97o/a24D5BU9vBXaP0F4xjbnwYrD9R7sr+bIiIlWpnLOADFgNbHb3bxdsehQYOJNnFfBIQfvno7OBlgMd0SGiXwNXmdnUaPL3qqitYiblwovB9h3VPICISLqMPpcDNwGvmdkrUdtXgLuBdWZ2M/Ae8Nlo26+A64BtwAngTwHc/ZCZfQ14Iep3l7sfGpd3UabGKAD2dmgEICIyagC4+zMUP34PsKJIfwduLbGvNcCasRQ4ngZHAJ0KABGRxFwJDJBJBdRlAvZpBCAikqwAgHAUoDkAEZEEBkBDNs1enQUkIpK8AGjMpRQAIiIkMQCyado7e8jndTWwiCRb4gJgUi5Nf951XwARSbzkBUDdwMVgOgwkIsmWuABozOpiMBERSGAATI5GALuOdMVciYhIvBIXAA3ZFOnAaDt8Iu5SRERilbgAMDOa6zO0HdYIQESSLXEBAOFE8E6NAEQk4RIZAE11GdoOaQQgIsmW0ABIc6Srl2O6M5iIJFgiA2BF7z/yTPY2Gv/nDLjnQti0Lu6SREQqrpwbwtSU8/Y/zifb/w/ZIPpE0I6d8MvbwuWLPhdfYSIiFZa4EcDH37uPrA/5OOjeLthwVzwFiYjEJHEBMLlnX/ENHW2VLUREJGaJC4DO3KziG5pbK1uIiEjMEhcAzyz4Ar1B3emNmXpYcUc8BYmIxCRxk8BbZl4LwLJ37mVGfzs2ZR624k5NAItI4iQuACAMgV/0Xc76zft46qZPsHhGY9wliYhUXOIOAQ2Y0pABYMeBYzFXIiISj8QGwNTGLADb24/HXImISDwSGwD1mRT1mRTbDygARCSZEhsAEB4G2qERgIgkVOIDYOv+zrjLEBGJRaIDYMakHAeOneTgsZ7RO4uI1JjEBwDAlr0aBYhI8iQ8AMIzgTYrAEQkgRIdAA3ZNJNyad7aczTuUkREKi7RAQAwrTHLZgWAiCRQ4gNgxqQsW/cfo68/H3cpIiIVpQCYlKOnL8+7B3U9gIgkS+IDoGVyeCbQpraOmCsREamsxAfAtMYs2XTAKzuPxF2KiEhFJT4AAjNmTc7x8nsKABFJllEDwMzWmNl+M3u9oG2ama03s63R49So3czsu2a2zcw2mdlHC56zKuq/1cxWTczbOTMzm+rYvOco3b39cZciIlIx5YwAHgCuGdL2ZWCDuy8BNkTrANcCS6KvW4DvQxgYwJ3AZcClwJ0DoVENZjfV0Zd33tit00FFJDlGDQB3/yfg0JDmlcDaaHktcENB+4899BwwxczmAFcD6939kLsfBtYzPFRiM7s5vEfwq5oHEJEEOdM5gFnuvgcgepwZtc8Ddhb0a4vaSrUPY2a3mNlGM9vY3t5+huWNzaRcmqa6NC/+/nBFXk9EpBqM9ySwFWnzEdqHN7rf7+7L3H1ZS0vLuBY3krlT6vnndw7gXrQsEZGac6YBsC86tEP0uD9qbwPmF/RrBXaP0F41WqfWc/hEL2/v0z2CRSQZzjQAHgUGzuRZBTxS0P756Gyg5UBHdIjo18BVZjY1mvy9KmqrGvOnNgDw7DsHYq5ERKQyyjkN9KfAs8B5ZtZmZjcDdwNXmtlW4MpoHeBXwHZgG/AD4AsA7n4I+BrwQvR1V9RWNZrqMzTXZ3h2+8G4SxERqYj0aB3c/d+V2LSiSF8Hbi2xnzXAmjFVV2HzptTz7DsH6c87qaDYtIWISO1I/JXAhRZOb+Bodx8vvaezgUSk9ikACiyc3kBg8A+b98VdiojIhFMAFMilU7RObeAf3lQAiEjtUwAMsXhGI++0H+fdA7o/gIjUNgXAEItnNALw6zf2xlyJiMjEUgAM0VyfYXZTHX//yq64SxERmVAKgCLOmz2ZzXs62bqvM+5SREQmjAKgiCUzJxEYGgWISE1TABTRmEszf1oDD7+0i/68PhxORGqTAqCEpXOa2NPRzVNv7R+9s4jIWUgBUMI5LZOYXJdm7bPvxl2KiMiEUACUkAqMC+c285utB9jero+IFpHaowAYwdK5TaQC4we/2R53KSIi404BMILGXJqlc5pYt7GNtsMn4i5HRGRcKQBGsWzRVADue/qdmCsRERlfCoBRTK7LcMGcJta9sJMd+nwgEakhCoAyXLZ4GqnA+Npjb8ZdiojIuFEAlKExl+YPFk3jybf28/QWXRcgIrVBAVCmi+dPYVpjlq/+4nU6u3vjLkdE5H1TAJQpFRgrzp/J7o4uvv7Y5rjLERF53xQAYzB3Sj0fWzCVn23cyeOv7Ym7HBGR90UBMEbLz5nOnOY6/su6V9myVx8XLSJnLwXAGKUC47oL55AKjP/4440cONYTd0kiImdEAXAGJtWlue7Ds9nT0cVNq5+no0uTwiJy9lEAnKE5zfV86sNzeHvfMT6/5nmOnDgZd0kiImOiAHgfFk5v5NoLZ/P6rqP80ff/mV1HuuIuSUSkbAqA9+kDLZO44eK57Drcxcp7n+G57QfjLklEpCwKgHHQOrWBz3yslbzDv//Bc9z75Fb6+vNxlyUiMiIFwDiZPinHv102nyUzJ/GtJ97mhu/9ltd3dcRdlohISQqAcZRNB1y9dDbXXTibHQePc/29z/CVX7zG3o7uuEsTERkmHXcBtcbMWDJrMvOnNfDc9oP87IWd/PzFNv5k+UL+9PJFtE5tiLtEERFAATBh6jIpPnHeTC5ZMJXntx/kR7/dwY9+u4Orl87mpuULWX7OdILA4i5TRBJMATDBmuszXLV0Nss/MJ1NbR08vaWdx1/fy6ymHNd/ZC6fvmguH57XrDAQkYpTAFRIU12Gj39wBpctnsaOA8fZsreTNb99lx/8ZgfTGrN84rwWPnHeTJYvnsbMprq4yxWRBFAAVFgmFXDurMmcO2syXb39/P7Acd49eILHX9vLwy/tAmDelHqWLZrKxxZOZencJs6dNZnJdZnRd75pHWy4CzraoLkVVtwBF31ugt+RiJytFAAxqs+kOH9OE+fPaSLvzv6jPezu6GJPRzcbNu/nkVd2D/adO6WOC+Y0sWTWZBZOa2DB9AYWTGtgTnM9qcDCX/6/vA16o6uRO3aG66AQEJGiKh4AZnYN8DdACvihu99d6RqqUWDG7OY6ZjeHh3/cnc7uPg4c6+HA8ZMc7OzhlZ1HePKt/eT91PPSgTFvaj0PdX+Vlv4hH0XR20XvE/+dYx+4gab6TBgUIiKRigaAmaWA7wFXAm3AC2b2qLvrbutDmBlN9Rma6jOc03KqPZ93Onv66OjqHfw62tXL9P72ovtJde7mkq+tB2ByXZop9RmmNmaZ0pBlSn2G5voMjbk0jdkUDbk0k3IpGrJpGqPHSbk0DdkUjbk0uXRALp0ilw40aS0ynmI6fFvpEcClwDZ33w5gZg8CKwEFQJmCwGiOfnEXOrZxFk09e4f1P5xp4V8vbKG7tz/86svT0dVLe2cP3b399PTlOdmXp69wWFGGdGDk0gHZgVDIBNSlU2TTAXWZU0GRSQWkU0Y6MNKpgEzKSAVGOggG28JHI5MKom02ZFv4PDMjZUZg4fchKLKcsqhfEK4XLod9jCA41S8wou0W7SfsZwbGwGO4n/AxbMcouc2ibBxYH6hjsI8pPKVAjIdvKx0A84CdBettwGUVrqEmPbPgC1z5zl+TyZ+66rg3qOP5xX/GxTOnjPr8/rzT25+ntz8MhN7+gvX+PL19Tl8+T3/e6cv7aY/hcp6+fJ7u7n4OnzjVnncn7+Ehrf684w79HrXno+W8M7b4qQ3DAoXSITIQNhQJplP9TgXN4GsMyZrTVodsLPW80/c4dNuphbL6FWkofN5I9RYG59D9lVvvSO2l9j+8pvLqLedFDVh9+CvMzA8/fMuGu2ouAIp9f0772TezW4BbABYsWHDGL7Rs0TQ+tnDqGT//7LMEXpuDFwwj0yvu4NoPf5Zr4y6tDPkoUMIgcfr6nb7+aDkKJ3fIFwTJwPJgwERhkx8MnjBk8gXbfOhzouXB/gXPdwdn4BHcgdPWfbB9YJ3B9VP98vni7YXrlNjf4OuOsM0JGwtrCCs9XcGmItuKR/DQ5sKoPm1/w/qV3reXWBn6Z0Cpesutafhrjf17M1Ltw75lJb6HpV5zQMvBA8U3dLSNuL/xUOkAaAPmF6y3ArsLO7j7/cD9AMuWLXtffxgmbqh90efO2jN+UikjlYIcqbhLEamse1rDwz5DNbdO+EtX+sPgXgCWmNliM8sCNwKPVrgGEZHqseIOyNSf3papD9snWEVHAO7eZ2Z/Bvya8DTQNe7+RiVrEBGpKgOj9gScBYS7/wr4VaVfV0SkasV0+Fb3AxARSSgFgIhIQikAREQSSgEgIpJQCgARkYRSAIiIJJQCQEQkoRQAIiIJZaU+BKoamFk78Pu464jMAEp8alNVUH1nrpprg+qur5prg+qubyJrW+juLaN1quoAqCZmttHdl8VdRymq78xVc21Q3fVVc21Q3fVVQ206BCQiklAKABGRhFIAlO/+uAsYheo7c9VcG1R3fdVcG1R3fbHXpjkAEZGE0ghARCShFACjMLP5ZvaUmW02szfM7Itx1zSUmaXM7GUzeyzuWoYysylm9pCZvRV9D/8w7poKmdl/jv5dXzezn5pZXYy1rDGz/Wb2ekHbNDNbb2Zbo8fYbnRdor7/Hf3bbjKzX5jZlGqqr2DbfzUzN7MZ1VSbmf25mW2J/g/+r0rXpQAYXR/wl+7+IWA5cKuZXRBzTUN9EdgcdxEl/A3w/9z9fOAjVFGdZjYPuA1Y5u4XEt6l7sYYS3oAuGZI25eBDe6+BNgQrcflAYbXtx640N0vAt4Gbq90UQUeYHh9mNl84ErgvUoXVOABhtRmZv8GWAlc5O5LgW9VuigFwCjcfY+7vxQtdxL+ApsXb1WnmFkr8Cngh3HXMpSZNQH/ClgN4O4n3f1IvFUNkwbqzSwNNAC74yrE3f8JODSkeSWwNlpeC9xQ0aIKFKvP3Z9w975o9Tlg4u9kXkKJ7x/APcBfAbFNeJao7T8Bd7t7T9Rnf6XrUgCMgZktAi4Bno+3ktN8h/A/dz7uQoo4B2gHfhQdovqhmTXGXdQAd99F+FfXe8AeoMPdn4i3qmFmufseCP8YAWbGXM9I/gPweNxFFDKz64Fd7v5q3LUUcS7wL83seTP7RzP7g0oXoAAok5lNAn4O/IW7H427HgAz+zSw391fjLuWEtLAR4Hvu/slwHHiPYRxmuh4+kpgMTAXaDSzP4m3qrOTmX2V8HDpT+KuZYCZNQBfBe6Iu5YS0sBUwkPL/w1YZ2ZWyQIUAGUwswzhL/+fuPvDcddT4HLgejN7F3gQuMLM/i7ekk7TBrS5+8CI6SHCQKgWnwR2uHu7u/cCDwP/IuaahtpnZnMAoseKHyYYjZmtAj4N/LFX13nlHyAM91ejn5FW4CUzmx1rVae0AQ976HeEo/iKTlIrAEYRJfJqYLO7fzvuegq5++3u3uruiwgnL59096r5C9bd9wI7zey8qGkF8GaMJQ31HrDczBqif+cVVNEkdeRRYFW0vAp4JMZahjGza4AvAde7+4m46ynk7q+5+0x3XxT9jLQBH43+X1aDvweuADCzc4EsFf7gOgXA6C4HbiL86/qV6Ou6uIs6i/w58BMz2wRcDPx1zPUMikYmDwEvAa8R/jzEdnWmmf0UeBY4z8zazOxm4G7gSjPbSngmy91VVt+9wGRgffSz8X+rrL6qUKK2NcA50amhDwKrKj2C0pXAIiIJpRGAiEhCKQBERBJKASAiklAKABGRhFIAiIgklAJARCShFAAiIgmlABARSaj/D6Gz7+yGqKSPAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Feerates:\t1.1539704395225572, 1.4115360845240776, 4.139754128892224, 16.2278954349457 \n", + "Times:\t\t2769.889957638353, 1513.4606486459202, 60.00000000000002, 1.0000000000000007" + ] + }, + "execution_count": 107, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "estimator = FeerateEstimator(total_weight=total_weight + 100**ALPHA, \n", + " inclusion_interval=avg_mass/network_mass_rate)\n", + "\n", + "pred = estimator.calc_estimations()\n", + "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", + "y = estimator.feerate_to_time(x)\n", + "plt.figure()\n", + "plt.plot(x, y)\n", + "plt.fill_between(x, estimator.inclusion_interval, y2=y, alpha=0.5)\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "pred" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Outliers: solution\n", + "\n", + "Compute the estimator conditioned on the event the the top most transaction captures the first slot. This decreases `total_weight` on the one hand (thus increasing `p`), while increasing `inclusion_interval` on the other, by capturing a block slot. If this estimator gives lower prediction times we switch to it, and then repeat the process with the next highest transaction. The process convegres when the estimator is no longer improving or if all block slots are captured. " + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHmVJREFUeJzt3X2QXXWd5/H39z72U/op6TzQnZggEVREwR7Ah3IdIw5RxzCzYuG4Gl1qU7MwPoyzpehUyYxTM+PUuAO6A9REQeMui7DImjgLg5mAhSgBwlMMhJAQIOkkJB066Tx2p/ve7/5xTie307e7k7597+m+5/OqunXP+Z3fOed7fehPfufR3B0REYmfRNQFiIhINBQAIiIxpQAQEYkpBYCISEwpAEREYkoBICISUwoAEZGYGjcAzOwOM9tnZpuKLPtvZuZmNiucNzP7vpltM7ONZnZJQd/lZrY1/Cyf3J8hIiJn60xGAD8Grjy90czmA1cAOwqalwKLw88K4LawbytwI3AZcClwo5m1lFK4iIiUJjVeB3d/xMwWFll0E/A1YHVB2zLgJx7cXrzezJrNbB7wQWCtu/cAmNlaglC5a6x9z5o1yxcuLLZrEREZzVNPPbXf3dvG6zduABRjZp8Adrn7c2ZWuKgd2Fkw3xW2jdY+poULF7Jhw4aJlCgiEltm9tqZ9DvrADCzOuAvgY8UW1ykzcdoL7b9FQSHj1iwYMHZliciImdoIlcBvRlYBDxnZq8CHcDTZjaX4F/28wv6dgC7x2gfwd1Xununu3e2tY07ghERkQk66wBw99+5+2x3X+juCwn+uF/i7q8Da4DPhVcDXQ70uvse4EHgI2bWEp78/UjYJiIiETmTy0DvAh4DzjezLjO7dozu9wPbgW3AD4DrAMKTv38DPBl+vj10QlhERKJhU/l9AJ2dna6TwCIiZ8fMnnL3zvH66U5gEZGYUgCIiMRUVQbA4b4Bblr7Es/uPBh1KSIiU1ZVBkAu73xv3Vaefu1A1KWIiExZVRkA9dng/rYj/YMRVyIiMnVVZQCkkwlq0gkFgIjIGKoyACAYBRzuUwCIiIymegMgk+Jw30DUZYiITFlVGwC1maQOAYmIjKFqA6Auk+SIDgGJiIyqqgPgkA4BiYiMqooDQCeBRUTGUsUBoHMAIiJjqeoAONo/yFR+2qmISJSqNgDqsynyDsdO5KIuRURkSqraAKjLJAE9DkJEZDRVGwD1meB5QDoRLCJSXNUGgEYAIiJjq9oAGHoiqB4HISJSXNUGwMkRgA4BiYgUNW4AmNkdZrbPzDYVtP2jmb1oZhvN7P+aWXPBsm+Y2TYz22Jmf1DQfmXYts3Mbpj8nzJc3dA5AB0CEhEp6kxGAD8GrjytbS1wobtfBLwEfAPAzN4GXAO8PVznVjNLmlkSuAVYCrwN+HTYt2zqssEIQCeBRUSKGzcA3P0RoOe0tl+6+9Bf1vVARzi9DPipu/e7+yvANuDS8LPN3be7+wngp2Hfshm6CkiHgEREipuMcwD/GXggnG4HdhYs6wrbRmsvm2TCSCeNI/06CSwiUkxJAWBmfwkMAncONRXp5mO0F9vmCjPbYGYburu7SymPbErPAxIRGc2EA8DMlgMfBz7jpx640wXML+jWAeweo30Ed1/p7p3u3tnW1jbR8gDIpBIc0iEgEZGiJhQAZnYl8HXgE+5+rGDRGuAaM8ua2SJgMfAE8CSw2MwWmVmG4ETxmtJKH186aToHICIyitR4HczsLuCDwCwz6wJuJLjqJwusNTOA9e7+p+7+vJndA7xAcGjoenfPhdv5M+BBIAnc4e7Pl+H3DJNJJjh0XOcARESKGTcA3P3TRZpvH6P/3wJ/W6T9fuD+s6quRNl0kl4FgIhIUVV7JzBATSqhABARGUVVB0A2HbwXWC+FEREZqboDIJVgIOf0DeSjLkVEZMqp6gCoSQWPg9BhIBGRkao6ALLp4OcpAERERqruAEgpAERERlPVAVCT1iEgEZHRVHUAaAQgIjK6qg4AjQBEREZX1QGQ0QhARGRUVR0ACTNq0noekIhIMVUdABC8E0ABICIyUgwCQM8DEhEpRgEgIhJTVR8AmVSCg8cUACIip6v6AKjROwFERIqq+gDIphIc6lMAiIicrvoDIJ2kfzBP30Au6lJERKaUqg+AmvBmMF0KKiIyXNUHQG34OIgDOhEsIjLMuAFgZneY2T4z21TQ1mpma81sa/jdErabmX3fzLaZ2UYzu6RgneVh/61mtrw8P2ekoecB9Rw9UaldiohMC2cyAvgxcOVpbTcA69x9MbAunAdYCiwOPyuA2yAIDOBG4DLgUuDGodAot9rM0AhAASAiUmjcAHD3R4Ce05qXAavC6VXAVQXtP/HAeqDZzOYBfwCsdfcedz8ArGVkqJRFrUYAIiJFTfQcwBx33wMQfs8O29uBnQX9usK20drLbugQ0AEFgIjIMJN9EtiKtPkY7SM3YLbCzDaY2Ybu7u6SC0omjGwqQY8OAYmIDDPRANgbHtoh/N4XtncB8wv6dQC7x2gfwd1Xununu3e2tbVNsLzh6jJJjQBERE4z0QBYAwxdybMcWF3Q/rnwaqDLgd7wENGDwEfMrCU8+fuRsK0iatJJenQZqIjIMKnxOpjZXcAHgVlm1kVwNc93gHvM7FpgB3B12P1+4KPANuAY8AUAd+8xs78Bngz7fdvdTz+xXDbZVIKeo/2V2p2IyLQwbgC4+6dHWbSkSF8Hrh9lO3cAd5xVdZOkNp3UVUAiIqep+juBIbgX4MBRHQISESkUiwCoSSc5PpDTA+FERArEIgB0M5iIyEjxCICMAkBE5HSxCICTdwPrZjARkZNiEQA6BCQiMlKsAkB3A4uInBKLAMimExgaAYiIFIpFACTMqMsm6T6iABARGRKLAACoy6TYf0SPgxARGRKLADh/3wP8YvBP+ZftH4abLoSN90RdkohI5MZ9FtB0d/6+B7ji5b8j7X1BQ+9O+MWXgumLPhVdYSIiEav6EcD7d9xKOt83vHHgOKz7djQFiYhMEVUfADP69xZf0NtV2UJERKaYqg+Aw9k5xRc0dVS2EBGRKabqA+DRBdcxkKgZ3piuhSXfiqYgEZEpoupPAm+ZvRSA97x6C00n9tFXN4+6pX+tE8AiEntVPwKAIARue9dqzu2/k7vff7/++IuIEJMAAKhJJ0gYdB/WzWAiIhCjADAzGrK6G1hEZEhJAWBmf25mz5vZJjO7y8xqzGyRmT1uZlvN7G4zy4R9s+H8tnD5wsn4AWejNpPUCEBEJDThADCzduBLQKe7XwgkgWuAfwBucvfFwAHg2nCVa4ED7n4ecFPYr6Jq00n2KQBERIDSDwGlgFozSwF1wB7gQ8C94fJVwFXh9LJwnnD5EjOzEvd/VuoyKQWAiEhowgHg7ruA7wI7CP7w9wJPAQfdfTDs1gW0h9PtwM5w3cGw/8yJ7n8iGrIpeo6cYDCXr+RuRUSmpFIOAbUQ/Kt+EXAOUA8sLdLVh1YZY1nhdleY2QYz29Dd3T3R8oqqzybJubNf7wUQESnpENCHgVfcvdvdB4D7gPcCzeEhIYAOYHc43QXMBwiXNwE9p2/U3Ve6e6e7d7a1tZVQ3kgNNUFZe3qPT+p2RUSmo1ICYAdwuZnVhcfylwAvAA8Dnwz7LAdWh9NrwnnC5Q+5+4gRQDnNyKYBeL23b5yeIiLVr5RzAI8TnMx9GvhduK2VwNeBr5rZNoJj/LeHq9wOzAzbvwrcUELdE9KQHRoBKABEREp6FpC73wjceFrzduDSIn37gKtL2V+patIJUglj7yEFgIhIbO4EhuBu4Bk1KY0ARESIWQAA1GdTOgksIkJsA0AjABGR2AVAQzbF3kN95PMVvQBJRGTKiWUADOScnmO6GUxE4i2WAQC6F0BEJH4BUKN7AUREIIYBMCMcAew+qCuBRCTeYhcAdZkkqYTRdeBY1KWIiEQqdgFgZjTVptnZoxGAiMRb7AIAgvMAO3o0AhCReItlADTWpNmpQ0AiEnOxDICm2jSH+wbpPT4QdSkiIpGJZQA0hpeC6kSwiMRZPAOgNngxjE4Ei0icxToANAIQkTiLZQDUpBJkUwm6DmgEICLxFcsAMLPgSiBdCioiMRbLAACYoXsBRCTmYhsATXVpdvQc03sBRCS2SgoAM2s2s3vN7EUz22xm7zGzVjNba2Zbw++WsK+Z2ffNbJuZbTSzSybnJ0xMS22G/sE8e/SCeBGJqVJHAN8D/s3dLwDeCWwGbgDWuftiYF04D7AUWBx+VgC3lbjvkjTXBVcCvdJ9NMoyREQiM+EAMLNG4APA7QDufsLdDwLLgFVht1XAVeH0MuAnHlgPNJvZvAlXXqKW+gwAr+w/ElUJIiKRKmUEcC7QDfzIzJ4xsx+aWT0wx933AITfs8P+7cDOgvW7wrZI1GeSZJIJXtYIQERiqpQASAGXALe5+8XAUU4d7inGirSNOANrZivMbIOZbeju7i6hvLGZGS31aV7ZrwAQkXgqJQC6gC53fzycv5cgEPYOHdoJv/cV9J9fsH4HsPv0jbr7SnfvdPfOtra2EsobX1NNmu3dOgQkIvE04QBw99eBnWZ2fti0BHgBWAMsD9uWA6vD6TXA58KrgS4HeocOFUWluT7DroPH6R/MRVmGiEgkUiWu/0XgTjPLANuBLxCEyj1mdi2wA7g67Hs/8FFgG3As7Buplro0eYedPcc4b/aMqMsREamokgLA3Z8FOossWlKkrwPXl7K/ydZcF1wJ9HL3UQWAiMRObO8EBmgNA2DbPp0HEJH4iXUAZFIJmmrTvPj64ahLERGpuFgHAEBrfYYX9xyKugwRkYqLfQDMrM+wff9RTgzmoy5FRKSiFAANGXJ5Z7seCSEiMRP7AJjVkAVgi84DiEjMxD4AWuoyJEwBICLxE/sASCaM1vqMAkBEYif2AQDBlUAv6EogEYkZBQCwzH7D/zm+Av+rZrjpQth4T9QliYiUXanPApr2zt/3AB/uvZlMoj9o6N0Jv/hSMH3Rp6IrTESkzGI/Anj/jlvJeP/wxoHjsO7b0RQkIlIhsQ+AGf17iy/o7apsISIiFRb7ADicnVN8QVNHZQsREamw2AfAowuuYyBRM7wxXQtLvhVNQSIiFRL7k8BbZi8F4PJXbqF5YB/99fOovfKvdQJYRKpe7EcAEITAynev5s39d/Ivl6zWH38RiQUFQCibSjKrIcNTrx2IuhQRkYpQABSY11TLU68dYDCnR0OLSPVTABQ4p7mWYydyekOYiMRCyQFgZkkze8bM/jWcX2Rmj5vZVjO728wyYXs2nN8WLl9Y6r4n2znNwdVAT77aE3ElIiLlNxkjgC8Dmwvm/wG4yd0XAweAa8P2a4ED7n4ecFPYb0qZUZOmqTbNhld1HkBEql9JAWBmHcDHgB+G8wZ8CLg37LIKuCqcXhbOEy5fEvafUuY21vDEKz24e9SliIiUVakjgJuBrwFDZ01nAgfdfTCc7wLaw+l2YCdAuLw37D+lnNNcQ/eRfl5941jUpYiIlNWEA8DMPg7sc/enCpuLdPUzWFa43RVmtsHMNnR3d0+0vAmb31oHwKNbK79vEZFKKmUE8D7gE2b2KvBTgkM/NwPNZjZ0h3EHsDuc7gLmA4TLm4ARZ1vdfaW7d7p7Z1tbWwnlTUxzbXAe4JGt+yu+bxGRSppwALj7N9y9w90XAtcAD7n7Z4CHgU+G3ZYDq8PpNeE84fKHfAoeaDcz5rfU8tjLbzCg+wFEpIqV4z6ArwNfNbNtBMf4bw/bbwdmhu1fBW4ow74nxYLWOo70D/LczoNRlyIiUjaT8jA4d/8V8KtwejtwaZE+fcDVk7G/cpvfWocZPLJ1P50LW6MuR0SkLHQncBE16SRzG2v41ZZ9UZciIlI2CoBRLJxZz8auXl7v7Yu6FBGRslAAjOLNbfUArN08yisjRUSmOQXAKFrrM7TWpXlw0+tRlyIiUhYKgFGYGYvaGnhs+xv0Hh+IuhwRkUmnABjDm9vqyeWdh17UYSARqT4KgDHMbayhsSbF6md2j99ZRGSaUQCMwcx4y5wZ/HrrfroP90ddjojIpFIAjOOCuTPIufOL5zQKEJHqogAYx8yGLHMas9z3dFfUpYiITCoFwBl4y5wZbNp9iC16V7CIVBEFwBl469xGUgnjf65/NepSREQmjQLgDNRmkiye3cB9T+/icJ/uCRCR6qAAOEMXdTRz7ESO+57eFXUpIiKTQgFwhuY21TC3sYZVv32VfH7KvcdGROSsKQDOwjvnN7F9/1E9IE5EqoIC4Cy8ZfYMmuvS/PND25iCb7MUETkrCoCzkEgY717Qwu929fJrvTReRKY5BcBZumDeDGbUpLj531/SKEBEpjUFwFlKJRL83sJWnt5xkAef17kAEZm+JhwAZjbfzB42s81m9ryZfTlsbzWztWa2NfxuCdvNzL5vZtvMbKOZXTJZP6LS3j6vkZn1Gf7+gc0M5PJRlyMiMiGljAAGgb9w97cClwPXm9nbgBuAde6+GFgXzgMsBRaHnxXAbSXsO1KJhPHe82by2hvHuHP9a1GXIyIyIRMOAHff4+5Ph9OHgc1AO7AMWBV2WwVcFU4vA37igfVAs5nNm3DlEVs0s54FrXV895cvsfeQXhwvItPPpJwDMLOFwMXA48Acd98DQUgAs8Nu7cDOgtW6wrZpycz4/fPb6BvI8Vdrno+6HBGRs1ZyAJhZA/Az4CvufmisrkXaRlxGY2YrzGyDmW3o7u4utbyyaq7LcOmiVh7Y9DoPPq+Xx4vI9FJSAJhZmuCP/53ufl/YvHfo0E74vS9s7wLmF6zeAYx4y4q7r3T3TnfvbGtrK6W8irhkQQttM7Lc8LON7NOhIBGZRkq5CsiA24HN7v5PBYvWAMvD6eXA6oL2z4VXA10O9A4dKprOkgnjyrfP5Uj/IF+5+1k9J0hEpo1SRgDvAz4LfMjMng0/HwW+A1xhZluBK8J5gPuB7cA24AfAdSXse0pprc/wgcVt/PblN7jl4W1RlyMickZSE13R3R+l+HF9gCVF+jtw/UT3N9W9/ZxGdh08zn9f+xKL5zRw5YXT9gInEYkJ3Qk8ScyMJRfMZl5TDV+5+1k27eqNuiQRkTEpACZRKpngY++YRyaV4HN3PMHL3UeiLklEZFQKgElWn01x1Tvb6R/I8Sc/WM+ON45FXZKISFEKgDJoqc9w1cXtHDo+yKdWPsa2fYejLklEZAQFQJnMasjyRxe3c7hvgP9422M89dqBqEsSERlGAVBGbTOyXP3u+SQM/uQH6/n5M3qhvIhMHQqAMmuqTfPJd3cwqyHLV+5+lhtXb+LEoB4hLSLRUwBUQF0mxR9d3M7FC5pZ9dhr/PGtv2HL6zovICLRUgBUSDJhfGBxGx97xzy27z/Kx//Hr7nl4W16oYyIREYBUGHnzW7gM5ctYOHMev7xwS1cefMj/GrLvvFXFBGZZAqACNRlUnz0HfP4w4vm0XP0BJ//0ZN8/o4n2Nh1MOrSRCRGJvwsICnduW0NLJhZx3M7e1n/yht84p9/w++f38YXlyzmkgUtUZcnIlVOARCxVCLBu9/UwoXtjTzX1cvjr/Tw8K2/5Z0dTXz2PQv5+EXzqEkng84b74F134beLmjqgCXfgos+Fe0PEJFpy4KHdE5NnZ2dvmHDhgmt+9Lew/y/jdPvdQMnBvO8sOcQm3b18sbREzTVpln2rnP4/IwnWPTYN7GB46c6p2vhD7+vEBCRYczsKXfvHK+fzgFMMZlUgnfNb+Yzly3gjy9uZ3Zjlv/9+A4yv/rb4X/8AQaOByMCEZEJ0CGgKcrMmN9ax/zWOvoHc7Q//kbRft7bxe6Dx2lvrq1whSIy3SkApoFsKsnh7Bwa+0e+eH5Xfibv/85DzGuq4bJFrfzeolYuWdDCebMbSCc1wBOR0SkApolHF1zHFS//Hen8qRfPDyRqeHT+f+U/WBu7Dh7n3zfv4+fP7gYgk0xw/twZXNjexNvPaeSt8xo5r62Bprp0VD9BRKYYBcA0sWX2UgDev+NWZvTv5XB2Do8uuI49s5fyLuBd85txdw4eH2DvoT66D/fTfaSfnz+zi7ue2HFyOy11ac6b3cCb24LPm2bW0d5SS0dzHY21KcxGe8uniFQbBcA0smX20pNBUIyZ0VKXoaUuwwVzgzZ353DfIPuP9HPg2AAHjp1gz8E+Xth9iKMncsPWr8skaW+upaOllnOaa5nXVMOshixtM4LPrIbgk0np0JJINah4AJjZlcD3gCTwQ3f/TqVriBMzo7E2TWPtyEM/fQM5eo8PcKhvgMN9gxw+Psjh/gFe2HOI9dt7OD6QK7JFaKxJ0TYjy8yGLE21aZpr0zSFn+a6YF/NdZmTbTNqUtRnUtSkExphiEwhFQ0AM0sCtwBXAF3Ak2a2xt1fqGQdEqhJJ6lJJ5nTWFN0+WAuz7ETufAzyLETOY6G38f6c+w6cJxX9h+lfyDH8YEcA7mx7ylJGNRmktRnUtRnU9RnkzRkg3Coy6ZoyCapy6SoTSfJphLUpJNk0wlqUsF3NjV8/uR3Qf900kgnEiQSChqZRiK6ybPSI4BLgW3uvh3AzH4KLAMUAFNQKpmgsTZRdPRQTC7v9A3k6B/MD/s+kcszMJhnIOfBdC7PicE8R/oGOXB0gMF8MD+Q8/A7T6m3JybNSCWDTzqRIJ1MBNPJRBASyUTR6VRBWyqRIJmA5NC3GYmEkUoE30kzkonwEy5LDi0Plw31P7U83J4F04mwzoSd+piBWbDMgETCCPIs+B7qc7IvRiJxqr9Z0G/oOxGOuoa2Y5xaPrSdoXYr2E6xWoZGcMF08J+1RnUl2ngP/OJLwX09AL07g3koewhUOgDagZ0F813AZRWuQcokmbDwX/albcfdyTsM5vPk8s5gzoPvvDOYz4+czzu5XDCfcyefd/Lu5PMMm88VtJ0YzHN8IHdyX0GfsP/JdSDvjo/yPTQ9de+lr6yhGBgKpaEGK2yjMDiGlg8F2akNDbWdWt+Gbf9k15PBdGpbBbs+uV87vZaCjZxe2+lBR5HlheueKttGto23HLjj4DeZnR/lJs8qC4Bi/1QY9v8fM1sBrABYsGDBhHd07qx6/ssHzp3w+iJnKl8QPCdDJp8n56eW5fLBJ18wXRhIubzjhSHDqSA8FTxBe9BWGEQAwwPr5LrDtuMj1g0C7PT2YBmc2v7QsmBPhdOE00GDe2Hb8L6EtQxfHsz4yeUjt0/YVrj9wr4Mq+X0bZ3amY9Y3yko/eR+C9dnxG899WMK/3AVe6JOsb7D/rMIv9t69o9cGYLDQWVW6QDoAuYXzHcAuws7uPtKYCUEzwKa6I5SyQQNuhFKRKa6mzqCwz6na+oo+64r/RfySWCxmS0yswxwDbCmwjWIiEwdS74VPNixULo2aC+zio4A3H3QzP4MeJDgMtA73P35StYgIjKlDB3nj8FVQLj7/cD9ld6viMiUddGnInmsuw6Si4jElAJARCSmFAAiIjGlABARiSkFgIhITCkARERiSgEgIhJTCgARkZgyL/YUoynCzLqB16KuYwJmAaM84anqxOm3gn5vNaum3/omd28br9OUDoDpysw2uHtn1HVUQpx+K+j3VrM4/dYhOgQkIhJTCgARkZhSAJTHyqgLqKA4/VbQ761mcfqtgM4BiIjElkYAIiIxpQCYJGY238weNrPNZva8mX056prKzcySZvaMmf1r1LWUm5k1m9m9ZvZi+N/xe6KuqZzM7M/D/x1vMrO7zKwm6pomk5ndYWb7zGxTQVurma01s63hd0uUNVaCAmDyDAJ/4e5vBS4Hrjezt0VcU7l9GdgcdREV8j3g39z9AuCdVPHvNrN24EtAp7tfSPD2vmuirWrS/Ri48rS2G4B17r4YWBfOVzUFwCRx9z3u/nQ4fZjgD0R7tFWVj5l1AB8Dfhh1LeVmZo3AB4DbAdz9hLsfjLaqsksBtWaWAuqA3RHXM6nc/RGg57TmZcCqcHoVcFVFi4qAAqAMzGwhcDHweLSVlNXNwNeAfNSFVMC5QDfwo/CQ1w/NrD7qosrF3XcB3wV2AHuAXnf/ZbRVVcQcd98DwT/ogNkR11N2CoBJZmYNwM+Ar7j7oajrKQcz+ziwz92firqWCkkBlwC3ufvFwFGq+PBAeOx7GbAIOAeoN7P/FG1VUg4KgElkZmmCP/53uvt9UddTRu8DPmFmrwI/BT5kZv8r2pLKqgvocvehEd29BIFQrT4MvOLu3e4+ANwHvDfimiphr5nNAwi/90VcT9kpACaJmRnBMeLN7v5PUddTTu7+DXfvcPeFBCcHH3L3qv0Xoru/Duw0s/PDpiXACxGWVG47gMvNrC783/USqvikd4E1wPJwejmwOsJaKiIVdQFV5H3AZ4HfmdmzYds33f3+CGuSyfNF4E4zywDbgS9EXE/ZuPvjZnYv8DTB1W3PUGV3yZrZXcAHgVlm1gXcCHwHuMfMriUIwaujq7AydCewiEhM6RCQiEhMKQBERGJKASAiElMKABGRmFIAiIjElAJARCSmFAAiIjGlABARian/D+hfAG5Faoo0AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Feerates:\t1.1531420689155165, 1.4085104512204296, 2.816548045571761, 11.10120050773006 \n", + "Times:\t\t874.010579873836, 479.615551452334, 60.00000000000001, 1.0000000000000004" + ] + }, + "execution_count": 108, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def build_estimator():\n", + " _feerates = [1.0]*10 + [1.1]*10 + [1.2]*10 + [1.5]*3000 + [2]*3000\\\n", + "+ [2.1]*3000 + [3]*10 + [4]*10 + [5]*10 + [6] + [7] + [10] + [100] + [200]*200\n", + " _total_weight = sum(np.array(_feerates)**ALPHA)\n", + " _network_mass_rate = bps * block_mass_limit\n", + " estimator = FeerateEstimator(total_weight=_total_weight, \n", + " inclusion_interval=avg_mass/_network_mass_rate)\n", + " \n", + " nr = _network_mass_rate\n", + " for i in range(len(_feerates)-1, -1, -1):\n", + " tw = sum(np.array(_feerates[:i])**ALPHA)\n", + " nr -= avg_mass\n", + " if nr <= 0:\n", + " print(\"net mass rate {}\", nr)\n", + " break\n", + " e = FeerateEstimator(total_weight=tw, \n", + " inclusion_interval=avg_mass/nr)\n", + " if e.feerate_to_time(1.0) < estimator.feerate_to_time(1.0):\n", + " # print(\"removing {}\".format(_feerates[i]))\n", + " estimator = e\n", + " else:\n", + " break\n", + " \n", + " return estimator\n", + "\n", + "estimator = build_estimator()\n", + "pred = estimator.calc_estimations()\n", + "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", + "y = estimator.feerate_to_time(x)\n", + "plt.figure()\n", + "plt.plot(x, y)\n", + "plt.fill_between(x, estimator.inclusion_interval, y2=y, alpha=0.5)\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "pred" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:gr]", + "language": "python", + "name": "conda-env-gr-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs new file mode 100644 index 000000000..5ef3579a5 --- /dev/null +++ b/mining/src/feerate/mod.rs @@ -0,0 +1,231 @@ +//! See the accompanying fee_estimation.ipynb Jupyter Notebook which details the reasoning +//! behind this fee estimator. + +use crate::block_template::selector::ALPHA; +use itertools::Itertools; +use std::fmt::Display; + +/// A type representing fee/mass of a transaction in `sompi/gram` units. +/// Given a feerate value recommendation, calculate the required fee by +/// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` +pub type Feerate = f64; + +#[derive(Clone, Copy, Debug)] +pub struct FeerateBucket { + pub feerate: f64, + pub estimated_seconds: f64, +} + +impl Display for FeerateBucket { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "({:.4}, {:.4}s)", self.feerate, self.estimated_seconds) + } +} + +#[derive(Clone, Debug)] +pub struct FeerateEstimations { + /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + /// + /// Note: for all buckets, feerate values represent fee/mass of a transaction in `sompi/gram` units. + /// Given a feerate value recommendation, calculate the required fee by + /// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` + pub priority_bucket: FeerateBucket, + + /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and + /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + /// between them, one can compose a complete feerate function on the client side. The API makes an effort + /// to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + pub normal_buckets: Vec, + + /// A vector of *low* priority feerate values. The first value of this vector is guaranteed to + /// exist and provide an estimation for sub-*hour* DAG inclusion. + pub low_buckets: Vec, +} + +impl FeerateEstimations { + pub fn ordered_buckets(&self) -> Vec { + std::iter::once(self.priority_bucket) + .chain(self.normal_buckets.iter().copied()) + .chain(self.low_buckets.iter().copied()) + .collect() + } +} + +impl Display for FeerateEstimations { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "(fee/mass, secs) priority: {}, ", self.priority_bucket)?; + write!(f, "normal: {}, ", self.normal_buckets.iter().format(", "))?; + write!(f, "low: {}", self.low_buckets.iter().format(", ")) + } +} + +pub struct FeerateEstimatorArgs { + pub network_blocks_per_second: u64, + pub maximum_mass_per_block: u64, +} + +impl FeerateEstimatorArgs { + pub fn new(network_blocks_per_second: u64, maximum_mass_per_block: u64) -> Self { + Self { network_blocks_per_second, maximum_mass_per_block } + } + + pub fn network_mass_per_second(&self) -> u64 { + self.network_blocks_per_second * self.maximum_mass_per_block + } +} + +#[derive(Debug, Clone)] +pub struct FeerateEstimator { + /// The total probability weight of current mempool ready transactions, i.e., `Σ_{tx in mempool}(tx.fee/tx.mass)^alpha`. + /// Note that some estimators might consider a reduced weight which excludes outliers. See [`Frontier::build_feerate_estimator`] + total_weight: f64, + + /// The amortized time **in seconds** between transactions, given the current transaction masses present in the mempool. Or in + /// other words, the inverse of the transaction inclusion rate. For instance, if the average transaction mass is 2500 grams, + /// the block mass limit is 500,000 and the network has 10 BPS, then this number would be 1/2000 seconds. + inclusion_interval: f64, +} + +impl FeerateEstimator { + pub fn new(total_weight: f64, inclusion_interval: f64) -> Self { + assert!(total_weight >= 0.0); + assert!((0f64..1f64).contains(&inclusion_interval)); + Self { total_weight, inclusion_interval } + } + + pub(crate) fn feerate_to_time(&self, feerate: f64) -> f64 { + let (c1, c2) = (self.inclusion_interval, self.total_weight); + c1 * c2 / feerate.powi(ALPHA) + c1 + } + + fn time_to_feerate(&self, time: f64) -> f64 { + let (c1, c2) = (self.inclusion_interval, self.total_weight); + assert!(c1 < time, "{c1}, {time}"); + ((c1 * c2 / time) / (1f64 - c1 / time)).powf(1f64 / ALPHA as f64) + } + + /// The antiderivative function of [`feerate_to_time`] excluding the constant shift `+ c1` + #[inline] + fn feerate_to_time_antiderivative(&self, feerate: f64) -> f64 { + let (c1, c2) = (self.inclusion_interval, self.total_weight); + c1 * c2 / (-2f64 * feerate.powi(ALPHA - 1)) + } + + /// Returns the feerate value for which the integral area is `frac` of the total area between `lower` and `upper`. + fn quantile(&self, lower: f64, upper: f64, frac: f64) -> f64 { + assert!((0f64..=1f64).contains(&frac)); + assert!(0.0 < lower && lower <= upper, "{lower}, {upper}"); + let (c1, c2) = (self.inclusion_interval, self.total_weight); + if c1 == 0.0 || c2 == 0.0 { + // if c1 · c2 == 0.0, the integral area is empty, so we simply return `lower` + return lower; + } + let z1 = self.feerate_to_time_antiderivative(lower); + let z2 = self.feerate_to_time_antiderivative(upper); + // Get the total area corresponding to `frac` of the integral area between `lower` and `upper` + // which can be expressed as z1 + frac * (z2 - z1) + let z = frac * z2 + (1f64 - frac) * z1; + // Calc the x value (feerate) corresponding to said area + ((c1 * c2) / (-2f64 * z)).powf(1f64 / (ALPHA - 1) as f64) + } + + pub fn calc_estimations(&self, minimum_standard_feerate: f64) -> FeerateEstimations { + let min = minimum_standard_feerate; + // Choose `high` such that it provides sub-second waiting time + let high = self.time_to_feerate(1f64).max(min); + // Choose `low` feerate such that it provides sub-hour waiting time AND it covers (at least) the 0.25 quantile + let low = self.time_to_feerate(3600f64).max(self.quantile(min, high, 0.25)); + // Choose `normal` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.66 quantile between low and high. + let normal = self.time_to_feerate(60f64).max(self.quantile(low, high, 0.66)); + // Choose an additional point between normal and low + let mid = self.time_to_feerate(1800f64).max(self.quantile(min, high, 0.5)); + /* Intuition for the above: + 1. The quantile calculations make sure that we return interesting points on the `feerate_to_time` curve. + 2. They also ensure that the times don't diminish too high if small increments to feerate would suffice + to cover large fractions of the integral area (reflecting the position within the waiting-time distribution) + */ + FeerateEstimations { + priority_bucket: FeerateBucket { feerate: high, estimated_seconds: self.feerate_to_time(high) }, + normal_buckets: vec![ + FeerateBucket { feerate: normal, estimated_seconds: self.feerate_to_time(normal) }, + FeerateBucket { feerate: mid, estimated_seconds: self.feerate_to_time(mid) }, + ], + low_buckets: vec![FeerateBucket { feerate: low, estimated_seconds: self.feerate_to_time(low) }], + } + } +} + +#[derive(Clone, Debug)] +pub struct FeeEstimateVerbose { + pub estimations: FeerateEstimations, + + pub mempool_ready_transactions_count: u64, + pub mempool_ready_transactions_total_mass: u64, + pub network_mass_per_second: u64, + + pub next_block_template_feerate_min: f64, + pub next_block_template_feerate_median: f64, + pub next_block_template_feerate_max: f64, +} + +#[cfg(test)] +mod tests { + use super::*; + use itertools::Itertools; + + #[test] + fn test_feerate_estimations() { + let estimator = FeerateEstimator { total_weight: 1002283.659, inclusion_interval: 0.004f64 }; + let estimations = estimator.calc_estimations(1.0); + let buckets = estimations.ordered_buckets(); + for (i, j) in buckets.into_iter().tuple_windows() { + assert!(i.feerate >= j.feerate); + } + dbg!(estimations); + } + + #[test] + fn test_min_feerate_estimations() { + let estimator = FeerateEstimator { total_weight: 0.00659, inclusion_interval: 0.004f64 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + println!("{estimations}"); + let buckets = estimations.ordered_buckets(); + assert!(buckets.last().unwrap().feerate >= minimum_feerate); + for (i, j) in buckets.into_iter().tuple_windows() { + assert!(i.feerate >= j.feerate); + assert!(i.estimated_seconds <= j.estimated_seconds); + } + } + + #[test] + fn test_zero_values() { + let estimator = FeerateEstimator { total_weight: 0.0, inclusion_interval: 0.0 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + let buckets = estimations.ordered_buckets(); + for bucket in buckets { + assert_eq!(minimum_feerate, bucket.feerate); + assert_eq!(0.0, bucket.estimated_seconds); + } + + let estimator = FeerateEstimator { total_weight: 0.0, inclusion_interval: 0.1 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + let buckets = estimations.ordered_buckets(); + for bucket in buckets { + assert_eq!(minimum_feerate, bucket.feerate); + assert_eq!(estimator.inclusion_interval, bucket.estimated_seconds); + } + + let estimator = FeerateEstimator { total_weight: 0.1, inclusion_interval: 0.0 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + let buckets = estimations.ordered_buckets(); + for bucket in buckets { + assert_eq!(minimum_feerate, bucket.feerate); + assert_eq!(0.0, bucket.estimated_seconds); + } + } +} diff --git a/mining/src/lib.rs b/mining/src/lib.rs index 2986577ef..745fb63f9 100644 --- a/mining/src/lib.rs +++ b/mining/src/lib.rs @@ -8,12 +8,17 @@ use mempool::tx::Priority; mod block_template; pub(crate) mod cache; pub mod errors; +pub mod feerate; pub mod manager; mod manager_tests; pub mod mempool; pub mod model; pub mod monitor; +// Exposed for benchmarks +pub use block_template::{policy::Policy, selector::RebalancingWeightedTransactionSelector}; +pub use mempool::model::frontier::{feerate_key::FeerateTransactionKey, search_tree::SearchTree, Frontier}; + #[cfg(test)] pub mod testutils; diff --git a/mining/src/manager.rs b/mining/src/manager.rs index c3bf61261..8bd46c3d8 100644 --- a/mining/src/manager.rs +++ b/mining/src/manager.rs @@ -2,6 +2,7 @@ use crate::{ block_template::{builder::BlockTemplateBuilder, errors::BuilderError}, cache::BlockTemplateCache, errors::MiningManagerResult, + feerate::{FeeEstimateVerbose, FeerateEstimations, FeerateEstimatorArgs}, mempool::{ config::Config, model::tx::{MempoolTransaction, TransactionPostValidation, TransactionPreValidation, TxRemovalReason}, @@ -12,7 +13,6 @@ use crate::{ Mempool, }, model::{ - candidate_tx::CandidateTransaction, owner_txs::{GroupedOwnerTransactions, ScriptPublicKeySet}, topological_sort::IntoIterTopologically, tx_insert::TransactionInsertion, @@ -26,7 +26,7 @@ use kaspa_consensus_core::{ args::{TransactionValidationArgs, TransactionValidationBatchArgs}, ConsensusApi, }, - block::{BlockTemplate, TemplateBuildMode}, + block::{BlockTemplate, TemplateBuildMode, TemplateTransactionSelector}, coinbase::MinerData, errors::{block::RuleError as BlockRuleError, tx::TxRuleError}, tx::{MutableTransaction, Transaction, TransactionId, TransactionOutput}, @@ -107,14 +107,14 @@ impl MiningManager { loop { attempts += 1; - let transactions = self.block_candidate_transactions(); - let block_template_builder = BlockTemplateBuilder::new(self.config.maximum_mass_per_block); + let selector = self.build_selector(); + let block_template_builder = BlockTemplateBuilder::new(); let build_mode = if attempts < self.config.maximum_build_block_template_attempts { TemplateBuildMode::Standard } else { TemplateBuildMode::Infallible }; - match block_template_builder.build_block_template(consensus, miner_data, transactions, build_mode) { + match block_template_builder.build_block_template(consensus, miner_data, selector, build_mode) { Ok(block_template) => { let block_template = cache_lock.set_immutable_cached_template(block_template); match attempts { @@ -197,8 +197,37 @@ impl MiningManager { } } - pub(crate) fn block_candidate_transactions(&self) -> Vec { - self.mempool.read().block_candidate_transactions() + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier + pub(crate) fn build_selector(&self) -> Box { + self.mempool.read().build_selector() + } + + /// Returns realtime feerate estimations based on internal mempool state + pub(crate) fn get_realtime_feerate_estimations(&self) -> FeerateEstimations { + let args = FeerateEstimatorArgs::new(self.config.network_blocks_per_second, self.config.maximum_mass_per_block); + let estimator = self.mempool.read().build_feerate_estimator(args); + estimator.calc_estimations(self.config.minimum_feerate()) + } + + /// Returns realtime feerate estimations based on internal mempool state with additional verbose data + pub(crate) fn get_realtime_feerate_estimations_verbose(&self) -> FeeEstimateVerbose { + let args = FeerateEstimatorArgs::new(self.config.network_blocks_per_second, self.config.maximum_mass_per_block); + let network_mass_per_second = args.network_mass_per_second(); + let mempool_read = self.mempool.read(); + let estimator = mempool_read.build_feerate_estimator(args); + let ready_transactions_count = mempool_read.ready_transaction_count(); + let ready_transaction_total_mass = mempool_read.ready_transaction_total_mass(); + drop(mempool_read); + FeeEstimateVerbose { + estimations: estimator.calc_estimations(self.config.minimum_feerate()), + network_mass_per_second, + mempool_ready_transactions_count: ready_transactions_count as u64, + mempool_ready_transactions_total_mass: ready_transaction_total_mass, + // TODO: Next PR + next_block_template_feerate_min: -1.0, + next_block_template_feerate_median: -1.0, + next_block_template_feerate_max: -1.0, + } } /// Clears the block template cache, forcing the next call to get_block_template to build a new block template. @@ -209,7 +238,7 @@ impl MiningManager { #[cfg(test)] pub(crate) fn block_template_builder(&self) -> BlockTemplateBuilder { - BlockTemplateBuilder::new(self.config.maximum_mass_per_block) + BlockTemplateBuilder::new() } /// validate_and_insert_transaction validates the given transaction, and @@ -799,6 +828,16 @@ impl MiningManagerProxy { consensus.clone().spawn_blocking(move |c| self.inner.get_block_template(c, &miner_data)).await } + /// Returns realtime feerate estimations based on internal mempool state + pub async fn get_realtime_feerate_estimations(self) -> FeerateEstimations { + spawn_blocking(move || self.inner.get_realtime_feerate_estimations()).await.unwrap() + } + + /// Returns realtime feerate estimations based on internal mempool state with additional verbose data + pub async fn get_realtime_feerate_estimations_verbose(self) -> FeeEstimateVerbose { + spawn_blocking(move || self.inner.get_realtime_feerate_estimations_verbose()).await.unwrap() + } + /// Validates a transaction and adds it to the set of known transactions that have not yet been /// added to any block. /// diff --git a/mining/src/manager_tests.rs b/mining/src/manager_tests.rs index a308e5657..0a2f04faf 100644 --- a/mining/src/manager_tests.rs +++ b/mining/src/manager_tests.rs @@ -7,9 +7,10 @@ mod tests { mempool::{ config::{Config, DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE}, errors::RuleError, + model::frontier::selectors::TakeAllSelector, tx::{Orphan, Priority, RbfPolicy}, }, - model::{candidate_tx::CandidateTransaction, tx_query::TransactionQuery}, + model::tx_query::TransactionQuery, testutils::consensus_mock::ConsensusMock, MiningCounters, }; @@ -1095,7 +1096,7 @@ mod tests { // Collect all parent transactions for the next block template. // They are ready since they have no parents in the mempool. - let transactions = mining_manager.block_candidate_transactions(); + let transactions = mining_manager.build_selector().select_transactions(); assert_eq!( TX_PAIRS_COUNT, transactions.len(), @@ -1103,7 +1104,7 @@ mod tests { ); parent_txs.iter().for_each(|x| { assert!( - transactions.iter().any(|tx| tx.tx.id() == x.id()), + transactions.iter().any(|tx| tx.id() == x.id()), "the parent transaction {} should be candidate for the next block template", x.id() ); @@ -1119,8 +1120,9 @@ mod tests { consensus: &dyn ConsensusApi, address_prefix: Prefix, mining_manager: &MiningManager, - transactions: Vec, + transactions: Vec, ) { + let transactions = transactions.into_iter().map(Arc::new).collect::>(); for _ in 0..4 { // Run a few times to get more randomness compare_modified_template_to_built( @@ -1187,7 +1189,7 @@ mod tests { consensus: &dyn ConsensusApi, address_prefix: Prefix, mining_manager: &MiningManager, - transactions: Vec, + transactions: Vec>, first_op: OpType, second_op: OpType, ) { @@ -1196,7 +1198,12 @@ mod tests { // Build a fresh template for coinbase2 as a reference let builder = mining_manager.block_template_builder(); - let result = builder.build_block_template(consensus, &miner_data_2, transactions, TemplateBuildMode::Standard); + let result = builder.build_block_template( + consensus, + &miner_data_2, + Box::new(TakeAllSelector::new(transactions)), + TemplateBuildMode::Standard, + ); assert!(result.is_ok(), "build block template failed for miner data 2"); let expected_template = result.unwrap(); diff --git a/mining/src/mempool/config.rs b/mining/src/mempool/config.rs index aecbc0711..419a4362a 100644 --- a/mining/src/mempool/config.rs +++ b/mining/src/mempool/config.rs @@ -1,7 +1,6 @@ use kaspa_consensus_core::constants::TX_VERSION; -pub(crate) const DEFAULT_MAXIMUM_TRANSACTION_COUNT: u64 = 1_000_000; -pub(crate) const DEFAULT_MAXIMUM_READY_TRANSACTION_COUNT: u64 = 50_000; +pub(crate) const DEFAULT_MAXIMUM_TRANSACTION_COUNT: u32 = 1_000_000; pub(crate) const DEFAULT_MAXIMUM_BUILD_BLOCK_TEMPLATE_ATTEMPTS: u64 = 5; pub(crate) const DEFAULT_TRANSACTION_EXPIRE_INTERVAL_SECONDS: u64 = 60; @@ -29,8 +28,7 @@ pub(crate) const DEFAULT_MAXIMUM_STANDARD_TRANSACTION_VERSION: u16 = TX_VERSION; #[derive(Clone, Debug)] pub struct Config { - pub maximum_transaction_count: u64, - pub maximum_ready_transaction_count: u64, + pub maximum_transaction_count: u32, pub maximum_build_block_template_attempts: u64, pub transaction_expire_interval_daa_score: u64, pub transaction_expire_scan_interval_daa_score: u64, @@ -47,13 +45,13 @@ pub struct Config { pub minimum_relay_transaction_fee: u64, pub minimum_standard_transaction_version: u16, pub maximum_standard_transaction_version: u16, + pub network_blocks_per_second: u64, } impl Config { #[allow(clippy::too_many_arguments)] pub fn new( - maximum_transaction_count: u64, - maximum_ready_transaction_count: u64, + maximum_transaction_count: u32, maximum_build_block_template_attempts: u64, transaction_expire_interval_daa_score: u64, transaction_expire_scan_interval_daa_score: u64, @@ -70,10 +68,10 @@ impl Config { minimum_relay_transaction_fee: u64, minimum_standard_transaction_version: u16, maximum_standard_transaction_version: u16, + network_blocks_per_second: u64, ) -> Self { Self { maximum_transaction_count, - maximum_ready_transaction_count, maximum_build_block_template_attempts, transaction_expire_interval_daa_score, transaction_expire_scan_interval_daa_score, @@ -90,6 +88,7 @@ impl Config { minimum_relay_transaction_fee, minimum_standard_transaction_version, maximum_standard_transaction_version, + network_blocks_per_second, } } @@ -98,7 +97,6 @@ impl Config { pub const fn build_default(target_milliseconds_per_block: u64, relay_non_std_transactions: bool, max_block_mass: u64) -> Self { Self { maximum_transaction_count: DEFAULT_MAXIMUM_TRANSACTION_COUNT, - maximum_ready_transaction_count: DEFAULT_MAXIMUM_READY_TRANSACTION_COUNT, maximum_build_block_template_attempts: DEFAULT_MAXIMUM_BUILD_BLOCK_TEMPLATE_ATTEMPTS, transaction_expire_interval_daa_score: DEFAULT_TRANSACTION_EXPIRE_INTERVAL_SECONDS * 1000 / target_milliseconds_per_block, transaction_expire_scan_interval_daa_score: DEFAULT_TRANSACTION_EXPIRE_SCAN_INTERVAL_SECONDS * 1000 @@ -118,11 +116,18 @@ impl Config { minimum_relay_transaction_fee: DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, minimum_standard_transaction_version: DEFAULT_MINIMUM_STANDARD_TRANSACTION_VERSION, maximum_standard_transaction_version: DEFAULT_MAXIMUM_STANDARD_TRANSACTION_VERSION, + network_blocks_per_second: 1000 / target_milliseconds_per_block, } } pub fn apply_ram_scale(mut self, ram_scale: f64) -> Self { - self.maximum_transaction_count = (self.maximum_transaction_count as f64 * ram_scale.min(1.0)) as u64; // Allow only scaling down + self.maximum_transaction_count = (self.maximum_transaction_count as f64 * ram_scale.min(1.0)) as u32; // Allow only scaling down self } + + /// Returns the minimum standard fee/mass ratio currently required by the mempool + pub(crate) fn minimum_feerate(&self) -> f64 { + // The parameter minimum_relay_transaction_fee is in sompi/kg units so divide by 1000 to get sompi/gram + self.minimum_relay_transaction_fee as f64 / 1000.0 + } } diff --git a/mining/src/mempool/mod.rs b/mining/src/mempool/mod.rs index 1f63a3f44..e5cd7dbeb 100644 --- a/mining/src/mempool/mod.rs +++ b/mining/src/mempool/mod.rs @@ -1,6 +1,6 @@ use crate::{ + feerate::{FeerateEstimator, FeerateEstimatorArgs}, model::{ - candidate_tx::CandidateTransaction, owner_txs::{GroupedOwnerTransactions, ScriptPublicKeySet}, tx_query::TransactionQuery, }, @@ -12,7 +12,10 @@ use self::{ model::{accepted_transactions::AcceptedTransactions, orphan_pool::OrphanPool, pool::Pool, transactions_pool::TransactionsPool}, tx::Priority, }; -use kaspa_consensus_core::tx::{MutableTransaction, TransactionId}; +use kaspa_consensus_core::{ + block::TemplateTransactionSelector, + tx::{MutableTransaction, TransactionId}, +}; use kaspa_core::time::Stopwatch; use std::sync::Arc; @@ -112,9 +115,23 @@ impl Mempool { count } - pub(crate) fn block_candidate_transactions(&self) -> Vec { - let _sw = Stopwatch::<10>::with_threshold("block_candidate_transactions op"); - self.transaction_pool.all_ready_transactions() + pub(crate) fn ready_transaction_count(&self) -> usize { + self.transaction_pool.ready_transaction_count() + } + + pub(crate) fn ready_transaction_total_mass(&self) -> u64 { + self.transaction_pool.ready_transaction_total_mass() + } + + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier + pub(crate) fn build_selector(&self) -> Box { + let _sw = Stopwatch::<10>::with_threshold("build_selector op"); + self.transaction_pool.build_selector() + } + + /// Builds a feerate estimator based on internal state of the ready transactions frontier + pub(crate) fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { + self.transaction_pool.build_feerate_estimator(args) } pub(crate) fn all_transaction_ids_with_priority(&self, priority: Priority) -> Vec { diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs new file mode 100644 index 000000000..e8d2b54ab --- /dev/null +++ b/mining/src/mempool/model/frontier.rs @@ -0,0 +1,454 @@ +use crate::{ + feerate::{FeerateEstimator, FeerateEstimatorArgs}, + model::candidate_tx::CandidateTransaction, + Policy, RebalancingWeightedTransactionSelector, +}; + +use feerate_key::FeerateTransactionKey; +use kaspa_consensus_core::block::TemplateTransactionSelector; +use kaspa_core::trace; +use rand::{distributions::Uniform, prelude::Distribution, Rng}; +use search_tree::SearchTree; +use selectors::{SequenceSelector, SequenceSelectorInput, TakeAllSelector}; +use std::collections::HashSet; + +pub(crate) mod feerate_key; +pub(crate) mod search_tree; +pub(crate) mod selectors; + +/// If the frontier contains less than 4x the block mass limit, we consider +/// inplace sampling to be less efficient (due to collisions) and thus use +/// the rebalancing selector +const COLLISION_FACTOR: u64 = 4; + +/// Multiplication factor for in-place sampling. We sample 20% more than the +/// hard limit in order to allow the SequenceSelector to compensate for consensus rejections. +const MASS_LIMIT_FACTOR: f64 = 1.2; + +/// A rough estimation for the average transaction mass. The usage is a non-important edge case +/// hence we just throw this here (as oppose to performing an accurate estimation) +const TYPICAL_TX_MASS: f64 = 2000.0; + +/// Management of the transaction pool frontier, that is, the set of transactions in +/// the transaction pool which have no mempool ancestors and are essentially ready +/// to enter the next block template. +#[derive(Default)] +pub struct Frontier { + /// Frontier transactions sorted by feerate order and searchable for weight sampling + search_tree: SearchTree, + + /// Total masses: Σ_{tx in frontier} tx.mass + total_mass: u64, +} + +impl Frontier { + pub fn total_weight(&self) -> f64 { + self.search_tree.total_weight() + } + + pub fn total_mass(&self) -> u64 { + self.total_mass + } + + pub fn len(&self) -> usize { + self.search_tree.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { + let mass = key.mass; + if self.search_tree.insert(key) { + self.total_mass += mass; + true + } else { + false + } + } + + pub fn remove(&mut self, key: &FeerateTransactionKey) -> bool { + let mass = key.mass; + if self.search_tree.remove(key) { + self.total_mass -= mass; + true + } else { + false + } + } + + /// Samples the frontier in-place based on the provided policy and returns a SequenceSelector. + /// + /// This sampling algorithm should be used when frontier total mass is high enough compared to + /// policy mass limit so that the probability of sampling collisions remains low. + /// + /// Convergence analysis: + /// 1. Based on the above we can safely assume that `k << n`, where `n` is the total number of frontier items + /// and `k` is the number of actual samples (since `desired_mass << total_mass` and mass per item is bounded) + /// 2. Indeed, if the weight distribution is not too spread (i.e., `max(weights) = O(min(weights))`), `k << n` means + /// that the probability of collisions is low enough and the sampling process will converge in `O(k log(n))` w.h.p. + /// 3. It remains to deal with the case where the weight distribution is highly biased. The process implemented below + /// keeps track of the top-weight element. If the distribution is highly biased, this element will be sampled with + /// sufficient probability (in constant time). Following each sampling collision we search for a consecutive range of + /// top elements which were already sampled and narrow the sampling space to exclude them all. We do this by computing + /// the prefix weight up to the top most item which wasn't sampled yet (inclusive) and then continue the sampling process + /// over the narrowed space. This process is repeated until acquiring the desired mass. + /// 4. Numerical stability. Naively, one would simply subtract `total_weight -= top.weight` in order to narrow the sampling + /// space. However, if `top.weight` is much larger than the remaining weight, the above f64 subtraction will yield a number + /// close or equal to zero. We fix this by implementing a `log(n)` prefix weight operation. + /// 5. Q. Why not just use u64 weights? + /// A. The current weight calculation is `feerate^alpha` with `alpha=3`. Using u64 would mean that the feerate space + /// is limited to a range of size `(2^64)^(1/3) = ~2^21 = ~2M`. Already with current usages, the feerate can vary + /// from `~1/50` (2000 sompi for a transaction with 100K storage mass), to `5M` (100 KAS fee for a transaction with + /// 2000 mass = 100·100_000_000/2000), resulting in a range of size 250M (`5M/(1/50)`). + /// By using floating point arithmetics we gain the adjustment of the probability space to the accuracy level required for + /// current samples. And if the space is highly biased, the repeated elimination of top items and the prefix weight computation + /// will readjust it. + pub fn sample_inplace(&self, rng: &mut R, policy: &Policy, _collisions: &mut u64) -> SequenceSelectorInput + where + R: Rng + ?Sized, + { + debug_assert!(!self.search_tree.is_empty(), "expected to be called only if not empty"); + + // Sample 20% more than the hard limit in order to allow the SequenceSelector to + // compensate for consensus rejections. + // Note: this is a soft limit which is why the loop below might pass it if the + // next sampled transaction happens to cross the bound + let desired_mass = (policy.max_block_mass as f64 * MASS_LIMIT_FACTOR) as u64; + + let mut distr = Uniform::new(0f64, self.total_weight()); + let mut down_iter = self.search_tree.descending_iter(); + let mut top = down_iter.next().unwrap(); + let mut cache = HashSet::new(); + let mut sequence = SequenceSelectorInput::default(); + let mut total_selected_mass: u64 = 0; + let mut collisions = 0; + + // The sampling process is converging so the cache will eventually hold all entries, which guarantees loop exit + 'outer: while cache.len() < self.search_tree.len() && total_selected_mass <= desired_mass { + let query = distr.sample(rng); + let item = { + let mut item = self.search_tree.search(query); + while !cache.insert(item.tx.id()) { + collisions += 1; + // Try to narrow the sampling space in order to reduce further sampling collisions + if cache.contains(&top.tx.id()) { + loop { + match down_iter.next() { + Some(next) => top = next, + None => break 'outer, + } + // Loop until finding a top item which was not sampled yet + if !cache.contains(&top.tx.id()) { + break; + } + } + let remaining_weight = self.search_tree.prefix_weight(top); + distr = Uniform::new(0f64, remaining_weight); + } + let query = distr.sample(rng); + item = self.search_tree.search(query); + } + item + }; + sequence.push(item.tx.clone(), item.mass); + total_selected_mass += item.mass; // Max standard mass + Mempool capacity bound imply this will not overflow + } + trace!("[mempool frontier sample inplace] collisions: {collisions}, cache: {}", cache.len()); + *_collisions += collisions; + sequence + } + + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier. + /// + /// The logic is divided into three cases: + /// 1. The frontier is small and can fit entirely into a block: perform no sampling and return + /// a TakeAllSelector + /// 2. The frontier has at least ~4x the capacity of a block: expected collision rate is low, perform + /// in-place k*log(n) sampling and return a SequenceSelector + /// 3. The frontier has 1-4x capacity of a block. In this case we expect a high collision rate while + /// the number of overall transactions is still low, so we take all of the transactions and use the + /// rebalancing weighted selector (performing the actual sampling out of the mempool lock) + /// + /// The above thresholds were selected based on benchmarks. Overall, this dynamic selection provides + /// full transaction selection in less than 150 µs even if the frontier has 1M entries (!!). See mining/benches + /// for more details. + pub fn build_selector(&self, policy: &Policy) -> Box { + if self.total_mass <= policy.max_block_mass { + Box::new(TakeAllSelector::new(self.search_tree.ascending_iter().map(|k| k.tx.clone()).collect())) + } else if self.total_mass > policy.max_block_mass * COLLISION_FACTOR { + let mut rng = rand::thread_rng(); + Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, policy, &mut 0), policy.clone())) + } else { + Box::new(RebalancingWeightedTransactionSelector::new( + policy.clone(), + self.search_tree.ascending_iter().cloned().map(CandidateTransaction::from_key).collect(), + )) + } + } + + /// Exposed for benchmarking purposes + pub fn build_selector_sample_inplace(&self, _collisions: &mut u64) -> Box { + let mut rng = rand::thread_rng(); + let policy = Policy::new(500_000); + Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, &policy, _collisions), policy)) + } + + /// Exposed for benchmarking purposes + pub fn build_selector_take_all(&self) -> Box { + Box::new(TakeAllSelector::new(self.search_tree.ascending_iter().map(|k| k.tx.clone()).collect())) + } + + /// Exposed for benchmarking purposes + pub fn build_rebalancing_selector(&self) -> Box { + Box::new(RebalancingWeightedTransactionSelector::new( + Policy::new(500_000), + self.search_tree.ascending_iter().cloned().map(CandidateTransaction::from_key).collect(), + )) + } + + /// Builds a feerate estimator based on internal state of the ready transactions frontier + pub fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { + let average_transaction_mass = match self.len() { + 0 => TYPICAL_TX_MASS, + n => self.total_mass() as f64 / n as f64, + }; + let bps = args.network_blocks_per_second as f64; + let mut mass_per_block = args.maximum_mass_per_block as f64; + let mut inclusion_interval = average_transaction_mass / (mass_per_block * bps); + let mut estimator = FeerateEstimator::new(self.total_weight(), inclusion_interval); + + // Search for better estimators by possibly removing extremely high outliers + let mut down_iter = self.search_tree.descending_iter().skip(1); + loop { + // Update values for the next iteration. In order to remove the outlier from the + // total weight, we must compensate by capturing a block slot. + mass_per_block -= average_transaction_mass; + if mass_per_block <= average_transaction_mass { + // Out of block slots, break + break; + } + + // Re-calc the inclusion interval based on the new block "capacity". + // Note that inclusion_interval < 1.0 as required by the estimator, since mass_per_block > average_transaction_mass (by condition above) and bps >= 1 + inclusion_interval = average_transaction_mass / (mass_per_block * bps); + + // Compute the weight up to, and including, current key (or use zero weight if next is none) + let next = down_iter.next(); + let prefix_weight = next.map(|key| self.search_tree.prefix_weight(key)).unwrap_or_default(); + let pending_estimator = FeerateEstimator::new(prefix_weight, inclusion_interval); + + // Test the pending estimator vs. the current one + if pending_estimator.feerate_to_time(1.0) < estimator.feerate_to_time(1.0) { + estimator = pending_estimator; + } else { + // The pending estimator is no better, break. Indicates that the reduction in + // network mass per second is more significant than the removed weight + break; + } + + if next.is_none() { + break; + } + } + estimator + } +} + +#[cfg(test)] +mod tests { + use super::*; + use feerate_key::tests::build_feerate_key; + use itertools::Itertools; + use rand::thread_rng; + use std::collections::HashMap; + + #[test] + pub fn test_highly_irregular_sampling() { + let mut rng = thread_rng(); + let cap = 1000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let mut fee: u64 = if i % (cap as u64 / 100) == 0 { 1000000 } else { rng.gen_range(1..10000) }; + if i == 0 { + // Add an extremely large fee in order to create extremely high variance + fee = 100_000_000 * 1_000_000; // 1M KAS + } + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let mut frontier = Frontier::default(); + for item in map.values().cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let _sample = frontier.sample_inplace(&mut rng, &Policy::new(500_000), &mut 0); + } + + #[test] + pub fn test_mempool_sampling_small() { + let mut rng = thread_rng(); + let cap = 2000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = rng.gen_range(1..1000000); + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let mut frontier = Frontier::default(); + for item in map.values().cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let mut selector = frontier.build_selector(&Policy::new(500_000)); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_rebalancing_selector(); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector_sample_inplace(&mut 0); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector_take_all(); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector(&Policy::new(500_000)); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + } + + #[test] + pub fn test_total_mass_tracking() { + let mut rng = thread_rng(); + let cap = 10000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = if i % (cap as u64 / 100) == 0 { 1000000 } else { rng.gen_range(1..10000) }; + let mass: u64 = rng.gen_range(1..100000); // Use distinct mass values to challenge the test + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let len = cap / 2; + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let prev_total_mass = frontier.total_mass(); + // Assert the total mass + assert_eq!(frontier.total_mass(), frontier.search_tree.ascending_iter().map(|k| k.mass).sum::()); + + // Add a bunch of duplicates and make sure the total mass remains the same + let mut dup_items = frontier.search_tree.ascending_iter().take(len / 2).cloned().collect_vec(); + for dup in dup_items.iter().cloned() { + (!frontier.insert(dup)).then_some(()).unwrap(); + } + assert_eq!(prev_total_mass, frontier.total_mass()); + assert_eq!(frontier.total_mass(), frontier.search_tree.ascending_iter().map(|k| k.mass).sum::()); + + // Remove a few elements from the map in order to randomize the iterator + dup_items.iter().take(10).for_each(|k| { + map.remove(&k.tx.id()); + }); + + // Add and remove random elements some of which will be duplicate insertions and some missing removals + for item in map.values().step_by(2) { + frontier.remove(item); + if let Some(item2) = dup_items.pop() { + frontier.insert(item2); + } + } + assert_eq!(frontier.total_mass(), frontier.search_tree.ascending_iter().map(|k| k.mass).sum::()); + } + + #[test] + fn test_feerate_estimator() { + let mut rng = thread_rng(); + let cap = 2000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let mut fee: u64 = rng.gen_range(1..1000000); + let mass: u64 = 1650; + // 304 (~500,000/1650) extreme outliers is an edge case where the build estimator logic should be tested at + if i <= 303 { + // Add an extremely large fee in order to create extremely high variance + fee = i * 10_000_000 * 1_000_000; + } + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + for len in [0, 1, 10, 100, 200, 300, 500, 750, cap / 2, (cap * 2) / 3, (cap * 4) / 5, (cap * 5) / 6, cap] { + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let args = FeerateEstimatorArgs { network_blocks_per_second: 1, maximum_mass_per_block: 500_000 }; + // We are testing that the build function actually returns and is not looping indefinitely + let estimator = frontier.build_feerate_estimator(args); + let estimations = estimator.calc_estimations(1.0); + + let buckets = estimations.ordered_buckets(); + // Test for the absence of NaN, infinite or zero values in buckets + for b in buckets.iter() { + assert!( + b.feerate.is_normal() && b.feerate >= 1.0, + "bucket feerate must be a finite number greater or equal to the minimum standard feerate" + ); + assert!( + b.estimated_seconds.is_normal() && b.estimated_seconds > 0.0, + "bucket estimated seconds must be a finite number greater than zero" + ); + } + dbg!(len, estimator); + dbg!(estimations); + } + } + + #[test] + fn test_constant_feerate_estimator() { + const MIN_FEERATE: f64 = 1.0; + let cap = 20_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let mass: u64 = 1650; + let fee = (mass as f64 * MIN_FEERATE) as u64; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + for len in [0, 1, 10, 100, 200, 300, 500, 750, cap / 2, (cap * 2) / 3, (cap * 4) / 5, (cap * 5) / 6, cap] { + println!(); + println!("Testing a frontier with {} txs...", len.min(cap)); + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let args = FeerateEstimatorArgs { network_blocks_per_second: 1, maximum_mass_per_block: 500_000 }; + // We are testing that the build function actually returns and is not looping indefinitely + let estimator = frontier.build_feerate_estimator(args); + let estimations = estimator.calc_estimations(MIN_FEERATE); + let buckets = estimations.ordered_buckets(); + // Test for the absence of NaN, infinite or zero values in buckets + for b in buckets.iter() { + assert!( + b.feerate.is_normal() && b.feerate >= 1.0, + "bucket feerate must be a finite number greater or equal to the minimum standard feerate" + ); + assert!( + b.estimated_seconds.is_normal() && b.estimated_seconds > 0.0, + "bucket estimated seconds must be a finite number greater than zero" + ); + } + dbg!(len, estimator); + dbg!(estimations); + } + } +} diff --git a/mining/src/mempool/model/frontier/feerate_key.rs b/mining/src/mempool/model/frontier/feerate_key.rs new file mode 100644 index 000000000..843ef0ff1 --- /dev/null +++ b/mining/src/mempool/model/frontier/feerate_key.rs @@ -0,0 +1,108 @@ +use crate::{block_template::selector::ALPHA, mempool::model::tx::MempoolTransaction}; +use kaspa_consensus_core::tx::Transaction; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct FeerateTransactionKey { + pub fee: u64, + pub mass: u64, + weight: f64, + pub tx: Arc, +} + +impl Eq for FeerateTransactionKey {} + +impl PartialEq for FeerateTransactionKey { + fn eq(&self, other: &Self) -> bool { + self.tx.id() == other.tx.id() + } +} + +impl FeerateTransactionKey { + pub fn new(fee: u64, mass: u64, tx: Arc) -> Self { + // NOTE: any change to the way this weight is calculated (such as scaling by some factor) + // requires a reversed update to total_weight in `Frontier::build_feerate_estimator`. This + // is because the math methods in FeeEstimator assume this specific weight function. + Self { fee, mass, weight: (fee as f64 / mass as f64).powi(ALPHA), tx } + } + + pub fn feerate(&self) -> f64 { + self.fee as f64 / self.mass as f64 + } + + pub fn weight(&self) -> f64 { + self.weight + } +} + +impl std::hash::Hash for FeerateTransactionKey { + fn hash(&self, state: &mut H) { + // Transaction id is a sufficient identifier for this key + self.tx.id().hash(state); + } +} + +impl PartialOrd for FeerateTransactionKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for FeerateTransactionKey { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // Our first priority is the feerate. + // The weight function is monotonic in feerate so we prefer using it + // since it is cached + match self.weight().total_cmp(&other.weight()) { + core::cmp::Ordering::Equal => {} + ord => return ord, + } + + // If feerates (and thus weights) are equal, prefer the higher fee in absolute value + match self.fee.cmp(&other.fee) { + core::cmp::Ordering::Equal => {} + ord => return ord, + } + + // + // At this point we don't compare the mass fields since if both feerate + // and fee are equal, mass must be equal as well + // + + // Finally, we compare transaction ids in order to allow multiple transactions with + // the same fee and mass to exist within the same sorted container + self.tx.id().cmp(&other.tx.id()) + } +} + +impl From<&MempoolTransaction> for FeerateTransactionKey { + fn from(tx: &MempoolTransaction) -> Self { + let mass = tx.mtx.tx.mass(); + let fee = tx.mtx.calculated_fee.expect("fee is expected to be populated"); + assert_ne!(mass, 0, "mass field is expected to be set when inserting to the mempool"); + Self::new(fee, mass, tx.mtx.tx.clone()) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use kaspa_consensus_core::{ + subnets::SUBNETWORK_ID_NATIVE, + tx::{Transaction, TransactionInput, TransactionOutpoint}, + }; + use kaspa_hashes::{HasherBase, TransactionID}; + use std::sync::Arc; + + fn generate_unique_tx(i: u64) -> Arc { + let mut hasher = TransactionID::new(); + let prev = hasher.update(i.to_le_bytes()).clone().finalize(); + let input = TransactionInput::new(TransactionOutpoint::new(prev, 0), vec![], 0, 0); + Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) + } + + /// Test helper for generating a feerate key with a unique tx (per u64 id) + pub(crate) fn build_feerate_key(fee: u64, mass: u64, id: u64) -> FeerateTransactionKey { + FeerateTransactionKey::new(fee, mass, generate_unique_tx(id)) + } +} diff --git a/mining/src/mempool/model/frontier/search_tree.rs b/mining/src/mempool/model/frontier/search_tree.rs new file mode 100644 index 000000000..fc18b2118 --- /dev/null +++ b/mining/src/mempool/model/frontier/search_tree.rs @@ -0,0 +1,335 @@ +use super::feerate_key::FeerateTransactionKey; +use std::iter::FusedIterator; +use sweep_bptree::tree::visit::{DescendVisit, DescendVisitResult}; +use sweep_bptree::tree::{Argument, SearchArgument}; +use sweep_bptree::{BPlusTree, NodeStoreVec}; + +type FeerateKey = FeerateTransactionKey; + +/// A struct for implementing "weight space" search using the SearchArgument customization. +/// The weight space is the range `[0, total_weight)` and each key has a "logical" interval allocation +/// within this space according to its tree position and weight. +/// +/// We implement the search efficiently by maintaining subtree weights which are updated with each +/// element insertion/removal. Given a search query `p ∈ [0, total_weight)` we then find the corresponding +/// element in log time by walking down from the root and adjusting the query according to subtree weights. +/// For instance if the query point is `123.56` and the top 3 subtrees have weights `120, 10.5 ,100` then we +/// recursively query the middle subtree with the point `123.56 - 120 = 3.56`. +/// +/// See SearchArgument implementation below for more details. +#[derive(Clone, Copy, Debug, Default)] +struct FeerateWeight(f64); + +impl FeerateWeight { + /// Returns the weight value + pub fn weight(&self) -> f64 { + self.0 + } +} + +impl Argument for FeerateWeight { + fn from_leaf(keys: &[FeerateKey]) -> Self { + Self(keys.iter().map(|k| k.weight()).sum()) + } + + fn from_inner(_keys: &[FeerateKey], arguments: &[Self]) -> Self { + Self(arguments.iter().map(|a| a.0).sum()) + } +} + +impl SearchArgument for FeerateWeight { + type Query = f64; + + fn locate_in_leaf(query: Self::Query, keys: &[FeerateKey]) -> Option { + let mut sum = 0.0; + for (i, k) in keys.iter().enumerate() { + let w = k.weight(); + sum += w; + if query < sum { + return Some(i); + } + } + // In order to avoid sensitivity to floating number arithmetics, + // we logically "clamp" the search, returning the last leaf if the query + // value is out of bounds + match keys.len() { + 0 => None, + n => Some(n - 1), + } + } + + fn locate_in_inner(mut query: Self::Query, _keys: &[FeerateKey], arguments: &[Self]) -> Option<(usize, Self::Query)> { + // Search algorithm: Locate the next subtree to visit by iterating through `arguments` + // and subtracting the query until the correct range is found + for (i, a) in arguments.iter().enumerate() { + if query >= a.0 { + query -= a.0; + } else { + return Some((i, query)); + } + } + // In order to avoid sensitivity to floating number arithmetics, + // we logically "clamp" the search, returning the last subtree if the query + // value is out of bounds. Eventually this will lead to the return of the + // last leaf (see locate_in_leaf as well) + match arguments.len() { + 0 => None, + n => Some((n - 1, arguments[n - 1].0)), + } + } +} + +/// Visitor struct which accumulates the prefix weight up to a provided key (inclusive) in log time. +/// +/// The basic idea is to use the subtree weights stored in the tree for walking down from the root +/// to the leaf (corresponding to the searched key), and accumulating all weights proceeding the walk-down path +struct PrefixWeightVisitor<'a> { + /// The key to search up to + key: &'a FeerateKey, + /// This field accumulates the prefix weight during the visit process + accumulated_weight: f64, +} + +impl<'a> PrefixWeightVisitor<'a> { + pub fn new(key: &'a FeerateKey) -> Self { + Self { key, accumulated_weight: Default::default() } + } + + /// Returns the index of the first `key ∈ keys` such that `key > self.key`. If no such key + /// exists, the returned index will be the length of `keys`. + fn search_in_keys(&self, keys: &[FeerateKey]) -> usize { + match keys.binary_search(self.key) { + Err(idx) => { + // self.key is not in keys, idx is the index of the following key + idx + } + Ok(idx) => { + // Exact match, return the following index + idx + 1 + } + } + } +} + +impl<'a> DescendVisit for PrefixWeightVisitor<'a> { + type Result = f64; + + fn visit_inner(&mut self, keys: &[FeerateKey], arguments: &[FeerateWeight]) -> DescendVisitResult { + let idx = self.search_in_keys(keys); + // Invariants: + // a. arguments.len() == keys.len() + 1 (n inner node keys are the separators between n+1 subtrees) + // b. idx <= keys.len() (hence idx < arguments.len()) + + // Based on the invariants, we first accumulate all the subtree weights up to idx + for argument in arguments.iter().take(idx) { + self.accumulated_weight += argument.weight(); + } + + // ..and then go down to the idx'th subtree + DescendVisitResult::GoDown(idx) + } + + fn visit_leaf(&mut self, keys: &[FeerateKey], _values: &[()]) -> Option { + // idx is the index of the key following self.key + let idx = self.search_in_keys(keys); + // Accumulate all key weights up to idx (which is inclusive if self.key ∈ tree) + for key in keys.iter().take(idx) { + self.accumulated_weight += key.weight(); + } + // ..and return the final result + Some(self.accumulated_weight) + } +} + +type InnerTree = BPlusTree>; + +/// A transaction search tree sorted by feerate order and searchable for probabilistic weighted sampling. +/// +/// All `log(n)` expressions below are in base 64 (based on constants chosen within the sweep_bptree crate). +/// +/// The tree has the following properties: +/// 1. Linear time ordered access (ascending / descending) +/// 2. Insertions/removals in log(n) time +/// 3. Search for a weight point `p ∈ [0, total_weight)` in log(n) time +/// 4. Compute the prefix weight of a key, i.e., the sum of weights up to that key (inclusive) +/// according to key order, in log(n) time +/// 5. Access the total weight in O(1) time. The total weight has numerical stability since it +/// is recomputed from subtree weights for each item insertion/removal +/// +/// Computing the prefix weight is a crucial operation if the tree is used for random sampling and +/// the tree is highly imbalanced in terms of weight variance. See [`Frontier::sample_inplace`] for +/// more details. +pub struct SearchTree { + tree: InnerTree, +} + +impl Default for SearchTree { + fn default() -> Self { + Self { tree: InnerTree::new(Default::default()) } + } +} + +impl SearchTree { + pub fn new() -> Self { + Self { tree: InnerTree::new(Default::default()) } + } + + pub fn len(&self) -> usize { + self.tree.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Inserts a key into the tree in log(n) time. Returns `false` if the key was already in the tree. + pub fn insert(&mut self, key: FeerateKey) -> bool { + self.tree.insert(key, ()).is_none() + } + + /// Remove a key from the tree in log(n) time. Returns `false` if the key was not in the tree. + pub fn remove(&mut self, key: &FeerateKey) -> bool { + self.tree.remove(key).is_some() + } + + /// Search for a weight point `query ∈ [0, total_weight)` in log(n) time + pub fn search(&self, query: f64) -> &FeerateKey { + self.tree.get_by_argument(query).expect("clamped").0 + } + + /// Access the total weight in O(1) time + pub fn total_weight(&self) -> f64 { + self.tree.root_argument().weight() + } + + /// Computes the prefix weight of a key, i.e., the sum of weights up to that key (inclusive) + /// according to key order, in log(n) time + pub fn prefix_weight(&self, key: &FeerateKey) -> f64 { + self.tree.descend_visit(PrefixWeightVisitor::new(key)).unwrap() + } + + /// Iterate the tree in descending key order (going down from the + /// highest key). Linear in the number of keys *actually* iterated. + pub fn descending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator + FusedIterator { + self.tree.iter().rev().map(|(key, ())| key) + } + + /// Iterate the tree in ascending key order (going up from the + /// lowest key). Linear in the number of keys *actually* iterated. + pub fn ascending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator + FusedIterator { + self.tree.iter().map(|(key, ())| key) + } + + /// The lowest key in the tree (by key order) + pub fn first(&self) -> Option<&FeerateKey> { + self.tree.first().map(|(k, ())| k) + } + + /// The highest key in the tree (by key order) + pub fn last(&self) -> Option<&FeerateKey> { + self.tree.last().map(|(k, ())| k) + } +} + +#[cfg(test)] +mod tests { + use super::super::feerate_key::tests::build_feerate_key; + use super::*; + use itertools::Itertools; + use std::collections::HashSet; + use std::ops::Sub; + + #[test] + fn test_feerate_weight_queries() { + let mut tree = SearchTree::new(); + let mass = 2000; + // The btree stores N=64 keys at each node/leaf, so we make sure the tree has more than + // 64^2 keys in order to trigger at least a few intermediate tree nodes + let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); + + #[allow(clippy::mutable_key_type)] + let mut s = HashSet::with_capacity(fees.len()); + for (i, fee) in fees.iter().copied().enumerate() { + let key = build_feerate_key(fee, mass, i as u64); + s.insert(key.clone()); + tree.insert(key); + } + + // Randomly remove 1/6 of the items + let remove = s.iter().take(fees.len() / 6).cloned().collect_vec(); + for r in remove { + s.remove(&r); + tree.remove(&r); + } + + // Collect to vec and sort for reference + let mut v = s.into_iter().collect_vec(); + v.sort(); + + // Test reverse iteration + for (expected, item) in v.iter().rev().zip(tree.descending_iter()) { + assert_eq!(&expected, &item); + assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well + } + + // Sweep through the tree and verify that weight search queries are handled correctly + let eps: f64 = 0.001; + let mut sum = 0.0; + for expected in v.iter() { + let weight = expected.weight(); + let eps = eps.min(weight / 3.0); + let samples = [sum + eps, sum + weight / 2.0, sum + weight - eps]; + for sample in samples { + let key = tree.search(sample); + assert_eq!(expected, key); + assert!(expected.cmp(key).is_eq()); // Assert Ord equality as well + } + sum += weight; + } + + println!("{}, {}", sum, tree.total_weight()); + + // Test clamped search bounds + assert_eq!(tree.first(), Some(tree.search(f64::NEG_INFINITY))); + assert_eq!(tree.first(), Some(tree.search(-1.0))); + assert_eq!(tree.first(), Some(tree.search(-eps))); + assert_eq!(tree.first(), Some(tree.search(0.0))); + assert_eq!(tree.last(), Some(tree.search(sum))); + assert_eq!(tree.last(), Some(tree.search(sum + eps))); + assert_eq!(tree.last(), Some(tree.search(sum + 1.0))); + assert_eq!(tree.last(), Some(tree.search(1.0 / 0.0))); + assert_eq!(tree.last(), Some(tree.search(f64::INFINITY))); + let _ = tree.search(f64::NAN); + + // Assert prefix weights + let mut prefix = Vec::with_capacity(v.len()); + prefix.push(v[0].weight()); + for i in 1..v.len() { + prefix.push(prefix[i - 1] + v[i].weight()); + } + let eps = v.iter().map(|k| k.weight()).min_by(f64::total_cmp).unwrap() * 1e-4; + for (expected_prefix, key) in prefix.into_iter().zip(v) { + let prefix = tree.prefix_weight(&key); + assert!(expected_prefix.sub(prefix).abs() < eps); + } + } + + #[test] + fn test_tree_rev_iter() { + let mut tree = SearchTree::new(); + let mass = 2000; + let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); + let mut v = Vec::with_capacity(fees.len()); + for (i, fee) in fees.iter().copied().enumerate() { + let key = build_feerate_key(fee, mass, i as u64); + v.push(key.clone()); + tree.insert(key); + } + v.sort(); + + for (expected, item) in v.into_iter().rev().zip(tree.descending_iter()) { + assert_eq!(&expected, item); + assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well + } + } +} diff --git a/mining/src/mempool/model/frontier/selectors.rs b/mining/src/mempool/model/frontier/selectors.rs new file mode 100644 index 000000000..a30ecc145 --- /dev/null +++ b/mining/src/mempool/model/frontier/selectors.rs @@ -0,0 +1,162 @@ +use crate::Policy; +use kaspa_consensus_core::{ + block::TemplateTransactionSelector, + tx::{Transaction, TransactionId}, +}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; + +pub struct SequenceSelectorTransaction { + pub tx: Arc, + pub mass: u64, +} + +impl SequenceSelectorTransaction { + pub fn new(tx: Arc, mass: u64) -> Self { + Self { tx, mass } + } +} + +type SequencePriorityIndex = u32; + +/// The input sequence for the [`SequenceSelector`] transaction selector +#[derive(Default)] +pub struct SequenceSelectorInput { + /// We use the btree map ordered by insertion order in order to follow + /// the initial sequence order while allowing for efficient removal of previous selections + inner: BTreeMap, +} + +impl FromIterator for SequenceSelectorInput { + fn from_iter>(iter: T) -> Self { + Self { inner: BTreeMap::from_iter(iter.into_iter().enumerate().map(|(i, v)| (i as SequencePriorityIndex, v))) } + } +} + +impl SequenceSelectorInput { + pub fn push(&mut self, tx: Arc, mass: u64) { + let idx = self.inner.len() as SequencePriorityIndex; + self.inner.insert(idx, SequenceSelectorTransaction::new(tx, mass)); + } + + pub fn iter(&self) -> impl Iterator { + self.inner.values() + } +} + +/// Helper struct for storing data related to previous selections +struct SequenceSelectorSelection { + tx_id: TransactionId, + mass: u64, + priority_index: SequencePriorityIndex, +} + +/// A selector which selects transactions in the order they are provided. The selector assumes +/// that the transactions were already selected via weighted sampling and simply tries them one +/// after the other until the block mass limit is reached. +pub struct SequenceSelector { + input_sequence: SequenceSelectorInput, + selected_vec: Vec, + /// Maps from selected tx ids to tx mass so that the total used mass can be subtracted on tx reject + selected_map: Option>, + total_selected_mass: u64, + overall_candidates: usize, + overall_rejections: usize, + policy: Policy, +} + +impl SequenceSelector { + pub fn new(input_sequence: SequenceSelectorInput, policy: Policy) -> Self { + Self { + overall_candidates: input_sequence.inner.len(), + selected_vec: Vec::with_capacity(input_sequence.inner.len()), + input_sequence, + selected_map: Default::default(), + total_selected_mass: Default::default(), + overall_rejections: Default::default(), + policy, + } + } + + #[inline] + fn reset_selection(&mut self) { + self.selected_vec.clear(); + self.selected_map = None; + } +} + +impl TemplateTransactionSelector for SequenceSelector { + fn select_transactions(&mut self) -> Vec { + // Remove selections from the previous round if any + for selection in self.selected_vec.drain(..) { + self.input_sequence.inner.remove(&selection.priority_index); + } + // Reset selection data structures + self.reset_selection(); + let mut transactions = Vec::with_capacity(self.input_sequence.inner.len()); + + // Iterate the input sequence in order + for (&priority_index, tx) in self.input_sequence.inner.iter() { + if self.total_selected_mass.saturating_add(tx.mass) > self.policy.max_block_mass { + // We assume the sequence is relatively small, hence we keep on searching + // for transactions with lower mass which might fit into the remaining gap + continue; + } + self.total_selected_mass += tx.mass; + self.selected_vec.push(SequenceSelectorSelection { tx_id: tx.tx.id(), mass: tx.mass, priority_index }); + transactions.push(tx.tx.as_ref().clone()) + } + transactions + } + + fn reject_selection(&mut self, tx_id: TransactionId) { + // Lazy-create the map only when there are actual rejections + let selected_map = self.selected_map.get_or_insert_with(|| self.selected_vec.iter().map(|tx| (tx.tx_id, tx.mass)).collect()); + let mass = selected_map.remove(&tx_id).expect("only previously selected txs can be rejected (and only once)"); + // Selections must be counted in total selected mass, so this subtraction cannot underflow + self.total_selected_mass -= mass; + self.overall_rejections += 1; + } + + fn is_successful(&self) -> bool { + const SUFFICIENT_MASS_THRESHOLD: f64 = 0.8; + const LOW_REJECTION_FRACTION: f64 = 0.2; + + // We consider the operation successful if either mass occupation is above 80% or rejection rate is below 20% + self.overall_rejections == 0 + || (self.total_selected_mass as f64) > self.policy.max_block_mass as f64 * SUFFICIENT_MASS_THRESHOLD + || (self.overall_rejections as f64) < self.overall_candidates as f64 * LOW_REJECTION_FRACTION + } +} + +/// A selector that selects all the transactions it holds and is always considered successful. +/// If all mempool transactions have combined mass which is <= block mass limit, this selector +/// should be called and provided with all the transactions. +pub struct TakeAllSelector { + txs: Vec>, +} + +impl TakeAllSelector { + pub fn new(txs: Vec>) -> Self { + Self { txs } + } +} + +impl TemplateTransactionSelector for TakeAllSelector { + fn select_transactions(&mut self) -> Vec { + // Drain on the first call so that subsequent calls return nothing + self.txs.drain(..).map(|tx| tx.as_ref().clone()).collect() + } + + fn reject_selection(&mut self, _tx_id: TransactionId) { + // No need to track rejections (for reduced mass), since there's nothing else to select + } + + fn is_successful(&self) -> bool { + // Considered successful because we provided all mempool transactions to this + // selector, so there's no point in retries + true + } +} diff --git a/mining/src/mempool/model/mod.rs b/mining/src/mempool/model/mod.rs index 88997e46f..bfe622293 100644 --- a/mining/src/mempool/model/mod.rs +++ b/mining/src/mempool/model/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod accepted_transactions; +pub(crate) mod frontier; pub(crate) mod map; pub(crate) mod orphan_pool; pub(crate) mod pool; diff --git a/mining/src/mempool/model/transactions_pool.rs b/mining/src/mempool/model/transactions_pool.rs index 988da5ddd..bc3469409 100644 --- a/mining/src/mempool/model/transactions_pool.rs +++ b/mining/src/mempool/model/transactions_pool.rs @@ -1,4 +1,5 @@ use crate::{ + feerate::{FeerateEstimator, FeerateEstimatorArgs}, mempool::{ config::Config, errors::{RuleError, RuleResult}, @@ -10,18 +11,21 @@ use crate::{ }, tx::Priority, }, - model::{candidate_tx::CandidateTransaction, topological_index::TopologicalIndex}, + model::topological_index::TopologicalIndex, + Policy, }; use kaspa_consensus_core::{ - tx::TransactionId, - tx::{MutableTransaction, TransactionOutpoint}, + block::TemplateTransactionSelector, + tx::{MutableTransaction, TransactionId, TransactionOutpoint}, }; use kaspa_core::{time::unix_now, trace, warn}; use std::{ - collections::{hash_map::Keys, hash_set::Iter, HashSet}, + collections::{hash_map::Keys, hash_set::Iter}, sync::Arc, }; +use super::frontier::Frontier; + /// Pool of transactions to be included in a block template /// /// ### Rust rewrite notes @@ -54,7 +58,7 @@ pub(crate) struct TransactionsPool { /// Transactions dependencies formed by outputs present in pool - successor relations. chained_transactions: TransactionsEdges, /// Transactions with no parents in the mempool -- ready to be inserted into a block template - ready_transactions: HashSet, + ready_transactions: Frontier, last_expire_scan_daa_score: u64, /// last expire scan time in milliseconds @@ -105,7 +109,7 @@ impl TransactionsPool { let parents = self.get_parent_transaction_ids_in_pool(&transaction.mtx); self.parent_transactions.insert(id, parents.clone()); if parents.is_empty() { - self.ready_transactions.insert(id); + self.ready_transactions.insert((&transaction).into()); } for parent_id in parents { let entry = self.chained_transactions.entry(parent_id).or_default(); @@ -133,18 +137,20 @@ impl TransactionsPool { if let Some(parents) = self.parent_transactions.get_mut(chain) { parents.remove(transaction_id); if parents.is_empty() { - self.ready_transactions.insert(*chain); + let tx = self.all_transactions.get(chain).unwrap(); + self.ready_transactions.insert(tx.into()); } } } } self.parent_transactions.remove(transaction_id); self.chained_transactions.remove(transaction_id); - self.ready_transactions.remove(transaction_id); // Remove the transaction itself let removed_tx = self.all_transactions.remove(transaction_id).ok_or(RuleError::RejectMissingTransaction(*transaction_id))?; + self.ready_transactions.remove(&(&removed_tx).into()); + // TODO: consider using `self.parent_transactions.get(transaction_id)` // The tradeoff to consider is whether it might be possible that a parent tx exists in the pool // however its relation as parent is not registered. This can supposedly happen in rare cases where @@ -161,15 +167,18 @@ impl TransactionsPool { self.ready_transactions.len() } - /// all_ready_transactions returns all fully populated mempool transactions having no parents in the mempool. - /// These transactions are ready for being inserted in a block template. - pub(crate) fn all_ready_transactions(&self) -> Vec { - // The returned transactions are leaving the mempool so they are cloned - self.ready_transactions - .iter() - .take(self.config.maximum_ready_transaction_count as usize) - .map(|id| CandidateTransaction::from_mutable(&self.all_transactions.get(id).unwrap().mtx)) - .collect() + pub(crate) fn ready_transaction_total_mass(&self) -> u64 { + self.ready_transactions.total_mass() + } + + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier + pub(crate) fn build_selector(&self) -> Box { + self.ready_transactions.build_selector(&Policy::new(self.config.maximum_mass_per_block)) + } + + /// Builds a feerate estimator based on internal state of the ready transactions frontier + pub(crate) fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { + self.ready_transactions.build_feerate_estimator(args) } /// Is the mempool transaction identified by `transaction_id` unchained, thus having no successor? @@ -229,8 +238,8 @@ impl TransactionsPool { // An error is returned if the mempool is filled with high priority and other unremovable transactions. let tx_count = self.len() + free_slots - transactions_to_remove.len(); - if tx_count as u64 > self.config.maximum_transaction_count { - let err = RuleError::RejectMempoolIsFull(tx_count - free_slots, self.config.maximum_transaction_count); + if tx_count as u64 > self.config.maximum_transaction_count as u64 { + let err = RuleError::RejectMempoolIsFull(tx_count - free_slots, self.config.maximum_transaction_count as u64); warn!("{}", err.to_string()); return Err(err); } diff --git a/mining/src/mempool/model/tx.rs b/mining/src/mempool/model/tx.rs index 48f24b9f6..9b65faeb2 100644 --- a/mining/src/mempool/model/tx.rs +++ b/mining/src/mempool/model/tx.rs @@ -2,7 +2,6 @@ use crate::mempool::tx::{Priority, RbfPolicy}; use kaspa_consensus_core::tx::{MutableTransaction, Transaction, TransactionId, TransactionOutpoint}; use kaspa_mining_errors::mempool::RuleError; use std::{ - cmp::Ordering, fmt::{Display, Formatter}, sync::Arc, }; @@ -35,26 +34,6 @@ impl MempoolTransaction { } } -impl Ord for MempoolTransaction { - fn cmp(&self, other: &Self) -> Ordering { - self.fee_rate().total_cmp(&other.fee_rate()).then(self.id().cmp(&other.id())) - } -} - -impl Eq for MempoolTransaction {} - -impl PartialOrd for MempoolTransaction { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl PartialEq for MempoolTransaction { - fn eq(&self, other: &Self) -> bool { - self.fee_rate() == other.fee_rate() - } -} - impl RbfPolicy { #[cfg(test)] /// Returns an alternate policy accepting a transaction insertion in case the policy requires a replacement diff --git a/mining/src/model/candidate_tx.rs b/mining/src/model/candidate_tx.rs index f1fdf7c71..b8cc34cc4 100644 --- a/mining/src/model/candidate_tx.rs +++ b/mining/src/model/candidate_tx.rs @@ -1,10 +1,11 @@ -use kaspa_consensus_core::tx::{MutableTransaction, Transaction}; +use crate::FeerateTransactionKey; +use kaspa_consensus_core::tx::Transaction; use std::sync::Arc; /// Transaction with additional metadata needed in order to be a candidate /// in the transaction selection algorithm #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct CandidateTransaction { +pub struct CandidateTransaction { /// The actual transaction pub tx: Arc, /// Populated fee @@ -14,9 +15,7 @@ pub(crate) struct CandidateTransaction { } impl CandidateTransaction { - pub(crate) fn from_mutable(tx: &MutableTransaction) -> Self { - let mass = tx.tx.mass(); - assert_ne!(mass, 0, "mass field is expected to be set when inserting to the mempool"); - Self { tx: tx.tx.clone(), calculated_fee: tx.calculated_fee.expect("fee is expected to be populated"), calculated_mass: mass } + pub fn from_key(key: FeerateTransactionKey) -> Self { + Self { tx: key.tx, calculated_fee: key.fee, calculated_mass: key.mass } } } diff --git a/mining/src/model/mod.rs b/mining/src/model/mod.rs index 66c91cae5..dcec6f17f 100644 --- a/mining/src/model/mod.rs +++ b/mining/src/model/mod.rs @@ -1,7 +1,7 @@ use kaspa_consensus_core::tx::TransactionId; use std::collections::HashSet; -pub(crate) mod candidate_tx; +pub mod candidate_tx; pub mod owner_txs; pub mod topological_index; pub mod topological_sort; diff --git a/mining/src/monitor.rs b/mining/src/monitor.rs index 517bd8276..876ce9b7a 100644 --- a/mining/src/monitor.rs +++ b/mining/src/monitor.rs @@ -1,4 +1,5 @@ use super::MiningCounters; +use crate::manager::MiningManagerProxy; use kaspa_core::{ debug, info, task::{ @@ -13,6 +14,8 @@ use std::{sync::Arc, time::Duration}; const MONITOR: &str = "mempool-monitor"; pub struct MiningMonitor { + mining_manager: MiningManagerProxy, + // Counters counters: Arc, @@ -24,11 +27,12 @@ pub struct MiningMonitor { impl MiningMonitor { pub fn new( + mining_manager: MiningManagerProxy, counters: Arc, tx_script_cache_counters: Arc, tick_service: Arc, ) -> MiningMonitor { - MiningMonitor { counters, tx_script_cache_counters, tick_service } + MiningMonitor { mining_manager, counters, tx_script_cache_counters, tick_service } } pub async fn worker(self: &Arc) { @@ -62,6 +66,8 @@ impl MiningMonitor { delta.low_priority_tx_counts, delta.tx_accepted_counts, ); + let feerate_estimations = self.mining_manager.clone().get_realtime_feerate_estimations().await; + debug!("Realtime feerate estimations: {}", feerate_estimations); } if tx_script_cache_snapshot != last_tx_script_cache_snapshot { debug!( diff --git a/rpc/core/src/api/ops.rs b/rpc/core/src/api/ops.rs index d312f499b..6d5357406 100644 --- a/rpc/core/src/api/ops.rs +++ b/rpc/core/src/api/ops.rs @@ -117,6 +117,10 @@ pub enum RpcApiOps { /// Extracts a transaction out of the request message and attempts to replace a matching transaction in the mempool with it, applying a mandatory Replace by Fee policy SubmitTransactionReplacement, + + // Fee estimation related commands + GetFeeEstimate, + GetFeeEstimateExperimental, } impl RpcApiOps { diff --git a/rpc/core/src/api/rpc.rs b/rpc/core/src/api/rpc.rs index f109ef8f6..8fd26b058 100644 --- a/rpc/core/src/api/rpc.rs +++ b/rpc/core/src/api/rpc.rs @@ -315,6 +315,22 @@ pub trait RpcApi: Sync + Send + AnySync { request: GetDaaScoreTimestampEstimateRequest, ) -> RpcResult; + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // Fee estimation API + + async fn get_fee_estimate(&self) -> RpcResult { + Ok(self.get_fee_estimate_call(GetFeeEstimateRequest {}).await?.estimate) + } + async fn get_fee_estimate_call(&self, request: GetFeeEstimateRequest) -> RpcResult; + + async fn get_fee_estimate_experimental(&self, verbose: bool) -> RpcResult { + self.get_fee_estimate_experimental_call(GetFeeEstimateExperimentalRequest { verbose }).await + } + async fn get_fee_estimate_experimental_call( + &self, + request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult; + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/core/src/model/feerate_estimate.rs b/rpc/core/src/model/feerate_estimate.rs new file mode 100644 index 000000000..f9de9de90 --- /dev/null +++ b/rpc/core/src/model/feerate_estimate.rs @@ -0,0 +1,55 @@ +use borsh::{BorshDeserialize, BorshSerialize}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcFeerateBucket { + /// The fee/mass ratio estimated to be required for inclusion time <= estimated_seconds + pub feerate: f64, + + /// The estimated inclusion time for a transaction with fee/mass = feerate + pub estimated_seconds: f64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcFeeEstimate { + /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + /// + /// Note: for all buckets, feerate values represent fee/mass of a transaction in `sompi/gram` units. + /// Given a feerate value recommendation, calculate the required fee by + /// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` + pub priority_bucket: RpcFeerateBucket, + + /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and + /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + /// between them, one can compose a complete feerate function on the client side. The API makes an effort + /// to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + pub normal_buckets: Vec, + + /// A vector of *low* priority feerate values. The first value of this vector is guaranteed to + /// exist and provide an estimation for sub-*hour* DAG inclusion. + pub low_buckets: Vec, +} + +impl RpcFeeEstimate { + pub fn ordered_buckets(&self) -> Vec { + std::iter::once(self.priority_bucket) + .chain(self.normal_buckets.iter().copied()) + .chain(self.low_buckets.iter().copied()) + .collect() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcFeeEstimateVerboseExperimentalData { + pub mempool_ready_transactions_count: u64, + pub mempool_ready_transactions_total_mass: u64, + pub network_mass_per_second: u64, + + pub next_block_template_feerate_min: f64, + pub next_block_template_feerate_median: f64, + pub next_block_template_feerate_max: f64, +} diff --git a/rpc/core/src/model/message.rs b/rpc/core/src/model/message.rs index d19984b58..5c003546c 100644 --- a/rpc/core/src/model/message.rs +++ b/rpc/core/src/model/message.rs @@ -850,6 +850,35 @@ impl GetDaaScoreTimestampEstimateResponse { } } +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// Fee rate estimations + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateRequest {} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateResponse { + pub estimate: RpcFeeEstimate, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateExperimentalRequest { + pub verbose: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateExperimentalResponse { + /// The usual feerate estimate response + pub estimate: RpcFeeEstimate, + + /// Experimental verbose data + pub verbose: Option, +} + // ---------------------------------------------------------------------------- // Subscriptions & notifications // ---------------------------------------------------------------------------- diff --git a/rpc/core/src/model/mod.rs b/rpc/core/src/model/mod.rs index fd07a109e..8950bd1cb 100644 --- a/rpc/core/src/model/mod.rs +++ b/rpc/core/src/model/mod.rs @@ -1,6 +1,7 @@ pub mod address; pub mod block; pub mod blue_work; +pub mod feerate_estimate; pub mod hash; pub mod header; pub mod hex_cnv; @@ -15,6 +16,7 @@ pub mod tx; pub use address::*; pub use block::*; pub use blue_work::*; +pub use feerate_estimate::*; pub use hash::*; pub use header::*; pub use hex_cnv::*; diff --git a/rpc/grpc/client/src/lib.rs b/rpc/grpc/client/src/lib.rs index 98e72ce3e..c19a28eb5 100644 --- a/rpc/grpc/client/src/lib.rs +++ b/rpc/grpc/client/src/lib.rs @@ -272,6 +272,8 @@ impl RpcApi for GrpcClient { route!(get_mempool_entries_by_addresses_call, GetMempoolEntriesByAddresses); route!(get_coin_supply_call, GetCoinSupply); route!(get_daa_score_timestamp_estimate_call, GetDaaScoreTimestampEstimate); + route!(get_fee_estimate_call, GetFeeEstimate); + route!(get_fee_estimate_experimental_call, GetFeeEstimateExperimental); // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/grpc/core/proto/messages.proto b/rpc/grpc/core/proto/messages.proto index 0358f1d19..01075c183 100644 --- a/rpc/grpc/core/proto/messages.proto +++ b/rpc/grpc/core/proto/messages.proto @@ -60,6 +60,8 @@ message KaspadRequest { GetSyncStatusRequestMessage getSyncStatusRequest = 1094; GetDaaScoreTimestampEstimateRequestMessage getDaaScoreTimestampEstimateRequest = 1096; SubmitTransactionReplacementRequestMessage submitTransactionReplacementRequest = 2000; + GetFeeEstimateRequestMessage getFeeEstimateRequest = 1106; + GetFeeEstimateExperimentalRequestMessage getFeeEstimateExperimentalRequest = 1108; } } @@ -120,6 +122,8 @@ message KaspadResponse { GetSyncStatusResponseMessage getSyncStatusResponse = 1095; GetDaaScoreTimestampEstimateResponseMessage getDaaScoreTimestampEstimateResponse = 1097; SubmitTransactionReplacementResponseMessage submitTransactionReplacementResponse = 2001; + GetFeeEstimateResponseMessage getFeeEstimateResponse = 1107; + GetFeeEstimateExperimentalResponseMessage getFeeEstimateExperimentalResponse = 1109; } } diff --git a/rpc/grpc/core/proto/rpc.proto b/rpc/grpc/core/proto/rpc.proto index 52ecfa669..a0bed8977 100644 --- a/rpc/grpc/core/proto/rpc.proto +++ b/rpc/grpc/core/proto/rpc.proto @@ -866,3 +866,59 @@ message GetDaaScoreTimestampEstimateResponseMessage{ repeated uint64 timestamps = 1; RPCError error = 1000; } + +message RpcFeerateBucket { + // Fee/mass of a transaction in `sompi/gram` units + double feerate = 1; + double estimated_seconds = 2; +} + +// Data required for making fee estimates. +// +// Feerate values represent fee/mass of a transaction in `sompi/gram` units. +// Given a feerate value recommendation, calculate the required fee by +// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` +message RpcFeeEstimate { + // Top-priority feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + RpcFeerateBucket priority_bucket = 1; + + // A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and + // provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + // times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + // between them, one can compose a complete feerate function on the client side. The API makes an effort + // to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + repeated RpcFeerateBucket normal_buckets = 2; + + // A vector of *low* priority feerate values. The first value of this vector is guaranteed to + // exist and provide an estimation for sub-*hour* DAG inclusion. + repeated RpcFeerateBucket low_buckets = 3; +} + +message RpcFeeEstimateVerboseExperimentalData { + uint64 mempool_ready_transactions_count = 1; + uint64 mempool_ready_transactions_total_mass = 2; + uint64 network_mass_per_second = 3; + + double next_block_template_feerate_min = 11; + double next_block_template_feerate_median = 12; + double next_block_template_feerate_max = 13; +} + +message GetFeeEstimateRequestMessage { +} + +message GetFeeEstimateResponseMessage { + RpcFeeEstimate estimate = 1; + RPCError error = 1000; +} + +message GetFeeEstimateExperimentalRequestMessage { + bool verbose = 1; +} + +message GetFeeEstimateExperimentalResponseMessage { + RpcFeeEstimate estimate = 1; + RpcFeeEstimateVerboseExperimentalData verbose = 2; + + RPCError error = 1000; +} diff --git a/rpc/grpc/core/src/convert/feerate_estimate.rs b/rpc/grpc/core/src/convert/feerate_estimate.rs new file mode 100644 index 000000000..d1bff8f45 --- /dev/null +++ b/rpc/grpc/core/src/convert/feerate_estimate.rs @@ -0,0 +1,66 @@ +use crate::protowire; +use crate::{from, try_from}; +use kaspa_rpc_core::RpcError; + +// ---------------------------------------------------------------------------- +// rpc_core to protowire +// ---------------------------------------------------------------------------- + +from!(item: &kaspa_rpc_core::RpcFeerateBucket, protowire::RpcFeerateBucket, { + Self { + feerate: item.feerate, + estimated_seconds: item.estimated_seconds, + } +}); + +from!(item: &kaspa_rpc_core::RpcFeeEstimate, protowire::RpcFeeEstimate, { + Self { + priority_bucket: Some((&item.priority_bucket).into()), + normal_buckets: item.normal_buckets.iter().map(|b| b.into()).collect(), + low_buckets: item.low_buckets.iter().map(|b| b.into()).collect(), + } +}); + +from!(item: &kaspa_rpc_core::RpcFeeEstimateVerboseExperimentalData, protowire::RpcFeeEstimateVerboseExperimentalData, { + Self { + network_mass_per_second: item.network_mass_per_second, + mempool_ready_transactions_count: item.mempool_ready_transactions_count, + mempool_ready_transactions_total_mass: item.mempool_ready_transactions_total_mass, + next_block_template_feerate_min: item.next_block_template_feerate_min, + next_block_template_feerate_median: item.next_block_template_feerate_median, + next_block_template_feerate_max: item.next_block_template_feerate_max, + } +}); + +// ---------------------------------------------------------------------------- +// protowire to rpc_core +// ---------------------------------------------------------------------------- + +try_from!(item: &protowire::RpcFeerateBucket, kaspa_rpc_core::RpcFeerateBucket, { + Self { + feerate: item.feerate, + estimated_seconds: item.estimated_seconds, + } +}); + +try_from!(item: &protowire::RpcFeeEstimate, kaspa_rpc_core::RpcFeeEstimate, { + Self { + priority_bucket: item.priority_bucket + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("RpcFeeEstimate".to_string(), "priority_bucket".to_string()))? + .try_into()?, + normal_buckets: item.normal_buckets.iter().map(|b| b.try_into()).collect::, _>>()?, + low_buckets: item.low_buckets.iter().map(|b| b.try_into()).collect::, _>>()?, + } +}); + +try_from!(item: &protowire::RpcFeeEstimateVerboseExperimentalData, kaspa_rpc_core::RpcFeeEstimateVerboseExperimentalData, { + Self { + network_mass_per_second: item.network_mass_per_second, + mempool_ready_transactions_count: item.mempool_ready_transactions_count, + mempool_ready_transactions_total_mass: item.mempool_ready_transactions_total_mass, + next_block_template_feerate_min: item.next_block_template_feerate_min, + next_block_template_feerate_median: item.next_block_template_feerate_median, + next_block_template_feerate_max: item.next_block_template_feerate_max, + } +}); diff --git a/rpc/grpc/core/src/convert/kaspad.rs b/rpc/grpc/core/src/convert/kaspad.rs index 0fc81e0f1..d43442fe3 100644 --- a/rpc/grpc/core/src/convert/kaspad.rs +++ b/rpc/grpc/core/src/convert/kaspad.rs @@ -58,6 +58,8 @@ pub mod kaspad_request_convert { impl_into_kaspad_request!(GetServerInfo); impl_into_kaspad_request!(GetSyncStatus); impl_into_kaspad_request!(GetDaaScoreTimestampEstimate); + impl_into_kaspad_request!(GetFeeEstimate); + impl_into_kaspad_request!(GetFeeEstimateExperimental); impl_into_kaspad_request!(NotifyBlockAdded); impl_into_kaspad_request!(NotifyNewBlockTemplate); @@ -190,6 +192,8 @@ pub mod kaspad_response_convert { impl_into_kaspad_response!(GetServerInfo); impl_into_kaspad_response!(GetSyncStatus); impl_into_kaspad_response!(GetDaaScoreTimestampEstimate); + impl_into_kaspad_response!(GetFeeEstimate); + impl_into_kaspad_response!(GetFeeEstimateExperimental); impl_into_kaspad_notify_response!(NotifyBlockAdded); impl_into_kaspad_notify_response!(NotifyNewBlockTemplate); diff --git a/rpc/grpc/core/src/convert/message.rs b/rpc/grpc/core/src/convert/message.rs index d005c8c79..75606dc9a 100644 --- a/rpc/grpc/core/src/convert/message.rs +++ b/rpc/grpc/core/src/convert/message.rs @@ -401,6 +401,25 @@ from!(item: RpcResult<&kaspa_rpc_core::GetDaaScoreTimestampEstimateResponse>, pr Self { timestamps: item.timestamps.clone(), error: None } }); +// Fee estimate API + +from!(&kaspa_rpc_core::GetFeeEstimateRequest, protowire::GetFeeEstimateRequestMessage); +from!(item: RpcResult<&kaspa_rpc_core::GetFeeEstimateResponse>, protowire::GetFeeEstimateResponseMessage, { + Self { estimate: Some((&item.estimate).into()), error: None } +}); +from!(item: &kaspa_rpc_core::GetFeeEstimateExperimentalRequest, protowire::GetFeeEstimateExperimentalRequestMessage, { + Self { + verbose: item.verbose + } +}); +from!(item: RpcResult<&kaspa_rpc_core::GetFeeEstimateExperimentalResponse>, protowire::GetFeeEstimateExperimentalResponseMessage, { + Self { + estimate: Some((&item.estimate).into()), + verbose: item.verbose.as_ref().map(|x| x.into()), + error: None + } +}); + from!(&kaspa_rpc_core::PingRequest, protowire::PingRequestMessage); from!(RpcResult<&kaspa_rpc_core::PingResponse>, protowire::PingResponseMessage); @@ -818,6 +837,30 @@ try_from!(item: &protowire::GetDaaScoreTimestampEstimateResponseMessage, RpcResu Self { timestamps: item.timestamps.clone() } }); +try_from!(&protowire::GetFeeEstimateRequestMessage, kaspa_rpc_core::GetFeeEstimateRequest); +try_from!(item: &protowire::GetFeeEstimateResponseMessage, RpcResult, { + Self { + estimate: item.estimate + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("GetFeeEstimateResponseMessage".to_string(), "estimate".to_string()))? + .try_into()? + } +}); +try_from!(item: &protowire::GetFeeEstimateExperimentalRequestMessage, kaspa_rpc_core::GetFeeEstimateExperimentalRequest, { + Self { + verbose: item.verbose + } +}); +try_from!(item: &protowire::GetFeeEstimateExperimentalResponseMessage, RpcResult, { + Self { + estimate: item.estimate + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("GetFeeEstimateExperimentalResponseMessage".to_string(), "estimate".to_string()))? + .try_into()?, + verbose: item.verbose.as_ref().map(|x| x.try_into()).transpose()? + } +}); + try_from!(&protowire::PingRequestMessage, kaspa_rpc_core::PingRequest); try_from!(&protowire::PingResponseMessage, RpcResult); diff --git a/rpc/grpc/core/src/convert/mod.rs b/rpc/grpc/core/src/convert/mod.rs index 2f3252d22..d4948f57d 100644 --- a/rpc/grpc/core/src/convert/mod.rs +++ b/rpc/grpc/core/src/convert/mod.rs @@ -1,6 +1,7 @@ pub mod address; pub mod block; pub mod error; +pub mod feerate_estimate; pub mod header; pub mod kaspad; pub mod mempool; diff --git a/rpc/grpc/core/src/ops.rs b/rpc/grpc/core/src/ops.rs index 20ebf6ab4..605d27efd 100644 --- a/rpc/grpc/core/src/ops.rs +++ b/rpc/grpc/core/src/ops.rs @@ -82,6 +82,8 @@ pub enum KaspadPayloadOps { GetServerInfo, GetSyncStatus, GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, // Subscription commands for starting/stopping notifications NotifyBlockAdded, diff --git a/rpc/grpc/server/src/request_handler/factory.rs b/rpc/grpc/server/src/request_handler/factory.rs index c598b759c..a70fb629f 100644 --- a/rpc/grpc/server/src/request_handler/factory.rs +++ b/rpc/grpc/server/src/request_handler/factory.rs @@ -76,6 +76,8 @@ impl Factory { GetServerInfo, GetSyncStatus, GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, NotifyBlockAdded, NotifyNewBlockTemplate, NotifyFinalityConflict, diff --git a/rpc/grpc/server/src/tests/rpc_core_mock.rs b/rpc/grpc/server/src/tests/rpc_core_mock.rs index f0e7bda1d..2f4afa9c9 100644 --- a/rpc/grpc/server/src/tests/rpc_core_mock.rs +++ b/rpc/grpc/server/src/tests/rpc_core_mock.rs @@ -235,6 +235,17 @@ impl RpcApi for RpcCoreMock { Err(RpcError::NotImplemented) } + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + Err(RpcError::NotImplemented) + } + + async fn get_fee_estimate_experimental_call( + &self, + _request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/service/src/converter/feerate_estimate.rs b/rpc/service/src/converter/feerate_estimate.rs new file mode 100644 index 000000000..8df695c0c --- /dev/null +++ b/rpc/service/src/converter/feerate_estimate.rs @@ -0,0 +1,49 @@ +use kaspa_mining::feerate::{FeeEstimateVerbose, FeerateBucket, FeerateEstimations}; +use kaspa_rpc_core::{ + message::GetFeeEstimateExperimentalResponse as RpcFeeEstimateVerboseResponse, RpcFeeEstimate, + RpcFeeEstimateVerboseExperimentalData as RpcFeeEstimateVerbose, RpcFeerateBucket, +}; + +pub trait FeerateBucketConverter { + fn into_rpc(self) -> RpcFeerateBucket; +} + +impl FeerateBucketConverter for FeerateBucket { + fn into_rpc(self) -> RpcFeerateBucket { + RpcFeerateBucket { feerate: self.feerate, estimated_seconds: self.estimated_seconds } + } +} + +pub trait FeeEstimateConverter { + fn into_rpc(self) -> RpcFeeEstimate; +} + +impl FeeEstimateConverter for FeerateEstimations { + fn into_rpc(self) -> RpcFeeEstimate { + RpcFeeEstimate { + priority_bucket: self.priority_bucket.into_rpc(), + normal_buckets: self.normal_buckets.into_iter().map(FeerateBucketConverter::into_rpc).collect(), + low_buckets: self.low_buckets.into_iter().map(FeerateBucketConverter::into_rpc).collect(), + } + } +} + +pub trait FeeEstimateVerboseConverter { + fn into_rpc(self) -> RpcFeeEstimateVerboseResponse; +} + +impl FeeEstimateVerboseConverter for FeeEstimateVerbose { + fn into_rpc(self) -> RpcFeeEstimateVerboseResponse { + RpcFeeEstimateVerboseResponse { + estimate: self.estimations.into_rpc(), + verbose: Some(RpcFeeEstimateVerbose { + network_mass_per_second: self.network_mass_per_second, + mempool_ready_transactions_count: self.mempool_ready_transactions_count, + mempool_ready_transactions_total_mass: self.mempool_ready_transactions_total_mass, + next_block_template_feerate_min: self.next_block_template_feerate_min, + next_block_template_feerate_median: self.next_block_template_feerate_median, + next_block_template_feerate_max: self.next_block_template_feerate_max, + }), + } + } +} diff --git a/rpc/service/src/converter/mod.rs b/rpc/service/src/converter/mod.rs index 2e1460385..fd167d349 100644 --- a/rpc/service/src/converter/mod.rs +++ b/rpc/service/src/converter/mod.rs @@ -1,3 +1,4 @@ pub mod consensus; +pub mod feerate_estimate; pub mod index; pub mod protocol; diff --git a/rpc/service/src/service.rs b/rpc/service/src/service.rs index c69fda4f1..e15fa3afa 100644 --- a/rpc/service/src/service.rs +++ b/rpc/service/src/service.rs @@ -1,6 +1,7 @@ //! Core server implementation for ClientAPI use super::collector::{CollectorFromConsensus, CollectorFromIndex}; +use crate::converter::feerate_estimate::{FeeEstimateConverter, FeeEstimateVerboseConverter}; use crate::converter::{consensus::ConsensusConverter, index::IndexConverter, protocol::ProtocolConverter}; use crate::service::NetworkType::{Mainnet, Testnet}; use async_trait::async_trait; @@ -61,9 +62,11 @@ use kaspa_rpc_core::{ Notification, RpcError, RpcResult, }; use kaspa_txscript::{extract_script_pub_key_address, pay_to_address_script}; +use kaspa_utils::expiring_cache::ExpiringCache; use kaspa_utils::{channel::Channel, triggers::SingleTrigger}; use kaspa_utils_tower::counters::TowerConnectionCounters; use kaspa_utxoindex::api::UtxoIndexProxy; +use std::time::Duration; use std::{ collections::HashMap, iter::once, @@ -109,6 +112,8 @@ pub struct RpcCoreService { perf_monitor: Arc>>, p2p_tower_counters: Arc, grpc_tower_counters: Arc, + fee_estimate_cache: ExpiringCache, + fee_estimate_verbose_cache: ExpiringCache, } const RPC_CORE: &str = "rpc-core"; @@ -208,6 +213,8 @@ impl RpcCoreService { perf_monitor, p2p_tower_counters, grpc_tower_counters, + fee_estimate_cache: ExpiringCache::new(Duration::from_millis(500), Duration::from_millis(1000)), + fee_estimate_verbose_cache: ExpiringCache::new(Duration::from_millis(500), Duration::from_millis(1000)), } } @@ -663,6 +670,30 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(GetDaaScoreTimestampEstimateResponse::new(timestamps)) } + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + let mining_manager = self.mining_manager.clone(); + let estimate = + self.fee_estimate_cache.get(async move { mining_manager.get_realtime_feerate_estimations().await.into_rpc() }).await; + Ok(GetFeeEstimateResponse { estimate }) + } + + async fn get_fee_estimate_experimental_call( + &self, + request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + if request.verbose { + let mining_manager = self.mining_manager.clone(); + let response = self + .fee_estimate_verbose_cache + .get(async move { mining_manager.get_realtime_feerate_estimations_verbose().await.into_rpc() }) + .await; + Ok(response) + } else { + let estimate = self.get_fee_estimate_call(GetFeeEstimateRequest {}).await?.estimate; + Ok(GetFeeEstimateExperimentalResponse { estimate, verbose: None }) + } + } + async fn ping_call(&self, _: PingRequest) -> RpcResult { Ok(PingResponse {}) } diff --git a/rpc/wrpc/client/src/client.rs b/rpc/wrpc/client/src/client.rs index 7d8548171..e57024c4b 100644 --- a/rpc/wrpc/client/src/client.rs +++ b/rpc/wrpc/client/src/client.rs @@ -596,21 +596,23 @@ impl RpcApi for KaspaRpcClient { GetBlockTemplate, GetCoinSupply, GetConnectedPeerInfo, - GetDaaScoreTimestampEstimate, - GetServerInfo, GetCurrentNetwork, + GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, GetHeaders, GetInfo, GetMempoolEntries, GetMempoolEntriesByAddresses, GetMempoolEntry, - GetPeerAddresses, GetMetrics, + GetPeerAddresses, + GetServerInfo, GetSink, - GetSyncStatus, + GetSinkBlueScore, GetSubnetwork, + GetSyncStatus, GetUtxosByAddresses, - GetSinkBlueScore, GetVirtualChainFromBlock, Ping, ResolveFinalityConflict, diff --git a/rpc/wrpc/server/src/router.rs b/rpc/wrpc/server/src/router.rs index 4c0d2b3aa..09330eb49 100644 --- a/rpc/wrpc/server/src/router.rs +++ b/rpc/wrpc/server/src/router.rs @@ -44,22 +44,24 @@ impl Router { GetBlockTemplate, GetCoinSupply, GetConnectedPeerInfo, - GetDaaScoreTimestampEstimate, - GetServerInfo, GetCurrentNetwork, + GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, GetHeaders, GetInfo, GetInfo, GetMempoolEntries, GetMempoolEntriesByAddresses, GetMempoolEntry, - GetPeerAddresses, GetMetrics, + GetPeerAddresses, + GetServerInfo, GetSink, + GetSinkBlueScore, GetSubnetwork, GetSyncStatus, GetUtxosByAddresses, - GetSinkBlueScore, GetVirtualChainFromBlock, Ping, ResolveFinalityConflict, diff --git a/testing/integration/src/mempool_benchmarks.rs b/testing/integration/src/mempool_benchmarks.rs index 3df716594..00d9b7803 100644 --- a/testing/integration/src/mempool_benchmarks.rs +++ b/testing/integration/src/mempool_benchmarks.rs @@ -295,8 +295,8 @@ async fn bench_bbt_latency_2() { const BLOCK_COUNT: usize = usize::MAX; const MEMPOOL_TARGET: u64 = 600_000; - const TX_COUNT: usize = 1_400_000; - const TX_LEVEL_WIDTH: usize = 20_000; + const TX_COUNT: usize = 1_000_000; + const TX_LEVEL_WIDTH: usize = 300_000; const TPS_PRESSURE: u64 = u64::MAX; const SUBMIT_BLOCK_CLIENTS: usize = 20; diff --git a/testing/integration/src/rpc_tests.rs b/testing/integration/src/rpc_tests.rs index c584e6b64..dc26b539f 100644 --- a/testing/integration/src/rpc_tests.rs +++ b/testing/integration/src/rpc_tests.rs @@ -557,6 +557,33 @@ async fn sanity_test() { }) } + KaspadPayloadOps::GetFeeEstimate => { + let rpc_client = client.clone(); + tst!(op, { + let response = rpc_client.get_fee_estimate().await.unwrap(); + info!("{:?}", response.priority_bucket); + assert!(!response.normal_buckets.is_empty()); + assert!(!response.low_buckets.is_empty()); + for bucket in response.ordered_buckets() { + info!("{:?}", bucket); + } + }) + } + + KaspadPayloadOps::GetFeeEstimateExperimental => { + let rpc_client = client.clone(); + tst!(op, { + let response = rpc_client.get_fee_estimate_experimental(true).await.unwrap(); + assert!(!response.estimate.normal_buckets.is_empty()); + assert!(!response.estimate.low_buckets.is_empty()); + for bucket in response.estimate.ordered_buckets() { + info!("{:?}", bucket); + } + assert!(response.verbose.is_some()); + info!("{:?}", response.verbose); + }) + } + KaspadPayloadOps::NotifyBlockAdded => { let rpc_client = client.clone(); let id = listener_id; diff --git a/testing/integration/src/tasks/tx/sender.rs b/testing/integration/src/tasks/tx/sender.rs index 26a334a76..d29e74373 100644 --- a/testing/integration/src/tasks/tx/sender.rs +++ b/testing/integration/src/tasks/tx/sender.rs @@ -114,7 +114,7 @@ impl Task for TransactionSenderTask { break; } prev_mempool_size = mempool_size; - sleep(Duration::from_secs(1)).await; + sleep(Duration::from_secs(2)).await; } if stopper == Stopper::Signal { warn!("Tx sender task signaling to stop"); diff --git a/utils/Cargo.toml b/utils/Cargo.toml index a3002afab..dda05cb0e 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true repository.workspace = true [dependencies] +arc-swap.workspace = true parking_lot.workspace = true async-channel.workspace = true borsh.workspace = true diff --git a/utils/src/expiring_cache.rs b/utils/src/expiring_cache.rs new file mode 100644 index 000000000..175bea548 --- /dev/null +++ b/utils/src/expiring_cache.rs @@ -0,0 +1,152 @@ +use arc_swap::ArcSwapOption; +use std::{ + future::Future, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; + +struct Entry { + item: T, + timestamp: Instant, +} + +/// An expiring cache for a single object +pub struct ExpiringCache { + store: ArcSwapOption>, + refetch: Duration, + expire: Duration, + fetching: AtomicBool, +} + +impl ExpiringCache { + /// Constructs a new expiring cache where `fetch` is the amount of time required to trigger a data + /// refetch and `expire` is the time duration after which the stored item is guaranteed not to be returned. + /// + /// Panics if `refetch > expire`. + pub fn new(refetch: Duration, expire: Duration) -> Self { + assert!(refetch <= expire); + Self { store: Default::default(), refetch, expire, fetching: Default::default() } + } + + /// Returns the cached item or possibly fetches a new one using the `refetch_future` task. The + /// decision whether to refetch depends on the configured expiration and refetch times for this cache. + pub async fn get(&self, refetch_future: F) -> T + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + let mut fetching = false; + + { + let guard = self.store.load(); + if let Some(entry) = guard.as_ref() { + if let Some(elapsed) = Instant::now().checked_duration_since(entry.timestamp) { + if elapsed < self.refetch { + return entry.item.clone(); + } + // Refetch is triggered, attempt to capture the task + fetching = self.fetching.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst).is_ok(); + // If the fetch task is not captured and expire time is not over yet, return with prev value. Another + // thread is refetching the data but we can return with the not-too-old value + if !fetching && elapsed < self.expire { + return entry.item.clone(); + } + } + // else -- In rare cases where now < timestamp, fall through to re-update the cache + } + } + + // We reach here if either we are the refetching thread or the current data has fully expired + let new_item = refetch_future.await; + let timestamp = Instant::now(); + // Update the store even if we were not in charge of refetching - let the last thread make the final update + self.store.store(Some(Arc::new(Entry { item: new_item.clone(), timestamp }))); + + if fetching { + let result = self.fetching.compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst); + assert!(result.is_ok(), "refetching was captured") + } + + new_item + } +} + +#[cfg(test)] +mod tests { + use super::ExpiringCache; + use std::time::Duration; + use tokio::join; + + #[tokio::test] + #[ignore] + // Tested during development but can be sensitive to runtime machine times so there's no point + // in keeping it part of CI. The test should be activated if the ExpiringCache struct changes. + async fn test_expiring_cache() { + let fetch = Duration::from_millis(500); + let expire = Duration::from_millis(1000); + let mid_point = Duration::from_millis(700); + let expire_point = Duration::from_millis(1200); + let cache: ExpiringCache = ExpiringCache::new(fetch, expire); + + // Test two consecutive calls + let item1 = cache + .get(async move { + println!("first call"); + 1 + }) + .await; + assert_eq!(1, item1); + let item2 = cache + .get(async move { + // cache was just updated with item1, refetch should not be triggered + panic!("should not be called"); + }) + .await; + assert_eq!(1, item2); + + // Test two calls after refetch point + // Sleep until after the refetch point but before expire + tokio::time::sleep(mid_point).await; + let call3 = cache.get(async move { + println!("third call before sleep"); + // keep this refetch busy so that call4 still gets the first item + tokio::time::sleep(Duration::from_millis(100)).await; + println!("third call after sleep"); + 3 + }); + let call4 = cache.get(async move { + // refetch is captured by call3 and we should be before expire + panic!("should not be called"); + }); + let (item3, item4) = join!(call3, call4); + println!("item 3: {}, item 4: {}", item3, item4); + assert_eq!(3, item3); + assert_eq!(1, item4); + + // Test 2 calls after expire + tokio::time::sleep(expire_point).await; + let call5 = cache.get(async move { + println!("5th call before sleep"); + tokio::time::sleep(Duration::from_millis(100)).await; + println!("5th call after sleep"); + 5 + }); + let call6 = cache.get(async move { 6 }); + let (item5, item6) = join!(call5, call6); + println!("item 5: {}, item 6: {}", item5, item6); + assert_eq!(5, item5); + assert_eq!(6, item6); + + let item7 = cache + .get(async move { + // cache was just updated with item5, refetch should not be triggered + panic!("should not be called"); + }) + .await; + // call 5 finished after call 6 + assert_eq!(5, item7); + } +} diff --git a/utils/src/lib.rs b/utils/src/lib.rs index bd3143719..79e96c44f 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -2,6 +2,7 @@ pub mod any; pub mod arc; pub mod binary_heap; pub mod channel; +pub mod expiring_cache; pub mod hashmap; pub mod hex; pub mod iter; diff --git a/utils/src/vec.rs b/utils/src/vec.rs index 01bd59b9e..fa1d67a27 100644 --- a/utils/src/vec.rs +++ b/utils/src/vec.rs @@ -4,6 +4,10 @@ pub trait VecExtensions { /// Inserts the provided `value` at `index` while swapping the item at index to the end of the container fn swap_insert(&mut self, index: usize, value: T); + + /// Merges two containers one into the other and returns the result. The method is identical + /// to [`Vec::append`] but can be used more ergonomically in a fluent calling fashion + fn merge(self, other: Self) -> Self; } impl VecExtensions for Vec { @@ -19,4 +23,9 @@ impl VecExtensions for Vec { let loc = self.len() - 1; self.swap(index, loc); } + + fn merge(mut self, mut other: Self) -> Self { + self.append(&mut other); + self + } } diff --git a/wallet/core/src/tests/rpc_core_mock.rs b/wallet/core/src/tests/rpc_core_mock.rs index 70d8792dc..1e6d70c2b 100644 --- a/wallet/core/src/tests/rpc_core_mock.rs +++ b/wallet/core/src/tests/rpc_core_mock.rs @@ -252,6 +252,17 @@ impl RpcApi for RpcCoreMock { Err(RpcError::NotImplemented) } + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + Err(RpcError::NotImplemented) + } + + async fn get_fee_estimate_experimental_call( + &self, + _request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API