MiningManager, Mempool & BockTemplateBuilder (#103)

* Extend and describe consensus core constants

* Enhance MutableTransaction with Clone and id()

* Add calculate_transaction_output_estimated_serialized_size to ConsensusAPI

* Initial mempool golang to rust rewrite

* Refactor process_orphans_after_accepted_transaction and unorphan_transaction

* clean-up mempool errors

* Minor comments fix

* Move mass calculation primitives to consensus-core

* Add unit tests for mempool check_transaction_standard

* Enhance ConsensusAPI with modify_coinbase_payload

* Implement and use BlockTemplateBuilder & its cache

* Replace ahash with std siphash in mempool

* Some minor code cleanings

* Add mempool get_transactions_by_addresses

* Explicit overflow cases with checked_add

* Use separate calculation paths in is_transaction_output_dust in case of overflow

* Make TransactionsPool.all_transactions private

* Handle some todos, add/modify/clean comments, rename/simplify some names, enhance code

* Rename fill_inputs_and_get_missing_parents.rs to populate_entries_and_try_validate

* Add a topological index to the transactions pool

* Let revalidate_high_priority_transactions use the topological index and change the returned type

* Refactor pools code, add chained txs to orphan pool, document

* Move MempoolUtxoSet from Mempool to TransactionsPool

* Enforce the pools maximum capacities

* Optimize revalidation of high priority txs

* Add MiningManager unit tests & fix 3 bugs in Mempool, OrphanPool and MempoolUtxoSet

* Add a todo about the need for integration tests

* Apply clippy advice

* Make the BlockTemplateCache lifetime configurable

* Add a todo to test TransactionsSelector

* Use Arcs

* Verify that now is in the future of last update

* todos and rename

* Use Mutex + continue removing also on errors

* Refactor MutableTx to support internal tx being an Arc

* Add SelectorSourceTransaction type and use it to get all ready txs with minimal clone cost

* Rename SelectorSourceTransaction to CandidateTransaction and move it under MiningManager

* Apply clippy new uninlined-format-args lint

Co-authored-by: msutton <[email protected]>

Cargo.toml

@@ -15,6 +15,7 @@ members = [
     "crypto/merkle",
     "rpc/core",
     "rpc/grpc",
+    "mining",
 ]
 
 [workspace.package]
@@ -40,6 +41,7 @@ pow = { path = "consensus/pow" }
 kaspa-utils = { path = "utils" }
 rpc-core = { path = "rpc/core" }
 rpc-grpc = { path = "rpc/grpc" }
+mining = { path = "mining" }
 
 thiserror = "1"
 faster-hex = "0.6"
@@ -63,4 +65,4 @@ clap = { version = "4.0.23", features = ["derive"] }
 async-std = { version = "1.12.0", features = ['attributes'] }
 derive_more = { version = "0.99" }
 log = "0.4"
-cfg-if = "1.0.0"
+cfg-if = "1.0.0"

consensus/core/src/api/mod.rs

@@ -7,6 +7,7 @@ use crate::{
     coinbase::MinerData,
     errors::{
         block::{BlockProcessResult, RuleError},
+        coinbase::CoinbaseResult,
         tx::TxResult,
     },
     tx::{MutableTransaction, Transaction},
@@ -29,6 +30,8 @@ pub trait ConsensusApi: Send + Sync {
     fn calculate_transaction_mass(self: Arc<Self>, transaction: &Transaction) -> u64;
 
     fn get_virtual_daa_score(self: Arc<Self>) -> u64;
+
+    fn modify_coinbase_payload(self: Arc<Self>, payload: Vec<u8>, miner_data: &MinerData) -> CoinbaseResult<Vec<u8>>;
 }
 
 pub type DynConsensus = Arc<dyn ConsensusApi>;

consensus/core/src/constants.rs

@@ -1,9 +1,24 @@
+/// BLOCK_VERSION represents the current block version
 pub const BLOCK_VERSION: u16 = 1;
+
+/// TX_VERSION is the current latest supported transaction version.
 pub const TX_VERSION: u16 = 0;
+
 pub const LOCK_TIME_THRESHOLD: u64 = 500_000_000_000;
+
+/// MAX_SCRIPT_PUBLIC_KEY_VERSION is the current latest supported public key script version.
+pub const MAX_SCRIPT_PUBLIC_KEY_VERSION: u16 = 0;
+
+/// SompiPerKaspa is the number of sompi in one kaspa (1 KAS).
 pub const SOMPI_PER_KASPA: u64 = 100_000_000;
+
+/// MaxSompi is the maximum transaction amount allowed in sompi.
 pub const MAX_SOMPI: u64 = 29_000_000_000 * SOMPI_PER_KASPA;
 
+// MAX_TX_IN_SEQUENCE_NUM is the maximum sequence number the sequence field
+// of a transaction input can be.
+pub const MAX_TX_IN_SEQUENCE_NUM: u64 = u64::MAX;
+
 // SEQUENCE_LOCK_TIME_MASK is a mask that extracts the relative lock time
 // when masked against the transaction input sequence number.
 pub const SEQUENCE_LOCK_TIME_MASK: u64 = 0x00000000ffffffff;
@@ -12,3 +27,7 @@ pub const SEQUENCE_LOCK_TIME_MASK: u64 = 0x00000000ffffffff;
 // input's sequence number, the sequence number will not be interpreted
 // as a relative lock time.
 pub const SEQUENCE_LOCK_TIME_DISABLED: u64 = 1 << 63;
+
+/// UNACCEPTED_DAA_SCORE is used to for UtxoEntries that were created by
+/// transactions in the mempool, or otherwise not-yet-accepted transactions.
+pub const UNACCEPTED_DAA_SCORE: u64 = u64::MAX;

consensus/core/src/errors/tx.rs

@@ -2,7 +2,7 @@ use crate::constants::MAX_SOMPI;
 use crate::tx::TransactionOutpoint;
 use thiserror::Error;
 
-#[derive(Error, Debug, Clone)]
+#[derive(Error, Debug, Clone, PartialEq, Eq)]
 pub enum TxRuleError {
     #[error("transaction has no inputs")]
     NoTxInputs,

consensus/core/src/lib.rs

@@ -12,6 +12,7 @@ pub mod constants;
 pub mod errors;
 pub mod hashing;
 pub mod header;
+pub mod mass;
 pub mod merkle;
 pub mod muhash;
 pub mod networktype;
@@ -21,6 +22,8 @@ pub mod subnets;
 pub mod tx;
 pub mod utxo;
 
+pub mod testutils;
+
 /// Integer type for accumulated PoW of blue blocks. We expect no more than
 /// 2^128 work in a single block (btc has ~2^80), and no more than 2^64
 /// overall blocks, so 2^192 is definitely a justified upper-bound.

consensus/core/src/mass/mod.rs

@@ -0,0 +1,57 @@
+use crate::{
+    subnets::SUBNETWORK_ID_SIZE,
+    tx::{Transaction, TransactionInput, TransactionOutput},
+};
+use hashes::HASH_SIZE;
+
+// transaction_estimated_serialized_size is the estimated size of a transaction in some
+// serialization. This has to be deterministic, but not necessarily accurate, since
+// it's only used as the size component in the transaction and block mass limit
+// calculation.
+pub fn transaction_estimated_serialized_size(tx: &Transaction) -> u64 {
+    let mut size: u64 = 0;
+    size += 2; // Tx version (u16)
+    size += 8; // Number of inputs (u64)
+    let inputs_size: u64 = tx.inputs.iter().map(transaction_input_estimated_serialized_size).sum();
+    size += inputs_size;
+
+    size += 8; // number of outputs (u64)
+    let outputs_size: u64 = tx.outputs.iter().map(transaction_output_estimated_serialized_size).sum();
+    size += outputs_size;
+
+    size += 8; // lock time (u64)
+    size += SUBNETWORK_ID_SIZE as u64;
+    size += 8; // gas (u64)
+    size += HASH_SIZE as u64; // payload hash
+
+    size += 8; // length of the payload (u64)
+    size += tx.payload.len() as u64;
+    size
+}
+
+fn transaction_input_estimated_serialized_size(input: &TransactionInput) -> u64 {
+    let mut size = 0;
+    size += outpoint_estimated_serialized_size();
+
+    size += 8; // length of signature script (u64)
+    size += input.signature_script.len() as u64;
+
+    size += 8; // sequence (uint64)
+    size
+}
+
+const fn outpoint_estimated_serialized_size() -> u64 {
+    let mut size: u64 = 0;
+    size += HASH_SIZE as u64; // Previous tx ID
+    size += 4; // Index (u32)
+    size
+}
+
+pub fn transaction_output_estimated_serialized_size(output: &TransactionOutput) -> u64 {
+    let mut size: u64 = 0;
+    size += 8; // value (u64)
+    size += 2; // output.ScriptPublicKey.Version (u16)
+    size += 8; // length of script public key (u64)
+    size += output.script_public_key.script().len() as u64;
+    size
+}

consensus/core/src/sign.rs

@@ -3,22 +3,22 @@ use crate::{
         sighash::{calc_schnorr_signature_hash, SigHashReusedValues},
         sighash_type::SIG_HASH_ALL,
     },
-    tx::MutableTransaction,
+    tx::SignableTransaction,
 };
 
 /// Sign a transaction using schnorr
-pub fn sign(mut mutable_tx: MutableTransaction, privkey: [u8; 32]) -> MutableTransaction {
+pub fn sign(mut signable_tx: SignableTransaction, privkey: [u8; 32]) -> SignableTransaction {
     let schnorr_key = secp256k1::KeyPair::from_seckey_slice(secp256k1::SECP256K1, &privkey).unwrap();
     let mut reused_values = SigHashReusedValues::new();
-    for i in 0..mutable_tx.tx.inputs.len() {
-        let sig_hash = calc_schnorr_signature_hash(&mutable_tx.as_verifiable(), i, SIG_HASH_ALL, &mut reused_values);
+    for i in 0..signable_tx.tx.inputs.len() {
+        let sig_hash = calc_schnorr_signature_hash(&signable_tx.as_verifiable(), i, SIG_HASH_ALL, &mut reused_values);
         let msg = secp256k1::Message::from_slice(sig_hash.as_bytes().as_slice()).unwrap();
         let sig: [u8; 64] = *schnorr_key.sign_schnorr(msg).as_ref();
         // This represents OP_DATA_65 <SIGNATURE+SIGHASH_TYPE> (since signature length is 64 bytes and SIGHASH_TYPE is one byte)
-        mutable_tx.tx.inputs[i].signature_script = std::iter::once(65u8).chain(sig).chain([SIG_HASH_ALL.to_u8()]).collect();
+        signable_tx.tx.inputs[i].signature_script = std::iter::once(65u8).chain(sig).chain([SIG_HASH_ALL.to_u8()]).collect();
         // TODO: update sig_op_counts
     }
-    mutable_tx
+    signable_tx
 }
 
 #[cfg(test)]

consensus/core/src/subnets.rs

@@ -8,7 +8,9 @@ use serde::{Deserialize, Serialize};
 pub const SUBNETWORK_ID_SIZE: usize = 20;
 
 /// The domain representation of a Subnetwork ID
-#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema)]
+#[derive(
+    Debug, Clone, Default, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema,
+)]
 pub struct SubnetworkId([u8; SUBNETWORK_ID_SIZE]);
 
 impl AsRef<[u8]> for SubnetworkId {

consensus/core/src/testutils/create_transaction.rs

@@ -0,0 +1,32 @@
+use crate::{
+    constants::{MAX_TX_IN_SEQUENCE_NUM, TX_VERSION},
+    subnets::SUBNETWORK_ID_NATIVE,
+    tx::{Transaction, TransactionInput, TransactionOutpoint, TransactionOutput},
+};
+
+use super::op_true_script::*;
+
+// create_transaction create a transaction that spends the first output of provided transaction.
+// Assumes that the output being spent has opTrueScript as it's scriptPublicKey.
+// Creates the value of the spent output minus provided `fee` (in sompi).
+pub fn create_transaction(tx_to_spend: &Transaction, fee: u64) -> Transaction {
+    let (script_public_key, redeem_script) = op_true_script();
+
+    // TODO: call txscript.PayToScriptHashSignatureScript(redeemScript, nil) when available
+    let signature_script = pay_to_script_hash_signature_script(redeem_script, vec![]);
+
+    let previous_outpoint = TransactionOutpoint::new(tx_to_spend.id(), 0);
+    let input = TransactionInput::new(previous_outpoint, signature_script, MAX_TX_IN_SEQUENCE_NUM, 1);
+    let output = TransactionOutput::new(tx_to_spend.outputs[0].value - fee, script_public_key);
+    Transaction::new(TX_VERSION, vec![input], vec![output], 0, SUBNETWORK_ID_NATIVE, 0, vec![])
+}
+
+pub fn pay_to_script_hash_signature_script(redeem_script: Vec<u8>, signature: Vec<u8>) -> Vec<u8> {
+    // TODO: replace all calls to this fn with the txscript PayToScriptHashSignatureScript equivalent
+    let mut signature_script = vec![redeem_script.len() as u8];
+    signature_script.extend_from_slice(&redeem_script);
+    signature_script.push(signature.len() as u8);
+    signature_script.extend_from_slice(&signature);
+
+    signature_script
+}

consensus/core/src/testutils/mod.rs

@@ -0,0 +1,2 @@
+pub mod create_transaction;
+pub mod op_true_script;

consensus/core/src/testutils/op_true_script.rs

@@ -0,0 +1,25 @@
+use hashes::HasherBase;
+
+use crate::{
+    constants::MAX_SCRIPT_PUBLIC_KEY_VERSION,
+    tx::{scriptvec, ScriptPublicKey},
+};
+
+// op_true_script returns a P2SH script paying to an anyone-can-spend address,
+// The second return value is a redeemScript to be used with txscript.PayToScriptHashSignatureScript
+pub fn op_true_script() -> (ScriptPublicKey, Vec<u8>) {
+    // TODO: use txscript.OpTrue instead when available
+    let redeem_script = vec![81u8];
+
+    // TODO: use txscript.PayToScriptHashScript(redeemScript) when available
+    // This is just a hack
+    let mut hasher = hashes::TransactionSigningHash::new();
+    let redeem_script_hash = hasher.update(redeem_script.clone()).clone().finalize();
+    let mut script_public_key_script = scriptvec![170u8];
+    script_public_key_script.push(redeem_script_hash.as_bytes().len() as u8);
+    script_public_key_script.extend_from_slice(&redeem_script_hash.as_bytes());
+    script_public_key_script.push(135u8);
+
+    let script_public_key = ScriptPublicKey::new(MAX_SCRIPT_PUBLIC_KEY_VERSION, script_public_key_script);
+    (script_public_key, redeem_script)
+}