Feat/txindex

stackslib/src/chainstate/coordinator/mod.rs

@@ -207,6 +207,9 @@ pub struct ChainsCoordinatorConfig {
     /// true: always wait for canonical anchor blocks, even if it stalls the chain
     /// false: proceed to process new chain history even if we're missing an anchor block.
     pub require_affirmed_anchor_blocks: bool,
+    /// true: enable transactions indexing
+    /// false: no transactions indexing
+    pub txindex: bool,
 }
 
 impl ChainsCoordinatorConfig {
@@ -215,6 +218,7 @@ impl ChainsCoordinatorConfig {
             always_use_affirmation_maps: true,
             require_affirmed_anchor_blocks: true,
             assume_present_anchor_blocks: true,
+            txindex: false,
         }
     }
 
@@ -223,6 +227,7 @@ impl ChainsCoordinatorConfig {
             always_use_affirmation_maps: false,
             require_affirmed_anchor_blocks: false,
             assume_present_anchor_blocks: false,
+            txindex: false,
         }
     }
 }
@@ -248,7 +253,7 @@ pub struct ChainsCoordinator<
     pub reward_set_provider: R,
     pub notifier: N,
     pub atlas_config: AtlasConfig,
-    config: ChainsCoordinatorConfig,
+    pub config: ChainsCoordinatorConfig,
     burnchain_indexer: B,
     /// Used to tell the P2P thread that the stackerdb
     ///  needs to be refreshed.

stackslib/src/chainstate/nakamoto/coordinator/mod.rs

@@ -801,6 +801,7 @@ impl<
                 &mut self.sortition_db,
                 &canonical_sortition_tip,
                 self.dispatcher,
+                self.config.txindex,
             ) {
                 Ok(receipt_opt) => receipt_opt,
                 Err(ChainstateError::InvalidStacksBlock(msg)) => {

stackslib/src/chainstate/nakamoto/coordinator/tests.rs

@@ -34,7 +34,7 @@ use stacks_common::types::chainstate::{
     BurnchainHeaderHash, StacksAddress, StacksBlockId, StacksPrivateKey, StacksPublicKey,
 };
 use stacks_common::types::{Address, StacksEpoch, StacksEpochId, StacksPublicKeyBuffer};
-use stacks_common::util::hash::Hash160;
+use stacks_common::util::hash::{to_hex, Hash160};
 use stacks_common::util::secp256k1::Secp256k1PrivateKey;
 use stacks_common::util::vrf::VRFProof;
 
@@ -1436,7 +1436,7 @@ fn pox_treatment() {
             false
         },
     );
-    let processing_result = peer.try_process_block(&invalid_block).unwrap_err();
+    let processing_result = peer.try_process_block(&invalid_block, false).unwrap_err();
     assert_eq!(
         processing_result.to_string(),
         "Bitvec does not match the block commit's PoX handling".to_string(),
@@ -1527,7 +1527,7 @@ fn pox_treatment() {
             false
         },
     );
-    let processing_result = peer.try_process_block(&invalid_block).unwrap_err();
+    let processing_result = peer.try_process_block(&invalid_block, false).unwrap_err();
     assert_eq!(
         processing_result.to_string(),
         "Bitvec does not match the block commit's PoX handling".to_string(),
@@ -1575,6 +1575,96 @@ fn pox_treatment() {
     );
 }
 
+#[test]
+// Test Transactions indexing system
+fn transactions_indexing() {
+    let private_key = StacksPrivateKey::from_seed(&[2]);
+    let addr = StacksAddress::p2pkh(false, &StacksPublicKey::from_private(&private_key));
+
+    let num_stackers: u32 = 4;
+    let mut signing_key_seed = num_stackers.to_be_bytes().to_vec();
+    signing_key_seed.extend_from_slice(&[1, 1, 1, 1]);
+    let signing_key = StacksPrivateKey::from_seed(signing_key_seed.as_slice());
+    let test_stackers = (0..num_stackers)
+        .map(|index| TestStacker {
+            signer_private_key: signing_key.clone(),
+            stacker_private_key: StacksPrivateKey::from_seed(&index.to_be_bytes()),
+            amount: u64::MAX as u128 - 10000,
+            pox_addr: Some(PoxAddress::Standard(
+                StacksAddress::new(
+                    C32_ADDRESS_VERSION_TESTNET_SINGLESIG,
+                    Hash160::from_data(&index.to_be_bytes()),
+                ),
+                Some(AddressHashMode::SerializeP2PKH),
+            )),
+            max_amount: None,
+        })
+        .collect::<Vec<_>>();
+    let test_signers = TestSigners::new(vec![signing_key]);
+    let mut pox_constants = TestPeerConfig::default().burnchain.pox_constants;
+    pox_constants.reward_cycle_length = 10;
+    pox_constants.v2_unlock_height = 21;
+    pox_constants.pox_3_activation_height = 26;
+    pox_constants.v3_unlock_height = 27;
+    pox_constants.pox_4_activation_height = 28;
+
+    let mut boot_plan = NakamotoBootPlan::new(function_name!())
+        .with_test_stackers(test_stackers.clone())
+        .with_test_signers(test_signers.clone())
+        .with_private_key(private_key);
+    boot_plan.pox_constants = pox_constants;
+
+    let mut peer = boot_plan.boot_into_nakamoto_peer(vec![], None);
+
+    // generate a new block with txindex
+    let (tracked_block, burn_height, ..) =
+        peer.single_block_tenure(&private_key, |_| {}, |_| {}, |_| false);
+
+    assert_eq!(peer.try_process_block(&tracked_block, true).unwrap(), true);
+
+    let tracked_block_id = tracked_block.block_id();
+
+    // generate a new block but without txindex
+    let (untracked_block, burn_height, ..) =
+        peer.single_block_tenure(&private_key, |_| {}, |_| {}, |_| false);
+
+    assert_eq!(
+        peer.try_process_block(&untracked_block, false).unwrap(),
+        true
+    );
+
+    let untracked_block_id = untracked_block.block_id();
+
+    let chainstate = &peer.stacks_node.unwrap().chainstate;
+
+    // compare transactions to what has been tracked
+    for tx in tracked_block.txs {
+        let current_tx_hex = to_hex(&tx.serialize_to_vec());
+        let (index_block_hash, tx_hex) =
+            NakamotoChainState::get_index_block_hash_and_tx_hex_from_txid(
+                &chainstate.index_conn(),
+                tx.txid(),
+            )
+            .unwrap()
+            .unwrap();
+        assert_eq!(index_block_hash, tracked_block_id);
+        assert_eq!(tx_hex, current_tx_hex);
+    }
+
+    // ensure untracked transactions are not recorded
+    for tx in untracked_block.txs {
+        assert_eq!(
+            NakamotoChainState::get_index_block_hash_and_tx_hex_from_txid(
+                &chainstate.index_conn(),
+                tx.txid(),
+            )
+            .unwrap()
+            .is_none(),
+            true
+        );
+    }
+}
+
 /// Test chainstate getters against an instantiated epoch2/Nakamoto chain.
 /// There are 11 epoch2 blocks and 2 nakamto tenure with 10 nakamoto blocks each
 /// Tests:

stackslib/src/chainstate/nakamoto/mod.rs

@@ -1869,6 +1869,7 @@ impl NakamotoChainState {
         sort_db: &mut SortitionDB,
         canonical_sortition_tip: &SortitionId,
         dispatcher_opt: Option<&T>,
+        txindex: bool,
     ) -> Result<Option<StacksEpochReceipt>, ChainstateError> {
         #[cfg(test)]
         fault_injection::stall_block_processing();
@@ -2162,6 +2163,20 @@ impl NakamotoChainState {
             tx_receipts.push(unlock_receipt);
         }
 
+        // if txindex is enabled, let's record each transaction in the transactions table
+        if txindex {
+            let block_id = next_ready_block.block_id();
+            let stacks_db_tx = stacks_chain_state.index_tx_begin();
+            for tx_receipt in tx_receipts.iter() {
+                Self::record_transaction(&stacks_db_tx, &block_id, tx_receipt);
+            }
+
+            let commit = stacks_db_tx.commit();
+            if commit.is_err() {
+                warn!("Could not index transactions: {}", commit.err().unwrap());
+            }
+        }
+
         // announce the block, if we're connected to an event dispatcher
         if let Some(dispatcher) = dispatcher_opt {
             let block_event = (
@@ -3539,6 +3554,42 @@ impl NakamotoChainState {
         Ok(())
     }
 
+    // Index a transaction in the transactions table (used by the lagacy STACKS_TRANSACTION_LOG and txindex)
+    pub fn record_transaction(
+        stacks_db_tx: &StacksDBTx,
+        block_id: &StacksBlockId,
+        tx_receipt: &StacksTransactionReceipt,
+    ) {
+        let insert =
+            "INSERT INTO transactions (txid, index_block_hash, tx_hex, result) VALUES (?, ?, ?, ?)";
+        let txid = tx_receipt.transaction.txid();
+        let tx_hex = tx_receipt.transaction.serialize_to_dbstring();
+        let result = tx_receipt.result.to_string();
+        let params = params![txid, block_id, tx_hex, result];
+        if let Err(e) = stacks_db_tx.execute(insert, params) {
+            warn!("Failed to record TX: {}", e);
+        }
+    }
+
+    // Get index_block_hash and transaction payload hex by txid from the transactions table
+    pub fn get_index_block_hash_and_tx_hex_from_txid(
+        conn: &Connection,
+        txid: Txid,
+    ) -> Result<Option<(StacksBlockId, String)>, ChainstateError> {
+        let sql = "SELECT index_block_hash, tx_hex FROM transactions WHERE txid = ?";
+        let args = params![txid];
+
+        let mut stmt = conn.prepare(sql)?;
+        Ok(stmt
+            .query_row(args, |row| {
+                let index_block_hash: StacksBlockId = row.get(0)?;
+                let tx_hex: String = row.get(1)?;
+
+                Ok((index_block_hash, tx_hex))
+            })
+            .optional()?)
+    }
+
     /// Fetch number of blocks signed for a given signer and reward cycle
     /// This is the data tracked by `record_block_signers()`
     pub fn get_signer_block_count(

stackslib/src/chainstate/nakamoto/shadow.rs

@@ -898,6 +898,7 @@ pub fn process_shadow_block(
             sort_db,
             &sort_tip.sortition_id,
             no_dispatch.as_ref(),
+            false,
         ) {
             Ok(receipt_opt) => receipt_opt,
             Err(ChainstateError::InvalidStacksBlock(msg)) => {

stackslib/src/chainstate/nakamoto/tests/node.rs

@@ -1384,7 +1384,11 @@ impl TestPeer<'_> {
         proof
     }
 
-    pub fn try_process_block(&mut self, block: &NakamotoBlock) -> Result<bool, ChainstateError> {
+    pub fn try_process_block(
+        &mut self,
+        block: &NakamotoBlock,
+        txindex: bool,
+    ) -> Result<bool, ChainstateError> {
         let mut sort_handle = self.sortdb.as_ref().unwrap().index_handle_at_tip();
         let stacks_tip = sort_handle.get_nakamoto_tip_block_id().unwrap().unwrap();
         let accepted = Relayer::process_new_nakamoto_block(
@@ -1407,6 +1411,7 @@ impl TestPeer<'_> {
                 self.sortdb.as_mut().unwrap(),
                 &sort_tip,
                 None,
+                txindex,
             )?
         else {
             return Ok(false);

stackslib/src/chainstate/stacks/db/mod.rs

@@ -643,16 +643,8 @@ impl<'a> ChainstateTx<'a> {
         events: &[StacksTransactionReceipt],
     ) {
         if *TRANSACTION_LOG {
-            let insert =
-                "INSERT INTO transactions (txid, index_block_hash, tx_hex, result) VALUES (?, ?, ?, ?)";
             for tx_event in events.iter() {
-                let txid = tx_event.transaction.txid();
-                let tx_hex = tx_event.transaction.serialize_to_dbstring();
-                let result = tx_event.result.to_string();
-                let params = params![txid, block_id, tx_hex, result];
-                if let Err(e) = self.tx.tx().execute(insert, params) {
-                    warn!("Failed to log TX: {}", e);
-                }
+                NakamotoChainState::record_transaction(&self.tx, block_id, tx_event);
             }
         }
         for tx_event in events.iter() {

stackslib/src/config/mod.rs

@@ -41,6 +41,7 @@ use crate::burnchains::bitcoin::BitcoinNetworkType;
 use crate::burnchains::{Burnchain, MagicBytes, PoxConstants, BLOCKSTACK_MAGIC_MAINNET};
 use crate::chainstate::nakamoto::signer_set::NakamotoSigners;
 use crate::chainstate::stacks::boot::MINERS_NAME;
+use crate::chainstate::stacks::db::TRANSACTION_LOG;
 use crate::chainstate::stacks::index::marf::MARFOpenOpts;
 use crate::chainstate::stacks::index::storage::TrieHashCalculationMode;
 use crate::chainstate::stacks::miner::{BlockBuilderSettings, MinerStatus};
@@ -1661,6 +1662,8 @@ pub struct NodeConfig {
     pub chain_liveness_poll_time_secs: u64,
     /// stacker DBs we replicate
     pub stacker_dbs: Vec<QualifiedContractIdentifier>,
+    /// enable transactions indexing
+    pub txindex: bool,
 }
 
 #[derive(Clone, Debug)]
@@ -1939,6 +1942,7 @@ impl Default for NodeConfig {
             fault_injection_hide_blocks: false,
             chain_liveness_poll_time_secs: 300,
             stacker_dbs: vec![],
+            txindex: false,
         }
     }
 }
@@ -2432,6 +2436,8 @@ pub struct NodeConfigFile {
     pub stacker_dbs: Option<Vec<String>>,
     /// fault injection: fail to push blocks with this probability (0-100)
     pub fault_injection_block_push_fail_probability: Option<u8>,
+    /// enable transactions indexing
+    pub txindex: bool,
 }
 
 impl NodeConfigFile {
@@ -2530,6 +2536,14 @@ impl NodeConfigFile {
             } else {
                 default_node_config.fault_injection_block_push_fail_probability
             },
+            txindex: if self.txindex && *TRANSACTION_LOG {
+                return Err(
+                    "STACKS_TRANSACTION_LOG is deprecated and cannot be used with txindex."
+                        .to_string(),
+                );
+            } else {
+                self.txindex
+            },
         };
         Ok(node_config)
     }

testnet/stacks-node/src/run_loop/nakamoto.rs

@@ -324,6 +324,7 @@ impl RunLoop {
                     require_affirmed_anchor_blocks: moved_config
                         .node
                         .require_affirmed_anchor_blocks,
+                    txindex: moved_config.node.txindex,
                 };
                 ChainsCoordinator::run(
                     coord_config,

testnet/stacks-node/src/run_loop/neon.rs

@@ -620,6 +620,7 @@ impl RunLoop {
                     require_affirmed_anchor_blocks: moved_config
                         .node
                         .require_affirmed_anchor_blocks,
+                    txindex: false,
                 };
                 ChainsCoordinator::run(
                     coord_config,