Only commit when necessary (ordinals#647)

popcnt1 · Oct 13, 2022 · 2aca6b2 · 2aca6b2
1 parent 592d8fe
commit 2aca6b2
Show file tree

Hide file tree

Showing 3 changed files with 79 additions and 17 deletions.
diff --git a/src/index.rs b/src/index.rs
@@ -4,7 +4,10 @@ use {
   bitcoin::BlockHeader,
   bitcoincore_rpc::{json::GetBlockHeaderResult, Auth, Client},
   rayon::iter::{IntoParallelRefIterator, ParallelIterator},
-  redb::{MultimapTableDefinition, ReadableMultimapTable, WriteStrategy},
+  redb::{
+    Database, MultimapTableDefinition, ReadableMultimapTable, ReadableTable, Table,
+    TableDefinition, WriteStrategy, WriteTransaction,
+  },
   std::sync::atomic::{AtomicBool, Ordering},
 };
 
@@ -52,9 +55,11 @@ pub(crate) enum List {
   Unspent(Vec<(u64, u64)>),
 }
 
+#[derive(Copy, Clone)]
 #[repr(u64)]
-enum Statistic {
+pub(crate) enum Statistic {
   OutputsTraversed = 0,
+  Commits = 1,
 }
 
 impl From<Statistic> for u64 {
@@ -213,6 +218,7 @@ impl Index {
       .map(|(height, _hash)| height + 1)
       .unwrap_or(0);
 
+    let mut uncomitted = 0;
     for (i, height) in (0..).zip(height..) {
       if let Some(height_limit) = self.height_limit {
         if height > height_limit {
@@ -222,17 +228,26 @@ impl Index {
 
       let done = self.index_block(&mut wtx, height)?;
 
-      if i > 0 && i % 1000 == 0 {
+      if !done {
+        uncomitted += 1;
+      }
+
+      if uncomitted > 0 && i % 1000 == 0 {
+        Self::increment_statistic(&wtx, Statistic::Commits, 1)?;
         wtx.commit()?;
         wtx = self.begin_write()?;
+        uncomitted = 0;
       }
 
       if done || INTERRUPTS.load(atomic::Ordering::Relaxed) > 0 {
         break;
       }
     }
 
-    wtx.commit()?;
+    if uncomitted > 0 {
+      Self::increment_statistic(&wtx, Statistic::Commits, 1)?;
+      wtx.commit()?;
+    }
 
     Ok(())
   }
@@ -245,7 +260,6 @@ impl Index {
     let mut height_to_hash = wtx.open_table(HEIGHT_TO_HASH)?;
     let mut ordinal_to_satpoint = wtx.open_table(ORDINAL_TO_SATPOINT)?;
     let mut outpoint_to_ordinal_ranges = wtx.open_table(OUTPOINT_TO_ORDINAL_RANGES)?;
-    let mut statistics = wtx.open_table(STATISTICS)?;
 
     let start = Instant::now();
     let mut ordinal_ranges_written = 0;
@@ -357,13 +371,7 @@ impl Index {
 
     height_to_hash.insert(&height, &block.block_hash().as_hash().into_inner())?;
 
-    statistics.insert(
-      &Statistic::OutputsTraversed.into(),
-      &(statistics
-        .get(&(Statistic::OutputsTraversed.into()))?
-        .unwrap_or(0)
-        + outputs_in_block),
-    )?;
+    Self::increment_statistic(wtx, Statistic::OutputsTraversed, outputs_in_block)?;
 
     log::info!(
       "Wrote {ordinal_ranges_written} ordinal ranges in {}ms",
@@ -377,7 +385,7 @@ impl Index {
     Ok(rtx::Rtx(self.database.begin_read()?))
   }
 
-  fn begin_write(&self) -> Result<redb::WriteTransaction> {
+  fn begin_write(&self) -> Result<WriteTransaction> {
     if cfg!(test) {
       let mut tx = self.database.begin_write()?;
       tx.set_durability(redb::Durability::None);
@@ -387,6 +395,27 @@ impl Index {
     }
   }
 
+  fn increment_statistic(wtx: &WriteTransaction, statistic: Statistic, n: u64) -> Result {
+    let mut statistics = wtx.open_table(STATISTICS)?;
+    statistics.insert(
+      &statistic.into(),
+      &(statistics.get(&(statistic.into()))?.unwrap_or(0) + n),
+    )?;
+    Ok(())
+  }
+
+  #[cfg(test)]
+  pub(crate) fn statistic(&self, statistic: Statistic) -> Result<u64> {
+    Ok(
+      self
+        .database
+        .begin_read()?
+        .open_table(STATISTICS)?
+        .get(&(statistic.into()))?
+        .unwrap_or(0),
+    )
+  }
+
   pub(crate) fn height(&self) -> Result<Height> {
     Ok(Height(self.begin_read()?.height()?))
   }
@@ -566,8 +595,7 @@ impl Index {
   pub(crate) fn insert_rune(&self, rune: &Rune) -> Result<(bool, sha256::Hash)> {
     let json = serde_json::to_string(rune)?;
     let hash = sha256::Hash::hash(json.as_ref());
-
-    let wtx = self.database.begin_write()?;
+    let wtx = self.begin_write()?;
 
     let created = wtx
       .open_table(HASH_TO_RUNE)?

diff --git a/src/main.rs b/src/main.rs
@@ -31,7 +31,6 @@ use {
   chrono::{DateTime, NaiveDateTime, Utc},
   clap::Parser,
   derive_more::{Display, FromStr},
-  redb::{Database, ReadableTable, Table, TableDefinition, WriteTransaction},
   regex::Regex,
   serde::{Deserialize, Serialize},
   std::{

diff --git a/src/subcommand/server.rs b/src/subcommand/server.rs
@@ -148,8 +148,8 @@ impl Server {
         .route("/output/:output", get(Self::output))
         .route("/range/:start/:end", get(Self::range))
         .route("/rare.txt", get(Self::rare_txt))
-        .route("/rune/:hash", get(Self::rune_get))
         .route("/rune", put(Self::rune_put))
+        .route("/rune/:hash", get(Self::rune_get))
         .route("/search", get(Self::search_by_query))
         .route("/search/:query", get(Self::search_by_path))
         .route("/static/*path", get(Self::static_asset))
@@ -1388,4 +1388,39 @@ mod tests {
     <dd><a href=/rune/8ca6ee12cb891766de56e5698a73cd6546f27a88bd27c8b8d914bc4162f9e4b5 class=monospace>8ca6ee12cb891766de56e5698a73cd6546f27a88bd27c8b8d914bc4162f9e4b5</a></dd>.*",
     );
   }
+
+  #[test]
+  fn commit_are_tracked() {
+    let server = TestServer::new();
+
+    assert_eq!(
+      server
+        .index
+        .statistic(crate::index::Statistic::Commits)
+        .unwrap(),
+      1
+    );
+
+    server.index.index().unwrap();
+
+    assert_eq!(
+      server
+        .index
+        .statistic(crate::index::Statistic::Commits)
+        .unwrap(),
+      1
+    );
+
+    server.bitcoin_rpc_server.mine_blocks(1);
+
+    server.index.index().unwrap();
+
+    assert_eq!(
+      server
+        .index
+        .statistic(crate::index::Statistic::Commits)
+        .unwrap(),
+      2
+    );
+  }
 }