From 14d42e4f33581dc0ea9c4eeaf0ff00ebec85dbb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=82=8E=E6=B3=BC?= Date: Thu, 25 Apr 2024 17:25:18 +0800 Subject: [PATCH] Fix: Load mebmership since `last_applied`, not `last_membership.index` on startup Modify `StorageHelper::last_membership_in_log()` to scan the log starting from the last applied index rather than the index of the last applied membership log. This change reduces unnecessary I/O operations during startup, previously caused by scanning from an incorrect starting point. --- openraft/src/storage/helper.rs | 9 ++--- openraft/src/testing/suite.rs | 61 ++++++++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/openraft/src/storage/helper.rs b/openraft/src/storage/helper.rs index 6b60b5bce..9a95bc06f 100644 --- a/openraft/src/storage/helper.rs +++ b/openraft/src/storage/helper.rs @@ -192,11 +192,9 @@ where /// /// Thus a raft node will only need to store at most two recent membership logs. pub async fn get_membership(&mut self) -> Result, StorageError> { - let (_, sm_mem) = self.state_machine.applied_state().await?; + let (last_applied, sm_mem) = self.state_machine.applied_state().await?; - let sm_mem_next_index = sm_mem.log_id().next_index(); - - let log_mem = self.last_membership_in_log(sm_mem_next_index).await?; + let log_mem = self.last_membership_in_log(last_applied.next_index()).await?; tracing::debug!(membership_in_sm=?sm_mem, membership_in_log=?log_mem, "RaftStorage::get_membership"); // There 2 membership configs in logs. @@ -234,6 +232,9 @@ where let st = self.log_store.get_log_state().await?; let mut end = st.last_log_id.next_index(); + + tracing::info!("load membership from log: [{}..{})", since_index, end); + let start = std::cmp::max(st.last_purged_log_id.next_index(), since_index); let step = 64; diff --git a/openraft/src/testing/suite.rs b/openraft/src/testing/suite.rs index 5038256cb..e3e3edc2d 100644 --- a/openraft/src/testing/suite.rs +++ b/openraft/src/testing/suite.rs @@ -85,7 +85,10 @@ where run_fut(run_test(builder, Self::last_membership_in_log_multi_step))?; run_fut(run_test(builder, Self::get_membership_initial))?; run_fut(run_test(builder, Self::get_membership_from_log_and_empty_sm))?; - run_fut(run_test(builder, Self::get_membership_from_log_and_sm))?; + run_fut(run_test(builder, Self::get_membership_from_empty_log_and_sm))?; + run_fut(run_test(builder, Self::get_membership_from_log_le_sm_last_applied))?; + run_fut(run_test(builder, Self::get_membership_from_log_gt_sm_last_applied_1))?; + run_fut(run_test(builder, Self::get_membership_from_log_gt_sm_last_applied_2))?; run_fut(run_test(builder, Self::get_initial_state_without_init))?; run_fut(run_test(builder, Self::get_initial_state_membership_from_log_and_sm))?; run_fut(run_test(builder, Self::get_initial_state_with_state))?; @@ -259,7 +262,10 @@ where Ok(()) } - pub async fn get_membership_from_log_and_sm(mut store: LS, mut sm: SM) -> Result<(), StorageError> { + pub async fn get_membership_from_empty_log_and_sm( + mut store: LS, + mut sm: SM, + ) -> Result<(), StorageError> { tracing::info!("--- no log, read membership from state machine"); { apply(&mut sm, [ @@ -279,10 +285,31 @@ where mem_state.effective().membership(), ); } + Ok(()) + } + pub async fn get_membership_from_log_le_sm_last_applied( + mut store: LS, + mut sm: SM, + ) -> Result<(), StorageError> { tracing::info!("--- membership presents in log, but smaller than last_applied, read from state machine"); { - append(&mut store, [membership_ent_0::(1, 1, btreeset! {1,2,3})]).await?; + apply(&mut sm, [ + blank_ent_0::(1, 1), + membership_ent_0::(1, 2, btreeset! {3,4,5}), + blank_ent_0::(1, 3), + blank_ent_0::(1, 4), + ]) + .await?; + + // Intentionally append a membership entry that does not match the state machine, + // in order to see which membership is loaded. + append(&mut store, [ + blank_ent_0::(1, 1), + blank_ent_0::(1, 2), + membership_ent_0::(1, 3, btreeset! {1,2,3}), + ]) + .await?; let mem_state = StorageHelper::new(&mut store, &mut sm).get_membership().await?; @@ -295,10 +322,23 @@ where mem_state.effective().membership(), ); } + Ok(()) + } + pub async fn get_membership_from_log_gt_sm_last_applied_1( + mut store: LS, + mut sm: SM, + ) -> Result<(), StorageError> { tracing::info!("--- membership presents in log and > sm.last_applied, read from log"); { + apply(&mut sm, [ + blank_ent_0::(1, 1), + membership_ent_0::(1, 2, btreeset! {3,4,5}), + ]) + .await?; + append(&mut store, [ + membership_ent_0::(1, 1, btreeset! {1,2,3}), blank_ent_0::(1, 2), membership_ent_0::(1, 3, btreeset! {7,8,9}), ]) @@ -315,10 +355,25 @@ where mem_state.effective().membership(), ); } + Ok(()) + } + pub async fn get_membership_from_log_gt_sm_last_applied_2( + mut store: LS, + mut sm: SM, + ) -> Result<(), StorageError> { tracing::info!("--- two membership present in log and > sm.last_applied, read 2 from log"); { + apply(&mut sm, [ + blank_ent_0::(1, 1), + membership_ent_0::(1, 2, btreeset! {3,4,5}), + ]) + .await?; + append(&mut store, [ + membership_ent_0::(1, 1, btreeset! {1,2,3}), + blank_ent_0::(1, 2), + membership_ent_0::(1, 3, btreeset! {7,8,9}), blank_ent_0::(1, 4), membership_ent_0::(1, 5, btreeset! {10,11}), ])