[TieredStorage] Make HotStorageReader use AccountOffset type (solana-…

…labs#33964) [TieredStorage] Improve param naming of IndexBlockFormat (solana-labs#34033) [TieredStorage] HotStorageReader::get_account_offset (solana-labs#34031) [TieredStorage] Rename owners_offset to owners_block_offset (solana-labs#34047) [TieredStorage] HotStorageReader::get_account_address (solana-labs#34032) [TieredStorage] OwnersBlock (solana-labs#34052) [TieredStorage] HotStorageReader::get_owner_address (solana-labs#34053) [TieredStorage] Define OwnerOffset as u32 (solana-labs#34105) [TieredStorage] Use OwnerOffset type in TieredAccountMeta (solana-labs#34106) Refactors TieredStorageFile read/write methods (solana-labs#34147) [TieredStorage] Make IndexBlock persist u32 offsets (solana-labs#34133) [TieredStorage] Make IndexOffset use u32 (solana-labs#34152) Move MatchAccountOwnerError from append_vec to accounts_file (solana-labs#34187) [TieredStorage] Make AccountOffset use u32 (solana-labs#34151) [TieredStorage] Allow HotStorage to handle more account data (solana-labs#34155) [TieredStorage] Make AccountOffset a trait, introduce HotAccountOffset (solana-labs#34335) [TieredStorage] Improve comments for HOT_ACCOUNT_ALIGNMENT (solana-labs#34404) [TieredStorage] Unit-tests for checking invalid HotAccountOffset (solana-labs#34376) [TieredStorage] Boundary check for accessing hot account meta (solana-labs#34349) [TieredStorage] boundary check for get_account_address() (solana-labs#34529) Sanitizes tiered storage footer after reading from disk (solana-labs#34200) Adds read/write/get_pod() fns to tiered storage (solana-labs#34415) Uses consistent error types in tiered storage (solana-labs#34110) [TieredStorage] Boundary check for get_account_offset() (solana-labs#34531) [TieredStorage] HotStorageReader::account_matches_owners (solana-labs#34350) [TieredStorage] Fix typos in index.rs (solana-labs#34546) [TieredStorage] HotAccountsReader::get_account (solana-labs#34499) [TieredStorage] Rename AddressAndBlockOffsetOnly to AddressesThenOffsets (solana-labs#34658) [TieredStorage] HotStorageWriter::new() (solana-labs#34659) [TieredStorage] Include executable field into AccountMetaFlags (solana-labs#34724) [TieredStorage] Code refactoring for OwnersBlock (solana-labs#34854) [TieredStorage] In-memory struct for writing OwnersBlock (solana-labs#34853) [TieredStorage] writing hot account blocks and index blocks (solana-labs#34828) [TieredStorage] Use RENT_EXEMPT_RENT_EPOCH in HotStorageWriter (solana-labs#34950) [TieredStorage] Write owners block for HotAccountStorage (solana-labs#34927) [TieredStorage] Avoid AccountHash copy in AccountMetaOptionalFields (solana-labs#34969) [TieredStorage] Correct the HotStorage API for account_matches_owners (solana-labs#34967) [TS] Add get_account() and account_matches_owner() to TieredStorageReader (solana-labs#34968) [TieredStorage] Have HotStorageWriter::write_account() return Vec<StoredAccountInfo> (solana-labs#34929) [TieredStorage] Use IndexOffset in TieredStorageMeta and get_account() (solana-labs#35046) [TieredStorage] TieredStorageReader:: and HotStorageReader:: accounts() (solana-labs#35031) [TieredStorage] Enable hot-storage in TieredStorage::write_accounts() (solana-labs#35049) [TieredStorage] Put commonly used test functions into test_utils.rs (solana-labs#35065) [TieredStorage] Make TieredStorage::write_accounts() thread-safe (solana-labs#35143)
yhchiang-sol · Feb 18, 2024 · 683e453 · 683e453
1 parent 8c5b5f1
commit 683e453
Show file tree

Hide file tree

Showing 19 changed files with 2,052 additions and 351 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/accounts-db/Cargo.toml b/accounts-db/Cargo.toml
@@ -24,6 +24,7 @@ fnv = { workspace = true }
 fs-err = { workspace = true }
 im = { workspace = true, features = ["rayon", "serde"] }
 index_list = { workspace = true }
+indexmap = { workspace = true }
 itertools = { workspace = true }
 lazy_static = { workspace = true }
 log = { workspace = true }

diff --git a/accounts-db/src/account_storage/meta.rs b/accounts-db/src/account_storage/meta.rs
@@ -142,7 +142,7 @@ impl<'storage> StoredAccountMeta<'storage> {
     pub fn offset(&self) -> usize {
         match self {
             Self::AppendVec(av) => av.offset(),
-            Self::Hot(hot) => hot.index(),
+            Self::Hot(hot) => hot.index().0 as usize,
         }
     }
 

diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs
@@ -31,7 +31,7 @@ use {
             AccountStorage, AccountStorageStatus, ShrinkInProgress,
         },
         accounts_cache::{AccountsCache, CachedAccount, SlotCache},
-        accounts_file::{AccountsFile, AccountsFileError},
+        accounts_file::{AccountsFile, AccountsFileError, MatchAccountOwnerError},
         accounts_hash::{
             AccountHash, AccountsDeltaHash, AccountsHash, AccountsHashKind, AccountsHasher,
             CalcAccountsHashConfig, CalculateHashIntermediate, HashStats, IncrementalAccountsHash,
@@ -54,8 +54,7 @@ use {
             get_ancient_append_vec_capacity, is_ancient, AccountsToStore, StorageSelector,
         },
         append_vec::{
-            aligned_stored_size, AppendVec, MatchAccountOwnerError, APPEND_VEC_MMAPPED_FILES_OPEN,
-            STORE_META_OVERHEAD,
+            aligned_stored_size, AppendVec, APPEND_VEC_MMAPPED_FILES_OPEN, STORE_META_OVERHEAD,
         },
         cache_hash_data::{CacheHashData, CacheHashDataFileReference},
         contains::Contains,

diff --git a/accounts-db/src/accounts_file.rs b/accounts-db/src/accounts_file.rs
@@ -4,7 +4,7 @@ use {
             StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredAccountMeta,
         },
         accounts_hash::AccountHash,
-        append_vec::{AppendVec, AppendVecError, MatchAccountOwnerError},
+        append_vec::{AppendVec, AppendVecError},
         storable_accounts::StorableAccounts,
         tiered_storage::error::TieredStorageError,
     },
@@ -40,6 +40,14 @@ pub enum AccountsFileError {
     TieredStorageError(#[from] TieredStorageError),
 }
 
+#[derive(Error, Debug, PartialEq, Eq)]
+pub enum MatchAccountOwnerError {
+    #[error("The account owner does not match with the provided list")]
+    NoMatch,
+    #[error("Unable to load the account")]
+    UnableToLoad,
+}
+
 pub type Result<T> = std::result::Result<T, AccountsFileError>;
 
 #[derive(Debug)]

diff --git a/accounts-db/src/append_vec.rs b/accounts-db/src/append_vec.rs
@@ -10,7 +10,7 @@ use {
             AccountMeta, StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo,
             StoredAccountMeta, StoredMeta, StoredMetaWriteVersion,
         },
-        accounts_file::{AccountsFileError, Result, ALIGN_BOUNDARY_OFFSET},
+        accounts_file::{AccountsFileError, MatchAccountOwnerError, Result, ALIGN_BOUNDARY_OFFSET},
         accounts_hash::AccountHash,
         storable_accounts::StorableAccounts,
         u64_align,
@@ -96,14 +96,6 @@ impl<'append_vec> Iterator for AppendVecAccountsIter<'append_vec> {
     }
 }
 
-#[derive(Error, Debug, PartialEq, Eq)]
-pub enum MatchAccountOwnerError {
-    #[error("The account owner does not match with the provided list")]
-    NoMatch,
-    #[error("Unable to load the account")]
-    UnableToLoad,
-}
-
 /// References to account data stored elsewhere. Getting an `Account` requires cloning
 /// (see `StoredAccountMeta::clone_account()`).
 #[derive(PartialEq, Eq, Debug)]