Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Uses mmap for names in tiered storage #34127

Merged
merged 1 commit into from
Nov 16, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions accounts-db/src/tiered_storage/footer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -204,11 +204,11 @@ impl TieredStorageFooter {
Ok(footer)
}

pub fn new_from_mmap(map: &Mmap) -> TieredStorageResult<&TieredStorageFooter> {
let offset = map.len().saturating_sub(FOOTER_TAIL_SIZE);
let (footer_size, offset) = get_type::<u64>(map, offset)?;
let (_footer_version, offset) = get_type::<u64>(map, offset)?;
let (magic_number, _offset) = get_type::<TieredStorageMagicNumber>(map, offset)?;
pub fn new_from_mmap(mmap: &Mmap) -> TieredStorageResult<&TieredStorageFooter> {
let offset = mmap.len().saturating_sub(FOOTER_TAIL_SIZE);
let (footer_size, offset) = get_type::<u64>(mmap, offset)?;
let (_footer_version, offset) = get_type::<u64>(mmap, offset)?;
let (magic_number, _offset) = get_type::<TieredStorageMagicNumber>(mmap, offset)?;

if *magic_number != TieredStorageMagicNumber::default() {
return Err(TieredStorageError::MagicNumberMismatch(
Expand All @@ -217,8 +217,10 @@ impl TieredStorageFooter {
));
}

let (footer, _offset) =
get_type::<TieredStorageFooter>(map, map.len().saturating_sub(*footer_size as usize))?;
let (footer, _offset) = get_type::<TieredStorageFooter>(
mmap,
mmap.len().saturating_sub(*footer_size as usize),
)?;

Ok(footer)
}
Expand Down
14 changes: 7 additions & 7 deletions accounts-db/src/tiered_storage/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ impl IndexBlockFormat {
/// Returns the address of the account given the specified index.
pub fn get_account_address<'a>(
&self,
map: &'a Mmap,
mmap: &'a Mmap,
footer: &TieredStorageFooter,
index_offset: IndexOffset,
) -> TieredStorageResult<&'a Pubkey> {
Expand All @@ -87,14 +87,14 @@ impl IndexBlockFormat {
footer.index_block_offset as usize + std::mem::size_of::<Pubkey>() * index_offset.0
}
};
let (address, _) = get_type::<Pubkey>(map, account_offset)?;
let (address, _) = get_type::<Pubkey>(mmap, account_offset)?;
Ok(address)
}

/// Returns the offset to the account given the specified index.
pub fn get_account_offset(
&self,
map: &Mmap,
mmap: &Mmap,
footer: &TieredStorageFooter,
index_offset: IndexOffset,
) -> TieredStorageResult<AccountOffset> {
Expand All @@ -103,7 +103,7 @@ impl IndexBlockFormat {
let account_offset = footer.index_block_offset as usize
+ std::mem::size_of::<Pubkey>() * footer.account_entry_count as usize
+ index_offset.0 * std::mem::size_of::<u64>();
let (account_block_offset, _) = get_type(map, account_offset)?;
let (account_block_offset, _) = get_type(mmap, account_offset)?;
Ok(AccountOffset {
block: *account_block_offset,
})
Expand Down Expand Up @@ -160,14 +160,14 @@ mod tests {
.create(false)
.open(&path)
.unwrap();
let map = unsafe { MmapOptions::new().map(&file).unwrap() };
let mmap = unsafe { MmapOptions::new().map(&file).unwrap() };
for (i, index_entry) in index_entries.iter().enumerate() {
let account_offset = indexer
.get_account_offset(&map, &footer, IndexOffset(i))
.get_account_offset(&mmap, &footer, IndexOffset(i))
.unwrap();
assert_eq!(index_entry.block_offset, account_offset.block as u64);
let address = indexer
.get_account_address(&map, &footer, IndexOffset(i))
.get_account_address(&mmap, &footer, IndexOffset(i))
.unwrap();
assert_eq!(index_entry.address, address);
}
Expand Down
8 changes: 4 additions & 4 deletions accounts-db/src/tiered_storage/mmap_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,21 @@ pub fn get_type<T>(map: &Mmap, offset: usize) -> IoResult<(&T, usize)> {
/// doesn't overrun the internal buffer. Otherwise return an Error.
/// Also return the offset of the first byte after the requested data that
/// falls on a 64-byte boundary.
pub fn get_slice(map: &Mmap, offset: usize, size: usize) -> IoResult<(&[u8], usize)> {
pub fn get_slice(mmap: &Mmap, offset: usize, size: usize) -> IoResult<(&[u8], usize)> {
let (next, overflow) = offset.overflowing_add(size);
if overflow || next > map.len() {
if overflow || next > mmap.len() {
error!(
"Requested offset {} and size {} while mmap only has length {}",
offset,
size,
map.len()
mmap.len()
);
return Err(std::io::Error::new(
std::io::ErrorKind::AddrNotAvailable,
"Requested offset and data length exceeds the mmap slice",
));
}
let data = &map[offset..next];
let data = &mmap[offset..next];
let next = u64_align!(next);
let ptr = data.as_ptr();

Expand Down