Skip to content

Commit

Permalink
Fixed error in declaring size of compressed dict page.
Browse files Browse the repository at this point in the history
  • Loading branch information
jorgecarleitao committed Dec 9, 2021
1 parent adbd7cd commit 7e5b059
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 4 deletions.
9 changes: 7 additions & 2 deletions src/page/page_dict/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,16 @@ impl EncodedDictPage {
pub struct CompressedDictPage {
pub(crate) buffer: Vec<u8>,
pub(crate) num_values: usize,
pub(crate) uncompressed_page_size: usize,
}

impl CompressedDictPage {
pub fn new(buffer: Vec<u8>, num_values: usize) -> Self {
Self { buffer, num_values }
pub fn new(buffer: Vec<u8>, uncompressed_page_size: usize, num_values: usize) -> Self {
Self {
buffer,
uncompressed_page_size,
num_values,
}
}
}

Expand Down
7 changes: 6 additions & 1 deletion src/write/compression.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,17 @@ fn compress_dict(
mut buffer,
num_values,
} = page;
let uncompressed_page_size = buffer.len();
if compression != Compression::Uncompressed {
compression::compress(compression, &buffer, &mut compressed_buffer)?;
} else {
std::mem::swap(&mut buffer, &mut compressed_buffer);
}
Ok(CompressedDictPage::new(compressed_buffer, num_values))
Ok(CompressedDictPage::new(
compressed_buffer,
uncompressed_page_size,
num_values,
))
}

pub fn compress(
Expand Down
2 changes: 1 addition & 1 deletion src/write/page.rs
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ fn assemble_data_page_header(compressed_page: &CompressedDataPage) -> ParquetPag
fn assemble_dict_page_header(page: &CompressedDictPage) -> ParquetPageHeader {
ParquetPageHeader {
type_: PageType::DICTIONARY_PAGE,
uncompressed_page_size: page.buffer.len() as i32,
uncompressed_page_size: page.uncompressed_page_size as i32,
compressed_page_size: page.buffer.len() as i32,
crc: None,
data_page_header: None,
Expand Down

0 comments on commit 7e5b059

Please sign in to comment.