Reduced memory usage in reading parquet

examples/parquet_read.rs

@@ -11,7 +11,7 @@ fn main() -> Result<()> {
     let file_path = &args[1];
 
     let reader = File::open(file_path)?;
-    let reader = read::FileReader::try_new(reader, Some(&[8]), None, None, None)?;
+    let reader = read::FileReader::try_new(reader, None, None, None, None)?;
 
     println!("{:#?}", reader.schema());
 
@@ -25,8 +25,8 @@ fn main() -> Result<()> {
 
     let start = SystemTime::now();
     for maybe_chunk in reader {
-        let columns = maybe_chunk?;
-        assert!(!columns.is_empty());
+        let chunk = maybe_chunk?;
+        assert!(!chunk.is_empty());
     }
     println!("took: {} ms", start.elapsed().unwrap().as_millis());
     Ok(())

src/io/parquet/read/deserialize/utils.rs

@@ -381,7 +381,7 @@ pub(super) fn extend_from_new_page<'a, T: Decoder<'a>>(
     let mut decoded = if let Some(decoded) = items.pop_back() {
         // there is a already a state => it must be incomplete...
         debug_assert!(
-            decoded.len() < chunk_size,
+            decoded.len() <= chunk_size,
             "the temp state is expected to be incomplete"
         );
         decoded
@@ -435,7 +435,7 @@ pub(super) fn next<'a, I: DataPages, D: Decoder<'a>>(
 
             extend_from_new_page(page, chunk_size, items, decoder);
 
-            if items.front().unwrap().len() < chunk_size {
+            if (items.len() == 1) && items.front().unwrap().len() < chunk_size {
                 MaybeNext::More
             } else {
                 let decoded = items.pop_front().unwrap();