From e70a464a5abe91cee93d0f5d32b203e8d230bb66 Mon Sep 17 00:00:00 2001 From: Ritchie Vink Date: Tue, 14 Feb 2023 14:51:02 +0100 Subject: [PATCH 1/2] fix: fix nested boolean offset --- src/io/parquet/write/boolean/nested.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/io/parquet/write/boolean/nested.rs b/src/io/parquet/write/boolean/nested.rs index d86c9b64567..ebc467fe549 100644 --- a/src/io/parquet/write/boolean/nested.rs +++ b/src/io/parquet/write/boolean/nested.rs @@ -18,18 +18,7 @@ pub fn array_to_page( ) -> Result { let is_optional = is_nullable(&type_.field_info); - // we slice the leaf by the offsets as dremel only computes lengths and thus - // does NOT take the starting offset into account. - // By slicing the leaf array we also don't write too many values. - let (start, len) = slice_nested_leaf(nested); - let mut nested = nested.to_vec(); - let array = array.clone().sliced(start, len); - if let Some(Nested::Primitive(_, _, c)) = nested.last_mut() { - *c = len; - } else { - unreachable!("") - } let mut buffer = vec![]; let (repetition_levels_byte_length, definition_levels_byte_length) = From eae66982331e74ab4ce77acebb11bf15fab06b57 Mon Sep 17 00:00:00 2001 From: Ritchie Vink Date: Tue, 14 Feb 2023 15:31:36 +0100 Subject: [PATCH 2/2] clippy --- src/io/parquet/write/boolean/nested.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/io/parquet/write/boolean/nested.rs b/src/io/parquet/write/boolean/nested.rs index ebc467fe549..9d9e49100f6 100644 --- a/src/io/parquet/write/boolean/nested.rs +++ b/src/io/parquet/write/boolean/nested.rs @@ -4,7 +4,7 @@ use parquet2::{encoding::Encoding, page::DataPage}; use super::super::{nested, utils, WriteOptions}; use super::basic::{build_statistics, encode_plain}; use crate::io::parquet::read::schema::is_nullable; -use crate::io::parquet::write::{slice_nested_leaf, Nested}; +use crate::io::parquet::write::Nested; use crate::{ array::{Array, BooleanArray}, error::Result, @@ -18,23 +18,21 @@ pub fn array_to_page( ) -> Result { let is_optional = is_nullable(&type_.field_info); - let mut nested = nested.to_vec(); - let mut buffer = vec![]; let (repetition_levels_byte_length, definition_levels_byte_length) = - nested::write_rep_and_def(options.version, &nested, &mut buffer)?; + nested::write_rep_and_def(options.version, nested, &mut buffer)?; - encode_plain(&array, is_optional, &mut buffer)?; + encode_plain(array, is_optional, &mut buffer)?; let statistics = if options.write_statistics { - Some(build_statistics(&array)) + Some(build_statistics(array)) } else { None }; utils::build_plain_page( buffer, - nested::num_values(&nested), + nested::num_values(nested), nested[0].len(), array.null_count(), repetition_levels_byte_length,