Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Fixed nested boolean offset (#1404)
Browse files Browse the repository at this point in the history
  • Loading branch information
ritchie46 authored Feb 14, 2023
1 parent b457d68 commit c12b600
Showing 1 changed file with 5 additions and 18 deletions.
23 changes: 5 additions & 18 deletions src/io/parquet/write/boolean/nested.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use parquet2::{encoding::Encoding, page::DataPage};
use super::super::{nested, utils, WriteOptions};
use super::basic::{build_statistics, encode_plain};
use crate::io::parquet::read::schema::is_nullable;
use crate::io::parquet::write::{slice_nested_leaf, Nested};
use crate::io::parquet::write::Nested;
use crate::{
array::{Array, BooleanArray},
error::Result,
Expand All @@ -18,34 +18,21 @@ pub fn array_to_page(
) -> Result<DataPage> {
let is_optional = is_nullable(&type_.field_info);

// we slice the leaf by the offsets as dremel only computes lengths and thus
// does NOT take the starting offset into account.
// By slicing the leaf array we also don't write too many values.
let (start, len) = slice_nested_leaf(nested);

let mut nested = nested.to_vec();
let array = array.clone().sliced(start, len);
if let Some(Nested::Primitive(_, _, c)) = nested.last_mut() {
*c = len;
} else {
unreachable!("")
}

let mut buffer = vec![];
let (repetition_levels_byte_length, definition_levels_byte_length) =
nested::write_rep_and_def(options.version, &nested, &mut buffer)?;
nested::write_rep_and_def(options.version, nested, &mut buffer)?;

encode_plain(&array, is_optional, &mut buffer)?;
encode_plain(array, is_optional, &mut buffer)?;

let statistics = if options.write_statistics {
Some(build_statistics(&array))
Some(build_statistics(array))
} else {
None
};

utils::build_plain_page(
buffer,
nested::num_values(&nested),
nested::num_values(nested),
nested[0].len(),
array.null_count(),
repetition_levels_byte_length,
Expand Down

0 comments on commit c12b600

Please sign in to comment.