Skip to content

Commit

Permalink
fix(parquet): nested struct /list writing (jorgecarleitao#1347)
Browse files Browse the repository at this point in the history
  • Loading branch information
ritchie46 committed Mar 29, 2023
1 parent 977c2e0 commit dcd4949
Showing 1 changed file with 30 additions and 2 deletions.
32 changes: 30 additions & 2 deletions src/io/parquet/write/nested/rep.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,21 @@ impl<A: Iterator<Item = usize> + std::fmt::Debug> DebugIter for A {}
fn iter<'a>(nested: &'a [Nested]) -> Vec<Box<dyn DebugIter + 'a>> {
nested
.iter()
.filter_map(|nested| match nested {
.enumerate()
.filter_map(|(i, nested)| match nested {
Nested::Primitive(_, _, _) => None,
Nested::List(nested) => Some(Box::new(to_length(nested.offsets)) as Box<dyn DebugIter>),
Nested::LargeList(nested) => {
Some(Box::new(to_length(nested.offsets)) as Box<dyn DebugIter>)
}
Nested::Struct(_, _, length) => {
Some(Box::new(std::iter::repeat(1usize).take(*length)) as Box<dyn DebugIter>)
// only return 1, 1, 1, (x len) if struct is outer structure.
// otherwise treat as leaf
if i == 0 {
Some(Box::new(std::iter::repeat(1usize).take(*length)) as Box<dyn DebugIter>)
} else {
None
}
}
})
.collect()
Expand Down Expand Up @@ -223,4 +230,25 @@ mod tests {

test(nested, expected)
}

#[test]
fn list_struct_list() {
let nested = vec![
Nested::List(ListNested {
is_optional: true,
offsets: &[0i32, 2, 3],
validity: None,
}),
Nested::Struct(None, true, 3),
Nested::List(ListNested {
is_optional: true,
offsets: &[0i32, 3, 6, 7],
validity: None,
}),
Nested::Primitive(None, true, 7),
];
let expected = vec![0, 2, 2, 1, 2, 2, 0];

test(nested, expected)
}
}

0 comments on commit dcd4949

Please sign in to comment.