Skip to content

Commit

Permalink
Update to latest arrow-rs, use updated UnionArray constructors
Browse files Browse the repository at this point in the history
  • Loading branch information
@mbrobbel
mbrobbel committed May 14, 2024
1 parent 6ef3144 commit c7febaa
Show file tree
Hide file tree
Showing 4 changed files with 42 additions and 47 deletions.
20 changes: 10 additions & 10 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 5 additions & 5 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,21 +43,21 @@ derive = ["dep:narrow-derive"]
uuid = ["dep:uuid"]

[dependencies]
arrow-array = { git = "https://github.com/apache/arrow-rs.git", rev = "c203785ca398b879960bffbd30b988c9728b7c23", default-features = false, optional = true }
arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "c203785ca398b879960bffbd30b988c9728b7c23", default-features = false, optional = true }
arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "c203785ca398b879960bffbd30b988c9728b7c23", default-features = false, optional = true }
arrow-array = { git = "https://github.com/apache/arrow-rs.git", rev = "d17b2067d637c563eb315571e719807a6f482784", default-features = false, optional = true }
arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "d17b2067d637c563eb315571e719807a6f482784", default-features = false, optional = true }
arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "d17b2067d637c563eb315571e719807a6f482784", default-features = false, optional = true }
narrow-derive = { path = "narrow-derive", version = "^0.5.0", optional = true }
uuid = { version = "1.8.0", default-features = false, optional = true }

[dev-dependencies]
arrow-cast = { git = "https://github.com/apache/arrow-rs.git", rev = "c203785ca398b879960bffbd30b988c9728b7c23", default-features = false, features = [
arrow-cast = { git = "https://github.com/apache/arrow-rs.git", rev = "d17b2067d637c563eb315571e719807a6f482784", default-features = false, features = [
"prettyprint",
] }
bytes = "1.5.0"
criterion = { version = "0.5.1", default-features = false }
rand = { version = "0.8.5", default-features = false, features = ["small_rng"] }
rustversion = "1.0.14"
parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "c203785ca398b879960bffbd30b988c9728b7c23", default-features = false, features = [
parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "d17b2067d637c563eb315571e719807a6f482784", default-features = false, features = [
"arrow",
] }
uuid = { version = "1.8.0", default-features = false }
Expand Down
11 changes: 2 additions & 9 deletions narrow-derive/src/enum.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -746,19 +746,12 @@ impl<'a> Enum<'a> {
let (impl_generics, _, where_clause) = generics.split_for_impl();
let (_, ty_generics, _) = generics.split_for_impl();
let idx = self.variant_indices();
let variant_idx = (0..self.variants.len()).map(|idx| idx.to_string());
let tokens = quote! {
impl #impl_generics ::std::convert::From<#ident #ty_generics> for ::std::vec::Vec<(::arrow_schema::Field, ::std::sync::Arc<dyn ::arrow_array::Array>)> #where_clause {
impl #impl_generics ::std::convert::From<#ident #ty_generics> for ::std::vec::Vec<::std::sync::Arc<dyn ::arrow_array::Array>> #where_clause {
fn from(value: #ident #ty_generics) -> Self {
vec![
#(
(
<<
<#self_ident as #narrow::array::union::EnumVariant<#idx>>::Data as #narrow::array::ArrayType
<<#self_ident #self_ty_generics as #narrow::array::union::EnumVariant<#idx>>::Data>>::Array<Buffer,OffsetItem,UnionLayout> as #narrow::arrow::Array
>::as_field(#variant_idx),
value.#idx.into()
),
value.#idx.into(),
)*
]
}
Expand Down
48 changes: 25 additions & 23 deletions src/arrow/array/union.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,22 +78,23 @@ impl<
where
for<'a> i8: From<&'a T>,
<T as UnionArrayType<VARIANTS>>::Array<Buffer, OffsetItem, SparseLayout>:
UnionArrayTypeFields<VARIANTS> + Into<Vec<(Field, Arc<dyn arrow_array::Array>)>>,
UnionArrayTypeFields<VARIANTS> + Into<Vec<Arc<dyn arrow_array::Array>>>,
arrow_buffer::ScalarBuffer<i8>: From<FixedSizePrimitiveArray<i8, false, Buffer>>,
UnionArray<T, VARIANTS, SparseLayout, Buffer, OffsetItem>: crate::arrow::Array,

Check warning on line 83 in src/arrow/array/union.rs

View check run for this annotation

Codecov / codecov/patch

src/arrow/array/union.rs#L81-L83 

Added lines #L81 - L83 were not covered by tests
{
fn from(value: UnionArray<T, VARIANTS, SparseLayout, Buffer, OffsetItem>) -> Self {
let union_fields = match < UnionArray<T, VARIANTS, SparseLayout, Buffer, OffsetItem> as crate::arrow::Array>::as_field("").data_type() {
DataType::Union(fields, _mode) => fields,
_ => unreachable!(),

Check warning on line 88 in src/arrow/array/union.rs

View check run for this annotation

Codecov / codecov/patch

src/arrow/array/union.rs#L88 

Added line #L88 was not covered by tests
}.to_owned();
// Safety:
// - todo
unsafe {
arrow_array::UnionArray::new_unchecked(
&<<T as UnionArrayType<VARIANTS>>::Array<
Buffer,
OffsetItem,
SparseLayout,
> as UnionArrayTypeFields<VARIANTS>>::type_ids(),
arrow_buffer::ScalarBuffer::from(value.0.types).into_inner(),
union_fields,
arrow_buffer::ScalarBuffer::from(value.0.types),
None,
value.0.variants.into()
value.0.variants.into(),
)
}
}
Expand All @@ -108,23 +109,24 @@ impl<
where
for<'a> i8: From<&'a T>,
<T as UnionArrayType<VARIANTS>>::Array<Buffer, OffsetItem, DenseLayout>:
UnionArrayTypeFields<VARIANTS> + Into<Vec<(Field, Arc<dyn arrow_array::Array>)>>,
UnionArrayTypeFields<VARIANTS> + Into<Vec<Arc<dyn arrow_array::Array>>>,

Check warning on line 112 in src/arrow/array/union.rs

View check run for this annotation

Codecov / codecov/patch

src/arrow/array/union.rs#L111-L112 

Added lines #L111 - L112 were not covered by tests
arrow_buffer::ScalarBuffer<i8>: From<FixedSizePrimitiveArray<i8, false, Buffer>>,
arrow_buffer::ScalarBuffer<i32>: From<FixedSizePrimitiveArray<i32, false, Buffer>>,
UnionArray<T, VARIANTS, SparseLayout, Buffer, OffsetItem>: crate::arrow::Array,
{
fn from(value: UnionArray<T, VARIANTS, DenseLayout, Buffer, OffsetItem>) -> Self {
let union_fields = match < UnionArray<T, VARIANTS, DenseLayout, Buffer, OffsetItem> as crate::arrow::Array>::as_field("").data_type() {
DataType::Union(fields, _mode) => fields,
_ => unreachable!(),

Check warning on line 120 in src/arrow/array/union.rs

View check run for this annotation

Codecov / codecov/patch

src/arrow/array/union.rs#L120 

Added line #L120 was not covered by tests
}.to_owned();
// Safety:
// - todo
unsafe {
arrow_array::UnionArray::new_unchecked(
&<<T as UnionArrayType<VARIANTS>>::Array<
Buffer,
OffsetItem,
DenseLayout,
> as UnionArrayTypeFields<VARIANTS>>::type_ids(),
arrow_buffer::ScalarBuffer::from(value.0.types).into_inner(),
Some(arrow_buffer::ScalarBuffer::<i32>::from(value.0.offsets).into_inner()),
value.0.variants.into()
union_fields,
arrow_buffer::ScalarBuffer::from(value.0.types),
Some(arrow_buffer::ScalarBuffer::<i32>::from(value.0.offsets)),
value.0.variants.into(),
)
}
}
Expand All @@ -143,12 +145,12 @@ where
FromIterator<Arc<dyn arrow_array::Array>>,
{
fn from(value: arrow_array::UnionArray) -> Self {
let (types, offsets_opt, _field_type_ids, variants) = value.into_parts();
let (_union_fields, type_ids, offsets_opt, variants) = value.into_parts();
match offsets_opt {
Some(_) => panic!("expected array without offsets"),
None => Self(SparseUnionArray {
variants: variants.into_iter().map(|(_, array)| array).collect(),
types: types.into(),
variants: variants.into_iter().collect(),
types: type_ids.into(),
}),
}
}
Expand All @@ -168,13 +170,13 @@ where
FromIterator<Arc<dyn arrow_array::Array>>,
{
fn from(value: arrow_array::UnionArray) -> Self {
let (types, offsets_opt, _field_types_ids, variants) = value.into_parts();
let (_union_fields, type_ids, offsets_opt, variants) = value.into_parts();
match offsets_opt {
None => panic!("expected array with offsets"),
Some(offsets) => Self(DenseUnionArray {
variants: variants.into_iter().map(|(_, array)| array).collect(),
variants: variants.into_iter().collect(),
offsets: offsets.into(),
types: types.into(),
types: type_ids.into(),
}),
}
}
Expand Down

0 comments on commit c7febaa

Please sign in to comment.