From 965d473a401df7a8deeb4d431d55ae9124d2ae24 Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Wed, 12 Oct 2022 04:51:44 +0000 Subject: [PATCH] Removed zip_validity --- src/array/binary/mod.rs | 4 ++-- src/array/boolean/iterator.rs | 4 ++-- src/array/boolean/mod.rs | 4 ++-- src/array/dictionary/mod.rs | 4 ++-- src/array/fixed_size_binary/iterator.rs | 6 +++--- src/array/fixed_size_list/iterator.rs | 4 ++-- src/array/iterator.rs | 3 +-- src/array/list/iterator.rs | 4 ++-- src/array/map/iterator.rs | 4 ++-- src/array/primitive/iterator.rs | 4 ++-- src/array/primitive/mod.rs | 4 ++-- src/array/struct_/iterator.rs | 4 ++-- src/array/utf8/mod.rs | 4 ++-- src/array/utf8/mutable.rs | 4 ++-- src/bitmap/utils/mod.rs | 8 -------- src/io/avro/write/serialize.rs | 6 +++--- src/io/json/write/serialize.rs | 6 +++--- tests/it/array/boolean/mod.rs | 12 ++++++++++++ tests/it/array/primitive/mod.rs | 12 ++++++++++++ tests/it/bitmap/utils/zip_validity.rs | 19 +++++++++++-------- 20 files changed, 69 insertions(+), 51 deletions(-) diff --git a/src/array/binary/mod.rs b/src/array/binary/mod.rs index a33e219aec1..17c29f1b30d 100644 --- a/src/array/binary/mod.rs +++ b/src/array/binary/mod.rs @@ -1,6 +1,6 @@ use crate::{ bitmap::{ - utils::{zip_validity, BitmapIter, ZipValidity}, + utils::{BitmapIter, ZipValidity}, Bitmap, }, buffer::Buffer, @@ -117,7 +117,7 @@ impl BinaryArray { /// Returns an iterator of `Option<&[u8]>` over every element of this array. pub fn iter(&self) -> ZipValidity<&[u8], BinaryValueIter, BitmapIter> { - zip_validity(self.values_iter(), self.validity.as_ref().map(|x| x.iter())) + ZipValidity::new(self.values_iter(), self.validity.as_ref().map(|x| x.iter())) } /// Returns an iterator of `&[u8]` over every element of this array, ignoring the validity diff --git a/src/array/boolean/iterator.rs b/src/array/boolean/iterator.rs index 53e297d6ed6..019ba5b388f 100644 --- a/src/array/boolean/iterator.rs +++ b/src/array/boolean/iterator.rs @@ -1,4 +1,4 @@ -use crate::bitmap::utils::{zip_validity, BitmapIter, ZipValidity}; +use crate::bitmap::utils::{BitmapIter, ZipValidity}; use crate::bitmap::IntoIter; use super::super::MutableArray; @@ -41,7 +41,7 @@ impl<'a> MutableBooleanArray { /// Returns an iterator over the optional values of this [`MutableBooleanArray`]. #[inline] pub fn iter(&'a self) -> ZipValidity, BitmapIter<'a>> { - zip_validity( + ZipValidity::new( self.values().iter(), self.validity().as_ref().map(|x| x.iter()), ) diff --git a/src/array/boolean/mod.rs b/src/array/boolean/mod.rs index 2d7f0af2285..2c33ef155ab 100644 --- a/src/array/boolean/mod.rs +++ b/src/array/boolean/mod.rs @@ -1,6 +1,6 @@ use crate::{ bitmap::{ - utils::{zip_validity, BitmapIter, ZipValidity}, + utils::{BitmapIter, ZipValidity}, Bitmap, MutableBitmap, }, datatypes::{DataType, PhysicalType}, @@ -88,7 +88,7 @@ impl BooleanArray { /// Returns an iterator over the optional values of this [`BooleanArray`]. #[inline] pub fn iter(&self) -> ZipValidity { - zip_validity( + ZipValidity::new( self.values().iter(), self.validity.as_ref().map(|x| x.iter()), ) diff --git a/src/array/dictionary/mod.rs b/src/array/dictionary/mod.rs index 0f70cd0ceb3..02d72ccd260 100644 --- a/src/array/dictionary/mod.rs +++ b/src/array/dictionary/mod.rs @@ -2,7 +2,7 @@ use std::hint::unreachable_unchecked; use crate::{ bitmap::{ - utils::{zip_validity, BitmapIter, ZipValidity}, + utils::{BitmapIter, ZipValidity}, Bitmap, }, datatypes::{DataType, IntegerType}, @@ -191,7 +191,7 @@ impl DictionaryArray { /// This function will allocate a new [`Scalar`] per item and is usually not performant. /// Consider calling `keys_iter` and `values`, downcasting `values`, and iterating over that. pub fn iter(&self) -> ZipValidity, DictionaryValuesIter, BitmapIter> { - zip_validity( + ZipValidity::new( DictionaryValuesIter::new(self), self.keys.validity().as_ref().map(|x| x.iter()), ) diff --git a/src/array/fixed_size_binary/iterator.rs b/src/array/fixed_size_binary/iterator.rs index 7081940ed6d..8eb7d58920b 100644 --- a/src/array/fixed_size_binary/iterator.rs +++ b/src/array/fixed_size_binary/iterator.rs @@ -1,5 +1,5 @@ use crate::array::{ArrayAccessor, ArrayValuesIter, MutableArray}; -use crate::bitmap::utils::{zip_validity, BitmapIter, ZipValidity}; +use crate::bitmap::utils::{BitmapIter, ZipValidity}; use super::{FixedSizeBinaryArray, MutableFixedSizeBinaryArray}; @@ -47,7 +47,7 @@ impl<'a> IntoIterator for &'a FixedSizeBinaryArray { impl<'a> FixedSizeBinaryArray { /// constructs a new iterator pub fn iter(&'a self) -> ZipValidity<&'a [u8], FixedSizeBinaryValuesIter<'a>, BitmapIter<'a>> { - zip_validity(self.values_iter(), self.validity.as_ref().map(|x| x.iter())) + ZipValidity::new(self.values_iter(), self.validity.as_ref().map(|x| x.iter())) } /// Returns iterator over the values of [`FixedSizeBinaryArray`] @@ -70,7 +70,7 @@ impl<'a> MutableFixedSizeBinaryArray { pub fn iter( &'a self, ) -> ZipValidity<&'a [u8], MutableFixedSizeBinaryValuesIter<'a>, BitmapIter<'a>> { - zip_validity( + ZipValidity::new( self.iter_values(), self.validity().as_ref().map(|x| x.iter()), ) diff --git a/src/array/fixed_size_list/iterator.rs b/src/array/fixed_size_list/iterator.rs index 423123852dd..4ee178da5a7 100644 --- a/src/array/fixed_size_list/iterator.rs +++ b/src/array/fixed_size_list/iterator.rs @@ -1,6 +1,6 @@ use crate::{ array::{Array, ArrayAccessor, ArrayValuesIter}, - bitmap::utils::{zip_validity, BitmapIter, ZipValidity}, + bitmap::utils::{BitmapIter, ZipValidity}, }; use super::FixedSizeListArray; @@ -36,7 +36,7 @@ impl<'a> IntoIterator for &'a FixedSizeListArray { impl<'a> FixedSizeListArray { /// Returns an iterator of `Option>` pub fn iter(&'a self) -> ZipIter<'a> { - zip_validity( + ZipValidity::new( FixedSizeListValuesIter::new(self), self.validity.as_ref().map(|x| x.iter()), ) diff --git a/src/array/iterator.rs b/src/array/iterator.rs index 97f440c2a43..5e8ed44d861 100644 --- a/src/array/iterator.rs +++ b/src/array/iterator.rs @@ -6,11 +6,10 @@ mod private { impl<'a, T: super::ArrayAccessor<'a>> Sealed for T {} } -/// +/// Sealed trait representing assess to a value of an array. /// # Safety /// Implementers of this trait guarantee that /// `value_unchecked` is safe when called up to `len` -/// Implementations must guarantee that pub unsafe trait ArrayAccessor<'a>: private::Sealed { type Item: 'a; unsafe fn value_unchecked(&'a self, index: usize) -> Self::Item; diff --git a/src/array/list/iterator.rs b/src/array/list/iterator.rs index b57cafee651..90ec9b4cb58 100644 --- a/src/array/list/iterator.rs +++ b/src/array/list/iterator.rs @@ -1,6 +1,6 @@ use crate::array::Offset; use crate::array::{Array, ArrayAccessor, ArrayValuesIter}; -use crate::bitmap::utils::{zip_validity, BitmapIter, ZipValidity}; +use crate::bitmap::utils::{BitmapIter, ZipValidity}; use super::ListArray; @@ -35,7 +35,7 @@ impl<'a, O: Offset> IntoIterator for &'a ListArray { impl<'a, O: Offset> ListArray { /// Returns an iterator of `Option>` pub fn iter(&'a self) -> ZipIter<'a, O> { - zip_validity( + ZipValidity::new( ListValuesIter::new(self), self.validity.as_ref().map(|x| x.iter()), ) diff --git a/src/array/map/iterator.rs b/src/array/map/iterator.rs index 485b8caf210..8bd8ce820ed 100644 --- a/src/array/map/iterator.rs +++ b/src/array/map/iterator.rs @@ -1,5 +1,5 @@ use crate::array::Array; -use crate::bitmap::utils::{zip_validity, BitmapIter, ZipValidity}; +use crate::bitmap::utils::{BitmapIter, ZipValidity}; use crate::trusted_len::TrustedLen; use super::MapArray; @@ -72,7 +72,7 @@ impl<'a> IntoIterator for &'a MapArray { impl<'a> MapArray { /// Returns an iterator of `Option>` pub fn iter(&'a self) -> ZipValidity, MapValuesIter<'a>, BitmapIter<'a>> { - zip_validity( + ZipValidity::new( MapValuesIter::new(self), self.validity.as_ref().map(|x| x.iter()), ) diff --git a/src/array/primitive/iterator.rs b/src/array/primitive/iterator.rs index 61ca2c9cb64..89a8842e8fd 100644 --- a/src/array/primitive/iterator.rs +++ b/src/array/primitive/iterator.rs @@ -1,6 +1,6 @@ use crate::{ array::MutableArray, - bitmap::utils::{zip_validity, BitmapIter, ZipValidity}, + bitmap::utils::{BitmapIter, ZipValidity}, bitmap::IntoIter as BitmapIntoIter, buffer::IntoIter, types::NativeType, @@ -35,7 +35,7 @@ impl<'a, T: NativeType> MutablePrimitiveArray { /// Returns an iterator over `Option` #[inline] pub fn iter(&'a self) -> ZipValidity<&'a T, std::slice::Iter<'a, T>, BitmapIter<'a>> { - zip_validity( + ZipValidity::new( self.values().iter(), self.validity().as_ref().map(|x| x.iter()), ) diff --git a/src/array/primitive/mod.rs b/src/array/primitive/mod.rs index 390b54631a7..2bbdbd05239 100644 --- a/src/array/primitive/mod.rs +++ b/src/array/primitive/mod.rs @@ -1,6 +1,6 @@ use crate::{ bitmap::{ - utils::{zip_validity, BitmapIter, ZipValidity}, + utils::{BitmapIter, ZipValidity}, Bitmap, }, buffer::Buffer, @@ -141,7 +141,7 @@ impl PrimitiveArray { /// Returns an iterator over the values and validity, `Option<&T>`. #[inline] pub fn iter(&self) -> ZipValidity<&T, std::slice::Iter, BitmapIter> { - zip_validity( + ZipValidity::new( self.values().iter(), self.validity().as_ref().map(|x| x.iter()), ) diff --git a/src/array/struct_/iterator.rs b/src/array/struct_/iterator.rs index fefdf32e835..c5c75a6b925 100644 --- a/src/array/struct_/iterator.rs +++ b/src/array/struct_/iterator.rs @@ -1,5 +1,5 @@ use crate::{ - bitmap::utils::{zip_validity, BitmapIter, ZipValidity}, + bitmap::utils::{BitmapIter, ZipValidity}, scalar::{new_scalar, Scalar}, trusted_len::TrustedLen, }; @@ -89,7 +89,7 @@ impl<'a> IntoIterator for &'a StructArray { impl<'a> StructArray { /// Returns an iterator of `Option>` pub fn iter(&'a self) -> ZipIter<'a> { - zip_validity( + ZipValidity::new( StructValueIter::new(self), self.validity.as_ref().map(|x| x.iter()), ) diff --git a/src/array/utf8/mod.rs b/src/array/utf8/mod.rs index 54419828d15..dc453a5c37b 100644 --- a/src/array/utf8/mod.rs +++ b/src/array/utf8/mod.rs @@ -1,6 +1,6 @@ use crate::{ bitmap::{ - utils::{zip_validity, BitmapIter, ZipValidity}, + utils::{BitmapIter, ZipValidity}, Bitmap, }, buffer::Buffer, @@ -133,7 +133,7 @@ impl Utf8Array { /// Returns an iterator of `Option<&str>` pub fn iter(&self) -> ZipValidity<&str, Utf8ValuesIter, BitmapIter> { - zip_validity(self.values_iter(), self.validity.as_ref().map(|x| x.iter())) + ZipValidity::new(self.values_iter(), self.validity.as_ref().map(|x| x.iter())) } /// Returns an iterator of `&str` diff --git a/src/array/utf8/mutable.rs b/src/array/utf8/mutable.rs index 0f4028850a3..4cc83da9011 100644 --- a/src/array/utf8/mutable.rs +++ b/src/array/utf8/mutable.rs @@ -4,7 +4,7 @@ use crate::array::physical_binary::*; use crate::{ array::{Array, MutableArray, Offset, TryExtend, TryPush}, bitmap::{ - utils::{zip_validity, BitmapIter, ZipValidity}, + utils::{BitmapIter, ZipValidity}, Bitmap, MutableBitmap, }, datatypes::DataType, @@ -206,7 +206,7 @@ impl MutableUtf8Array { /// Returns an iterator of `Option<&str>` pub fn iter(&self) -> ZipValidity<&str, MutableUtf8ValuesIter, BitmapIter> { - zip_validity(self.values_iter(), self.validity.as_ref().map(|x| x.iter())) + ZipValidity::new(self.values_iter(), self.validity.as_ref().map(|x| x.iter())) } /// Converts itself into an [`Array`]. diff --git a/src/bitmap/utils/mod.rs b/src/bitmap/utils/mod.rs index 9a56f7fed16..b064ffd8bed 100644 --- a/src/bitmap/utils/mod.rs +++ b/src/bitmap/utils/mod.rs @@ -141,11 +141,3 @@ pub fn count_zeros(slice: &[u8], offset: usize, len: usize) -> usize { len - set_count } - -/// Returns an iterator adapter that returns Option according to an optional validity. -pub fn zip_validity>( - values: I, - validity: Option, -) -> ZipValidity { - ZipValidity::new(values, validity) -} diff --git a/src/io/avro/write/serialize.rs b/src/io/avro/write/serialize.rs index 7fbcfdb7760..f3a56a22501 100644 --- a/src/io/avro/write/serialize.rs +++ b/src/io/avro/write/serialize.rs @@ -1,7 +1,7 @@ use avro_schema::schema::{Record, Schema as AvroSchema}; use avro_schema::write::encode; -use crate::bitmap::utils::zip_validity; +use crate::bitmap::utils::ZipValidity; use crate::datatypes::{IntervalUnit, PhysicalType, PrimitiveType}; use crate::types::months_days_ns; use crate::{array::*, datatypes::DataType}; @@ -126,7 +126,7 @@ fn list_optional<'a, O: Offset>(array: &'a ListArray, schema: &AvroSchema) -> .offsets() .windows(2) .map(|w| (w[1] - w[0]).to_usize() as i64); - let lengths = zip_validity(lengths, array.validity().as_ref().map(|x| x.iter())); + let lengths = ZipValidity::new(lengths, array.validity().as_ref().map(|x| x.iter())); Box::new(BufStreamingIterator::new( lengths, @@ -180,7 +180,7 @@ fn struct_optional<'a>(array: &'a StructArray, schema: &Record) -> BoxSerializer .map(|(x, schema)| new_serializer(x.as_ref(), schema)) .collect::>(); - let iterator = zip_validity(0..array.len(), array.validity().as_ref().map(|x| x.iter())); + let iterator = ZipValidity::new(0..array.len(), array.validity().as_ref().map(|x| x.iter())); Box::new(BufStreamingIterator::new( iterator, diff --git a/src/io/json/write/serialize.rs b/src/io/json/write/serialize.rs index afaf7dab04e..1e2c8445eb2 100644 --- a/src/io/json/write/serialize.rs +++ b/src/io/json/write/serialize.rs @@ -3,7 +3,7 @@ use lexical_core::ToLexical; use std::io::Write; use streaming_iterator::StreamingIterator; -use crate::bitmap::utils::zip_validity; +use crate::bitmap::utils::ZipValidity; use crate::datatypes::TimeUnit; use crate::io::iterator::BufStreamingIterator; use crate::temporal_conversions::{ @@ -103,7 +103,7 @@ fn struct_serializer<'a>( let names = array.fields().iter().map(|f| f.name.as_str()); Box::new(BufStreamingIterator::new( - zip_validity(0..array.len(), array.validity().map(|x| x.iter())), + ZipValidity::new(0..array.len(), array.validity().map(|x| x.iter())), move |maybe, buf| { if maybe.is_some() { let names = names.clone(); @@ -140,7 +140,7 @@ fn list_serializer<'a, O: Offset>( let mut serializer = new_serializer(array.values().as_ref()); Box::new(BufStreamingIterator::new( - zip_validity( + ZipValidity::new( array.offsets().windows(2), array.validity().map(|x| x.iter()), ), diff --git a/tests/it/array/boolean/mod.rs b/tests/it/array/boolean/mod.rs index 2dd47d480d9..63c1efd7869 100644 --- a/tests/it/array/boolean/mod.rs +++ b/tests/it/array/boolean/mod.rs @@ -131,3 +131,15 @@ fn from_iter() { let a: BooleanArray = iter.collect(); assert_eq!(a.len(), 2); } + +#[test] +fn into_iter() { + let data = vec![Some(true), None, Some(false)]; + let rev = data.clone().into_iter().rev(); + + let array: BooleanArray = data.clone().into_iter().collect(); + + assert_eq!(array.clone().into_iter().collect::>(), data); + + assert!(array.into_iter().rev().eq(rev)) +} diff --git a/tests/it/array/primitive/mod.rs b/tests/it/array/primitive/mod.rs index 5a618cba256..5189d4e3275 100644 --- a/tests/it/array/primitive/mod.rs +++ b/tests/it/array/primitive/mod.rs @@ -124,3 +124,15 @@ fn into_mut_3() { let array = PrimitiveArray::new(DataType::Int32, values, validity); assert!(array.into_mut().is_right()); } + +#[test] +fn into_iter() { + let data = vec![Some(1), None, Some(10)]; + let rev = data.clone().into_iter().rev(); + + let array: Int32Array = data.clone().into_iter().collect(); + + assert_eq!(array.clone().into_iter().collect::>(), data); + + assert!(array.into_iter().rev().eq(rev)) +} diff --git a/tests/it/bitmap/utils/zip_validity.rs b/tests/it/bitmap/utils/zip_validity.rs index d778fb001b0..9c6187f60a6 100644 --- a/tests/it/bitmap/utils/zip_validity.rs +++ b/tests/it/bitmap/utils/zip_validity.rs @@ -1,11 +1,14 @@ -use arrow2::bitmap::{utils::zip_validity, Bitmap}; +use arrow2::bitmap::{ + utils::{BitmapIter, ZipValidity}, + Bitmap, +}; #[test] fn basic() { let a = Bitmap::from([true, false]); let a = Some(a.iter()); let values = vec![0, 1]; - let zip = zip_validity(values.into_iter(), a); + let zip = ZipValidity::new(values.into_iter(), a); let a = zip.collect::>(); assert_eq!(a, vec![Some(0), None]); @@ -16,7 +19,7 @@ fn complete() { let a = Bitmap::from([true, false, true, false, true, false, true, false]); let a = Some(a.iter()); let values = vec![0, 1, 2, 3, 4, 5, 6, 7]; - let zip = zip_validity(values.into_iter(), a); + let zip = ZipValidity::new(values.into_iter(), a); let a = zip.collect::>(); assert_eq!( @@ -36,7 +39,7 @@ fn slices() { let end = x[1]; &values[start..end] }); - let zip = zip_validity(iter, a); + let zip = ZipValidity::new(iter, a); let a = zip.collect::>(); assert_eq!(a, vec![Some([1, 2].as_ref()), None]); @@ -47,7 +50,7 @@ fn byte() { let a = Bitmap::from([true, false, true, false, false, true, true, false, true]); let a = Some(a.iter()); let values = vec![0, 1, 2, 3, 4, 5, 6, 7, 8]; - let zip = zip_validity(values.into_iter(), a); + let zip = ZipValidity::new(values.into_iter(), a); let a = zip.collect::>(); assert_eq!( @@ -71,7 +74,7 @@ fn offset() { let a = Bitmap::from([true, false, true, false, false, true, true, false, true]).slice(1, 8); let a = Some(a.iter()); let values = vec![0, 1, 2, 3, 4, 5, 6, 7]; - let zip = zip_validity(values.into_iter(), a); + let zip = ZipValidity::new(values.into_iter(), a); let a = zip.collect::>(); assert_eq!( @@ -83,7 +86,7 @@ fn offset() { #[test] fn none() { let values = vec![0, 1, 2]; - let zip = zip_validity(values.into_iter(), None); + let zip = ZipValidity::new(values.into_iter(), None::); let a = zip.collect::>(); assert_eq!(a, vec![Some(0), Some(1), Some(2)]); @@ -94,7 +97,7 @@ fn rev() { let a = Bitmap::from([true, false, true, false, false, true, true, false, true]).slice(1, 8); let a = Some(a.iter()); let values = vec![0, 1, 2, 3, 4, 5, 6, 7]; - let zip = zip_validity(values.into_iter(), a); + let zip = ZipValidity::new(values.into_iter(), a); let result = zip.rev().collect::>(); let expected = vec![None, Some(1), None, None, Some(4), Some(5), None, Some(7)]