diff --git a/src/alloc/mod.rs b/src/alloc/mod.rs index 0686b613a77..36e8a78b8db 100644 --- a/src/alloc/mod.rs +++ b/src/alloc/mod.rs @@ -131,56 +131,3 @@ pub unsafe fn reallocate( handle_alloc_error(Layout::from_size_align_unchecked(new_size, ALIGNMENT)) }) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_allocate_dangling() { - let p = allocate_aligned::(0); - assert_eq!(0, (p.as_ptr() as usize) % ALIGNMENT); - } - - #[test] - fn test_allocate() { - let p = allocate_aligned::(1024); - assert_eq!(0, (p.as_ptr() as usize) % ALIGNMENT); - unsafe { free_aligned(p, 1024) }; - } - - #[test] - fn test_allocate_zeroed() { - let p = allocate_aligned_zeroed::(1024); - assert_eq!(0, (p.as_ptr() as usize) % ALIGNMENT); - unsafe { free_aligned(p, 1024) }; - } - - #[test] - fn test_reallocate_from_zero() { - let ptr = allocate_aligned::(0); - let ptr = unsafe { reallocate(ptr, 0, 512) }; - unsafe { free_aligned(ptr, 512) }; - } - - #[test] - fn test_reallocate_from_alloc() { - let ptr = allocate_aligned::(32); - let ptr = unsafe { reallocate(ptr, 32, 64) }; - unsafe { free_aligned(ptr, 64) }; - } - - #[test] - fn test_reallocate_smaller() { - let ptr = allocate_aligned::(32); - let ptr = unsafe { reallocate(ptr, 32, 16) }; - unsafe { free_aligned(ptr, 16) }; - } - - #[test] - fn test_reallocate_to_zero() { - let ptr = allocate_aligned::(32); - let ptr = unsafe { reallocate(ptr, 32, 0) }; - assert_eq!(ptr, unsafe { dangling() }); - } -} diff --git a/src/array/binary/from.rs b/src/array/binary/from.rs index 416047ad518..c8efdfe462a 100644 --- a/src/array/binary/from.rs +++ b/src/array/binary/from.rs @@ -87,18 +87,3 @@ where (null.into(), offsets.into(), values.into()) } - -#[cfg(test)] -mod tests { - use crate::array::Array; - - use super::*; - - #[test] - fn test_from() { - let array = BinaryArray::::from(&[Some(b"hello".as_ref()), Some(b" ".as_ref()), None]); - - let a = array.validity().as_ref().unwrap(); - assert_eq!(a, &Bitmap::from([true, true, false])); - } -} diff --git a/src/array/binary/mod.rs b/src/array/binary/mod.rs index 96425bac9e0..b082e954bfe 100644 --- a/src/array/binary/mod.rs +++ b/src/array/binary/mod.rs @@ -146,52 +146,3 @@ unsafe impl GenericBinaryArray for BinaryArray { self.offsets() } } - -#[cfg(test)] -mod tests { - use super::*; - use std::iter::FromIterator; - - #[test] - fn basics() { - let data = vec![Some(b"hello".to_vec()), None, Some(b"hello2".to_vec())]; - - let array = BinaryArray::::from_iter(data); - - assert_eq!(array.value(0), b"hello"); - assert_eq!(array.value(1), b""); - assert_eq!(array.value(2), b"hello2"); - assert_eq!(unsafe { array.value_unchecked(2) }, b"hello2"); - assert_eq!(array.values().as_slice(), b"hellohello2"); - assert_eq!(array.offsets().as_slice(), &[0, 5, 5, 11]); - assert_eq!( - array.validity(), - &Some(Bitmap::from_u8_slice(&[0b00000101], 3)) - ); - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(array.is_valid(2)); - - let array2 = BinaryArray::::from_data( - array.offsets().clone(), - array.values().clone(), - array.validity().clone(), - ); - assert_eq!(array, array2); - - let array = array.slice(1, 2); - assert_eq!(array.value(0), b""); - assert_eq!(array.value(1), b"hello2"); - // note how this keeps everything: the offsets were sliced - assert_eq!(array.values().as_slice(), b"hellohello2"); - assert_eq!(array.offsets().as_slice(), &[5, 5, 11]); - } - - #[test] - fn empty() { - let array = BinaryArray::::new_empty(); - assert_eq!(array.values().as_slice(), b""); - assert_eq!(array.offsets().as_slice(), &[0]); - assert_eq!(array.validity(), &None); - } -} diff --git a/src/array/boolean/from.rs b/src/array/boolean/from.rs index 4cd4d3a5a4f..5134dec092b 100644 --- a/src/array/boolean/from.rs +++ b/src/array/boolean/from.rs @@ -71,26 +71,3 @@ impl>> FromIterator for BooleanArray MutableBooleanArray::from_iter(iter).into() } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::array::Array; - use crate::error::Result; - - #[test] - fn from_trusted_len_iter() -> Result<()> { - let iter = std::iter::repeat(true).take(2).map(Some); - let a = BooleanArray::from_trusted_len_iter(iter); - assert_eq!(a.len(), 2); - Ok(()) - } - - #[test] - fn from_iter() -> Result<()> { - let iter = std::iter::repeat(true).take(2).map(Some); - let a = BooleanArray::from_iter(iter); - assert_eq!(a.len(), 2); - Ok(()) - } -} diff --git a/src/array/boolean/mod.rs b/src/array/boolean/mod.rs index af5087ee93f..212bc01163c 100644 --- a/src/array/boolean/mod.rs +++ b/src/array/boolean/mod.rs @@ -130,41 +130,3 @@ impl]>> From

for BooleanArray { Self::from_trusted_len_iter(slice.as_ref().iter().map(|x| x.as_ref())) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn basics() { - let data = vec![Some(true), None, Some(false)]; - - let array: BooleanArray = data.into_iter().collect(); - - assert!(array.value(0)); - assert!(!array.value(1)); - assert!(!array.value(2)); - assert_eq!(array.values(), &Bitmap::from_u8_slice(&[0b00000001], 3)); - assert_eq!( - array.validity(), - &Some(Bitmap::from_u8_slice(&[0b00000101], 3)) - ); - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(array.is_valid(2)); - - let array2 = BooleanArray::from_data(array.values().clone(), array.validity().clone()); - assert_eq!(array, array2); - - let array = array.slice(1, 2); - assert!(!array.value(0)); - assert!(!array.value(1)); - } - - #[test] - fn empty() { - let array = BooleanArray::new_empty(); - assert_eq!(array.values().len(), 0); - assert_eq!(array.validity(), &None); - } -} diff --git a/src/array/boolean/mutable.rs b/src/array/boolean/mutable.rs index 2f376329fa5..44231cc89a2 100644 --- a/src/array/boolean/mutable.rs +++ b/src/array/boolean/mutable.rs @@ -360,34 +360,3 @@ impl TryExtend> for MutableBooleanArray { Ok(()) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn push() { - let mut a = MutableBooleanArray::new(); - a.push(Some(true)); - a.push(None); - assert_eq!(a.len(), 2); - assert!(a.is_valid(0)); - assert!(!a.is_valid(1)); - - assert_eq!(a.values(), &MutableBitmap::from([true, false])); - } - - #[test] - fn from_trusted_len_iter() { - let iter = std::iter::repeat(true).take(2).map(Some); - let a = MutableBooleanArray::from_trusted_len_iter(iter); - assert_eq!(a.len(), 2); - } - - #[test] - fn from_iter() { - let iter = std::iter::repeat(true).take(2).map(Some); - let a = MutableBooleanArray::from_iter(iter); - assert_eq!(a.len(), 2); - } -} diff --git a/src/array/dictionary/mutable.rs b/src/array/dictionary/mutable.rs index 9249c789fad..07dfd3b13fd 100644 --- a/src/array/dictionary/mutable.rs +++ b/src/array/dictionary/mutable.rs @@ -152,47 +152,3 @@ where Ok(()) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::array::{MutableBinaryArray, MutableUtf8Array}; - - #[test] - fn primitive() -> Result<()> { - let data = vec![Some(1), Some(2), Some(1)]; - - let mut a = MutableDictionaryArray::>::new(); - a.try_extend(data)?; - assert_eq!(a.len(), 3); - assert_eq!(a.values().len(), 2); - Ok(()) - } - - #[test] - fn utf8_natural() -> Result<()> { - let data = vec![Some("a"), Some("b"), Some("a")]; - - let mut a = MutableDictionaryArray::>::new(); - a.try_extend(data)?; - - assert_eq!(a.len(), 3); - assert_eq!(a.values().len(), 2); - Ok(()) - } - - #[test] - fn binary_natural() -> Result<()> { - let data = vec![ - Some("a".as_bytes()), - Some("b".as_bytes()), - Some("a".as_bytes()), - ]; - - let mut a = MutableDictionaryArray::>::new(); - a.try_extend(data)?; - assert_eq!(a.len(), 3); - assert_eq!(a.values().len(), 2); - Ok(()) - } -} diff --git a/src/array/equal/fixed_size_list.rs b/src/array/equal/fixed_size_list.rs index ad1f50e83f3..aaf77910013 100644 --- a/src/array/equal/fixed_size_list.rs +++ b/src/array/equal/fixed_size_list.rs @@ -3,123 +3,3 @@ use crate::array::{Array, FixedSizeListArray}; pub(super) fn equal(lhs: &FixedSizeListArray, rhs: &FixedSizeListArray) -> bool { lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) } - -#[cfg(test)] -mod tests { - use crate::{ - array::{ - equal::tests::test_equal, fixed_size_list::MutableFixedSizeListArray, - MutablePrimitiveArray, - }, - datatypes::DataType, - }; - - use super::*; - - /// Create a fixed size list of 2 value lengths - fn create_fixed_size_list_array, T: AsRef<[Option]>>( - data: T, - ) -> FixedSizeListArray { - let data = data.as_ref().iter().map(|x| { - Some(match x { - Some(x) => x.as_ref().iter().map(|x| Some(*x)).collect::>(), - None => std::iter::repeat(None).take(3).collect::>(), - }) - }); - - MutableFixedSizeListArray::>::try_from_iter( - data, - 3, - DataType::Int32, - ) - .unwrap() - .into() - } - - #[test] - fn test_fixed_size_list_equal() { - let a = create_fixed_size_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 6])]); - let b = create_fixed_size_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 6])]); - test_equal(&a, &b, true); - - let b = create_fixed_size_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 7])]); - test_equal(&a, &b, false); - } - - // Test the case where null_count > 0 - #[test] - fn test_fixed_list_null() { - let a = create_fixed_size_list_array(&[ - Some(&[1, 2, 3]), - None, - None, - Some(&[4, 5, 6]), - None, - None, - ]); - /* - let b = create_fixed_size_list_array(&[ - Some(&[1, 2, 3]), - None, - None, - Some(&[4, 5, 6]), - None, - None, - ]); - test_equal(&a, &b, true); - - let b = create_fixed_size_list_array(&[ - Some(&[1, 2, 3]), - None, - Some(&[7, 8, 9]), - Some(&[4, 5, 6]), - None, - None, - ]); - test_equal(&a, &b, false); - */ - - let b = create_fixed_size_list_array(&[ - Some(&[1, 2, 3]), - None, - None, - Some(&[3, 6, 9]), - None, - None, - ]); - test_equal(&a, &b, false); - } - - #[test] - fn test_fixed_list_offsets() { - // Test the case where offset != 0 - let a = create_fixed_size_list_array(&[ - Some(&[1, 2, 3]), - None, - None, - Some(&[4, 5, 6]), - None, - None, - ]); - let b = create_fixed_size_list_array(&[ - Some(&[1, 2, 3]), - None, - None, - Some(&[3, 6, 9]), - None, - None, - ]); - - let a_slice = a.slice(0, 3); - let b_slice = b.slice(0, 3); - test_equal(&a_slice, &b_slice, true); - - let a_slice = a.slice(0, 5); - let b_slice = b.slice(0, 5); - test_equal(&a_slice, &b_slice, false); - - let a_slice = a.slice(4, 1); - let b_slice = b.slice(4, 1); - test_equal(&a_slice, &b_slice, true); - } -} diff --git a/src/array/equal/list.rs b/src/array/equal/list.rs index 12d223f194a..4eee0b821e2 100644 --- a/src/array/equal/list.rs +++ b/src/array/equal/list.rs @@ -3,101 +3,3 @@ use crate::array::{Array, ListArray, Offset}; pub(super) fn equal(lhs: &ListArray, rhs: &ListArray) -> bool { lhs.data_type() == rhs.data_type() && lhs.len() == rhs.len() && lhs.iter().eq(rhs.iter()) } - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use crate::array::equal::tests::test_equal; - use crate::array::{Int32Array, MutableListArray, MutablePrimitiveArray, TryExtend}; - use crate::bitmap::Bitmap; - use crate::buffer::Buffer; - use crate::datatypes::DataType; - - use super::*; - - fn create_list_array, T: AsRef<[Option]>>(data: T) -> ListArray { - let iter = data.as_ref().iter().map(|x| { - x.as_ref() - .map(|x| x.as_ref().iter().map(|x| Some(*x)).collect::>()) - }); - let mut array = MutableListArray::>::new(); - array.try_extend(iter).unwrap(); - array.into() - } - - #[test] - fn test_list_equal() { - let a = create_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 6])]); - let b = create_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 6])]); - test_equal(&a, &b, true); - - let b = create_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 7])]); - test_equal(&a, &b, false); - } - - // Test the case where null_count > 0 - #[test] - fn test_list_null() { - let a = create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 4]), None, None]); - let b = create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 4]), None, None]); - test_equal(&a, &b, true); - - let b = create_list_array(&[ - Some(&[1, 2]), - None, - Some(&[5, 6]), - Some(&[3, 4]), - None, - None, - ]); - test_equal(&a, &b, false); - - let b = create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 5]), None, None]); - test_equal(&a, &b, false); - } - - // Test the case where offset != 0 - #[test] - fn test_list_offsets() { - let a = create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 4]), None, None]); - let b = create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 5]), None, None]); - - let a_slice = a.slice(0, 3); - let b_slice = b.slice(0, 3); - test_equal(&a_slice, &b_slice, true); - - let a_slice = a.slice(0, 5); - let b_slice = b.slice(0, 5); - test_equal(&a_slice, &b_slice, false); - - let a_slice = a.slice(4, 1); - let b_slice = b.slice(4, 1); - test_equal(&a_slice, &b_slice, true); - } - - #[test] - fn test_bla() { - let offsets = Buffer::from([0, 3, 3, 6]); - let data_type = ListArray::::default_datatype(DataType::Int32); - let values = Arc::new(Int32Array::from([ - Some(1), - Some(2), - Some(3), - Some(4), - None, - Some(6), - ])); - let validity = Bitmap::from([true, false, true]); - let lhs = ListArray::::from_data(data_type, offsets, values, Some(validity)); - let lhs = lhs.slice(1, 2); - - let offsets = Buffer::from([0, 0, 3]); - let data_type = ListArray::::default_datatype(DataType::Int32); - let values = Arc::new(Int32Array::from([Some(4), None, Some(6)])); - let validity = Bitmap::from([false, true]); - let rhs = ListArray::::from_data(data_type, offsets, values, Some(validity)); - - assert_eq!(lhs, rhs); - } -} diff --git a/src/array/equal/mod.rs b/src/array/equal/mod.rs index be2b1c1976c..64b09b37d55 100644 --- a/src/array/equal/mod.rs +++ b/src/array/equal/mod.rs @@ -338,267 +338,3 @@ pub fn equal(lhs: &dyn Array, rhs: &dyn Array) -> bool { } } } - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use crate::array::{BooleanArray, Int16Array, Int32Array, Offset}; - - use super::*; - - #[test] - fn test_primitive() { - let cases = vec![ - ( - vec![Some(1), Some(2), Some(3)], - vec![Some(1), Some(2), Some(3)], - true, - ), - ( - vec![Some(1), Some(2), Some(3)], - vec![Some(1), Some(2), Some(4)], - false, - ), - ( - vec![Some(1), Some(2), None], - vec![Some(1), Some(2), None], - true, - ), - ( - vec![Some(1), None, Some(3)], - vec![Some(1), Some(2), None], - false, - ), - ( - vec![Some(1), None, None], - vec![Some(1), Some(2), None], - false, - ), - ]; - - for (lhs, rhs, expected) in cases { - let lhs = Int32Array::from(&lhs); - let rhs = Int32Array::from(&rhs); - test_equal(&lhs, &rhs, expected); - } - } - - #[test] - fn test_primitive_slice() { - let cases = vec![ - ( - vec![Some(1), Some(2), Some(3)], - (0, 1), - vec![Some(1), Some(2), Some(3)], - (0, 1), - true, - ), - ( - vec![Some(1), Some(2), Some(3)], - (1, 1), - vec![Some(1), Some(2), Some(3)], - (2, 1), - false, - ), - ( - vec![Some(1), Some(2), None], - (1, 1), - vec![Some(1), None, Some(2)], - (2, 1), - true, - ), - ( - vec![None, Some(2), None], - (1, 1), - vec![None, None, Some(2)], - (2, 1), - true, - ), - ( - vec![Some(1), None, Some(2), None, Some(3)], - (2, 2), - vec![None, Some(2), None, Some(3)], - (1, 2), - true, - ), - ]; - - for (lhs, slice_lhs, rhs, slice_rhs, expected) in cases { - let lhs = Int32Array::from(&lhs); - let lhs = lhs.slice(slice_lhs.0, slice_lhs.1); - let rhs = Int32Array::from(&rhs); - let rhs = rhs.slice(slice_rhs.0, slice_rhs.1); - - test_equal(&lhs, &rhs, expected); - } - } - - pub(super) fn test_equal(lhs: &dyn Array, rhs: &dyn Array, expected: bool) { - // equality is symmetric - assert!(equal(lhs, lhs), "\n{:?}\n{:?}", lhs, lhs); - assert!(equal(rhs, rhs), "\n{:?}\n{:?}", rhs, rhs); - - assert_eq!(equal(lhs, rhs), expected, "\n{:?}\n{:?}", lhs, rhs); - assert_eq!(equal(rhs, lhs), expected, "\n{:?}\n{:?}", rhs, lhs); - } - - #[test] - fn test_boolean_equal() { - let a = BooleanArray::from_slice([false, false, true]); - let b = BooleanArray::from_slice([false, false, true]); - test_equal(&a, &b, true); - - let b = BooleanArray::from_slice([false, false, false]); - test_equal(&a, &b, false); - } - - #[test] - fn test_boolean_equal_null() { - let a = BooleanArray::from(vec![Some(false), None, None, Some(true)]); - let b = BooleanArray::from(vec![Some(false), None, None, Some(true)]); - test_equal(&a, &b, true); - - let b = BooleanArray::from(vec![None, None, None, Some(true)]); - test_equal(&a, &b, false); - - let b = BooleanArray::from(vec![Some(true), None, None, Some(true)]); - test_equal(&a, &b, false); - } - - #[test] - fn test_boolean_equal_offset() { - let a = BooleanArray::from_slice(vec![false, true, false, true, false, false, true]); - let b = BooleanArray::from_slice(vec![true, false, false, false, true, false, true, true]); - test_equal(&a, &b, false); - - let a_slice = a.slice(2, 3); - let b_slice = b.slice(3, 3); - test_equal(&a_slice, &b_slice, true); - - let a_slice = a.slice(3, 4); - let b_slice = b.slice(4, 4); - test_equal(&a_slice, &b_slice, false); - - // Elements fill in `u8`'s exactly. - let mut vector = vec![false, false, true, true, true, true, true, true]; - let a = BooleanArray::from_slice(vector.clone()); - let b = BooleanArray::from_slice(vector.clone()); - test_equal(&a, &b, true); - - // Elements fill in `u8`s + suffix bits. - vector.push(true); - let a = BooleanArray::from_slice(vector.clone()); - let b = BooleanArray::from_slice(vector); - test_equal(&a, &b, true); - } - - #[allow(clippy::type_complexity)] - fn binary_cases() -> Vec<(Vec>, Vec>, bool)> { - let base = vec![ - Some("hello".to_owned()), - None, - None, - Some("world".to_owned()), - None, - None, - ]; - let not_base = vec![ - Some("hello".to_owned()), - Some("foo".to_owned()), - None, - Some("world".to_owned()), - None, - None, - ]; - vec![ - ( - vec![Some("hello".to_owned()), Some("world".to_owned())], - vec![Some("hello".to_owned()), Some("world".to_owned())], - true, - ), - ( - vec![Some("hello".to_owned()), Some("world".to_owned())], - vec![Some("hello".to_owned()), Some("arrow".to_owned())], - false, - ), - (base.clone(), base.clone(), true), - (base, not_base, false), - ] - } - - fn test_generic_string_equal() { - let cases = binary_cases(); - - for (lhs, rhs, expected) in cases { - let lhs = lhs.iter().map(|x| x.as_deref()).collect::>(); - let rhs = rhs.iter().map(|x| x.as_deref()).collect::>(); - let lhs = Utf8Array::::from(&lhs); - let rhs = Utf8Array::::from(&rhs); - test_equal(&lhs, &rhs, expected); - } - } - - #[test] - fn test_string_equal() { - test_generic_string_equal::() - } - - #[test] - fn test_large_string_equal() { - test_generic_string_equal::() - } - - fn create_dictionary_array(values: &[&str], keys: &[Option]) -> DictionaryArray { - let keys = Int16Array::from(keys); - let values = Utf8Array::::from_slice(values); - - DictionaryArray::from_data(keys, Arc::new(values)) - } - - #[test] - fn test_dictionary_equal() { - // (a, b, c), (0, 1, 0, 2) => (a, b, a, c) - let a = create_dictionary_array(&["a", "b", "c"], &[Some(0), Some(1), Some(0), Some(2)]); - // different representation (values and keys are swapped), same result - let b = create_dictionary_array(&["a", "c", "b"], &[Some(0), Some(2), Some(0), Some(1)]); - test_equal(&a, &b, true); - - // different len - let b = create_dictionary_array(&["a", "c", "b"], &[Some(0), Some(2), Some(1)]); - test_equal(&a, &b, false); - - // different key - let b = create_dictionary_array(&["a", "c", "b"], &[Some(0), Some(2), Some(0), Some(0)]); - test_equal(&a, &b, false); - - // different values, same keys - let b = create_dictionary_array(&["a", "b", "d"], &[Some(0), Some(1), Some(0), Some(2)]); - test_equal(&a, &b, false); - } - - #[test] - fn test_dictionary_equal_null() { - // (a, b, c), (1, 2, 1, 3) => (a, b, a, c) - let a = create_dictionary_array(&["a", "b", "c"], &[Some(0), None, Some(0), Some(2)]); - - // equal to self - test_equal(&a, &a, true); - - // different representation (values and keys are swapped), same result - let b = create_dictionary_array(&["a", "c", "b"], &[Some(0), None, Some(0), Some(1)]); - test_equal(&a, &b, true); - - // different null position - let b = create_dictionary_array(&["a", "c", "b"], &[Some(0), Some(2), Some(0), None]); - test_equal(&a, &b, false); - - // different key - let b = create_dictionary_array(&["a", "c", "b"], &[Some(0), None, Some(0), Some(0)]); - test_equal(&a, &b, false); - - // different values, same keys - let b = create_dictionary_array(&["a", "b", "d"], &[Some(0), None, Some(0), Some(2)]); - test_equal(&a, &b, false); - } -} diff --git a/src/array/fixed_size_binary/mutable.rs b/src/array/fixed_size_binary/mutable.rs index 1dba3c6ac4a..395ecfca3cd 100644 --- a/src/array/fixed_size_binary/mutable.rs +++ b/src/array/fixed_size_binary/mutable.rs @@ -130,29 +130,3 @@ impl MutableArray for MutableFixedSizeBinaryArray { self.values.extend_constant(self.size, 0); } } - -#[cfg(test)] -mod tests { - use crate::bitmap::Bitmap; - - use super::*; - - #[test] - fn basic() { - let array = MutableFixedSizeBinaryArray::try_from_iter( - vec![Some(b"ab"), Some(b"bc"), None, Some(b"fh")], - 2, - ) - .unwrap(); - assert_eq!(array.len(), 4); - } - - #[test] - fn push_null() { - let mut array = MutableFixedSizeBinaryArray::new(2); - array.push::<&[u8]>(None); - - let array: FixedSizeBinaryArray = array.into(); - assert_eq!(array.validity(), &Some(Bitmap::from([false]))); - } -} diff --git a/src/array/fixed_size_list/mutable.rs b/src/array/fixed_size_list/mutable.rs index 8d24bd54f4a..73a9ff21ed8 100644 --- a/src/array/fixed_size_list/mutable.rs +++ b/src/array/fixed_size_list/mutable.rs @@ -167,40 +167,3 @@ macro_rules! impl_offsets { impl_offsets!(MutableUtf8Array, str); impl_offsets!(MutableBinaryArray, [u8]); - -#[cfg(test)] -mod tests { - use crate::array::Int32Array; - - use super::*; - - #[test] - fn primitive() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - Some(vec![None, None, None]), - Some(vec![Some(4), None, Some(6)]), - ]; - - let list: FixedSizeListArray = - MutableFixedSizeListArray::>::try_from_iter( - data, - 3, - DataType::Int32, - ) - .unwrap() - .into(); - - let a = list.value(0); - let a = a.as_any().downcast_ref::().unwrap(); - - let expected = Int32Array::from(vec![Some(1i32), Some(2), Some(3)]); - assert_eq!(a, &expected); - - let a = list.value(1); - let a = a.as_any().downcast_ref::().unwrap(); - - let expected = Int32Array::from(vec![None, None, None]); - assert_eq!(a, &expected) - } -} diff --git a/src/array/growable/binary.rs b/src/array/growable/binary.rs index 5ae12c257f2..f12a3a2a78d 100644 --- a/src/array/growable/binary.rs +++ b/src/array/growable/binary.rs @@ -97,90 +97,3 @@ impl<'a, O: Offset> From> for BinaryArray { BinaryArray::::from_data(val.offsets.into(), val.values.into(), val.validity.into()) } } - -#[cfg(test)] -mod tests { - use std::iter::FromIterator; - - use super::*; - - /// tests extending from a variable-sized (strings and binary) array w/ offset with nulls - #[test] - fn test_variable_sized_validity() { - let array = BinaryArray::::from_iter(vec![Some("a"), Some("bc"), None, Some("defh")]); - - let mut a = GrowableBinary::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - - let result: BinaryArray = a.into(); - - let expected = BinaryArray::::from_iter(vec![Some("bc"), None]); - assert_eq!(result, expected); - } - - /// tests extending from a variable-sized (strings and binary) array - /// with an offset and nulls - #[test] - fn test_variable_sized_offsets() { - let array = BinaryArray::::from_iter(vec![Some("a"), Some("bc"), None, Some("defh")]); - let array = array.slice(1, 3); - - let mut a = GrowableBinary::new(vec![&array], false, 0); - - a.extend(0, 0, 3); - - let result: BinaryArray = a.into(); - - let expected = BinaryArray::::from_iter(vec![Some("bc"), None, Some("defh")]); - assert_eq!(result, expected); - } - - #[test] - fn test_string_offsets() { - let array = BinaryArray::::from_iter(vec![Some("a"), Some("bc"), None, Some("defh")]); - let array = array.slice(1, 3); - - let mut a = GrowableBinary::new(vec![&array], false, 0); - - a.extend(0, 0, 3); - - let result: BinaryArray = a.into(); - - let expected = BinaryArray::::from_iter(vec![Some("bc"), None, Some("defh")]); - assert_eq!(result, expected); - } - - #[test] - fn test_multiple_with_validity() { - let array1 = BinaryArray::::from_slice(vec![b"hello", b"world"]); - let array2 = BinaryArray::::from_iter(vec![Some("1"), None]); - - let mut a = GrowableBinary::new(vec![&array1, &array2], false, 5); - - a.extend(0, 0, 2); - a.extend(1, 0, 2); - - let result: BinaryArray = a.into(); - - let expected = - BinaryArray::::from_iter(vec![Some("hello"), Some("world"), Some("1"), None]); - assert_eq!(result, expected); - } - - #[test] - fn test_string_null_offset_validity() { - let array = BinaryArray::::from_iter(vec![Some("a"), Some("bc"), None, Some("defh")]); - let array = array.slice(1, 3); - - let mut a = GrowableBinary::new(vec![&array], true, 0); - - a.extend(0, 1, 2); - a.extend_validity(1); - - let result: BinaryArray = a.into(); - - let expected = BinaryArray::::from_iter(vec![None, Some("defh"), None]); - assert_eq!(result, expected); - } -} diff --git a/src/array/growable/boolean.rs b/src/array/growable/boolean.rs index 24bbf8267af..4f02f82e67b 100644 --- a/src/array/growable/boolean.rs +++ b/src/array/growable/boolean.rs @@ -78,22 +78,3 @@ impl<'a> From> for BooleanArray { BooleanArray::from_data(val.values.into(), val.validity.into()) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_bool() { - let array = BooleanArray::from(vec![Some(false), Some(true), None, Some(false)]); - - let mut a = GrowableBoolean::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - - let result: BooleanArray = a.into(); - - let expected = BooleanArray::from(vec![Some(true), None]); - assert_eq!(result, expected); - } -} diff --git a/src/array/growable/dictionary.rs b/src/array/growable/dictionary.rs index 94667b87852..ac9e98f5a7a 100644 --- a/src/array/growable/dictionary.rs +++ b/src/array/growable/dictionary.rs @@ -132,71 +132,3 @@ impl<'a, T: DictionaryKey> From> for DictionaryArray::from_data(keys, val.values) } } - -#[cfg(test)] -mod tests { - use super::*; - - use crate::array::*; - use crate::datatypes::DataType; - use crate::error::Result; - - #[test] - fn test_single() -> Result<()> { - let original_data = vec![Some("a"), Some("b"), Some("a")]; - - let data = original_data.clone(); - let mut array = MutableDictionaryArray::>::new(); - array.try_extend(data)?; - let array = array.into(); - - // same values, less keys - let expected = DictionaryArray::::from_data( - PrimitiveArray::from(vec![Some(1), Some(0)]).to(DataType::Int32), - Arc::new(Utf8Array::::from(&original_data)), - ); - - let mut growable = GrowableDictionary::new(&[&array], false, 0); - - growable.extend(0, 1, 2); - - let result: DictionaryArray = growable.into(); - - assert_eq!(result, expected); - Ok(()) - } - - #[test] - fn test_multi() -> Result<()> { - let mut original_data1 = vec![Some("a"), Some("b"), None, Some("a")]; - let original_data2 = vec![Some("c"), Some("b"), None, Some("a")]; - - let data1 = original_data1.clone(); - let data2 = original_data2.clone(); - - let mut array1 = MutableDictionaryArray::>::new(); - array1.try_extend(data1)?; - let array1: DictionaryArray = array1.into(); - - let mut array2 = MutableDictionaryArray::>::new(); - array2.try_extend(data2)?; - let array2: DictionaryArray = array2.into(); - - // same values, less keys - original_data1.extend(original_data2.iter().cloned()); - let expected = DictionaryArray::::from_data( - PrimitiveArray::from(vec![Some(1), None, Some(3), None]).to(DataType::Int32), - Arc::new(Utf8Array::::from_slice(&["a", "b", "c", "b", "a"])), - ); - - let mut growable = GrowableDictionary::new(&[&array1, &array2], false, 0); - - growable.extend(0, 1, 2); - growable.extend(1, 1, 2); - - let result: DictionaryArray = growable.into(); - - assert_eq!(result, expected); - Ok(()) - } -} diff --git a/src/array/growable/fixed_binary.rs b/src/array/growable/fixed_binary.rs index 1c15d37ddcd..945786968b8 100644 --- a/src/array/growable/fixed_binary.rs +++ b/src/array/growable/fixed_binary.rs @@ -96,96 +96,3 @@ impl<'a> From> for FixedSizeBinaryArray { ) } } - -#[cfg(test)] -mod tests { - use super::*; - - /// tests extending from a variable-sized (strings and binary) array w/ offset with nulls - #[test] - fn basic() { - let array = - FixedSizeBinaryArray::from_iter(vec![Some(b"ab"), Some(b"bc"), None, Some(b"de")], 2); - - let mut a = GrowableFixedSizeBinary::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - - let result: FixedSizeBinaryArray = a.into(); - - let expected = FixedSizeBinaryArray::from_iter(vec![Some("bc"), None], 2); - assert_eq!(result, expected); - } - - /// tests extending from a variable-sized (strings and binary) array - /// with an offset and nulls - #[test] - fn offsets() { - let array = - FixedSizeBinaryArray::from_iter(vec![Some(b"ab"), Some(b"bc"), None, Some(b"fh")], 2); - let array = array.slice(1, 3); - - let mut a = GrowableFixedSizeBinary::new(vec![&array], false, 0); - - a.extend(0, 0, 3); - - let result: FixedSizeBinaryArray = a.into(); - - let expected = FixedSizeBinaryArray::from_iter(vec![Some(b"bc"), None, Some(b"fh")], 2); - assert_eq!(result, expected); - } - - #[test] - fn multiple_with_validity() { - let array1 = FixedSizeBinaryArray::from_iter(vec![Some("hello"), Some("world")], 5); - let array2 = FixedSizeBinaryArray::from_iter(vec![Some("12345"), None], 5); - - let mut a = GrowableFixedSizeBinary::new(vec![&array1, &array2], false, 5); - - a.extend(0, 0, 2); - a.extend(1, 0, 2); - - let result: FixedSizeBinaryArray = a.into(); - - let expected = FixedSizeBinaryArray::from_iter( - vec![Some("hello"), Some("world"), Some("12345"), None], - 5, - ); - assert_eq!(result, expected); - } - - #[test] - fn null_offset_validity() { - let array = - FixedSizeBinaryArray::from_iter(vec![Some("aa"), Some("bc"), None, Some("fh")], 2); - let array = array.slice(1, 3); - - let mut a = GrowableFixedSizeBinary::new(vec![&array], true, 0); - - a.extend(0, 1, 2); - a.extend_validity(1); - - let result: FixedSizeBinaryArray = a.into(); - - let expected = FixedSizeBinaryArray::from_iter(vec![None, Some("fh"), None], 2); - assert_eq!(result, expected); - } - - #[test] - fn sized_offsets() { - let array = - FixedSizeBinaryArray::from_iter(vec![Some(&[0, 0]), Some(&[0, 1]), Some(&[0, 2])], 2); - let array = array.slice(1, 2); - // = [[0, 1], [0, 2]] due to the offset = 1 - - let mut a = GrowableFixedSizeBinary::new(vec![&array], false, 0); - - a.extend(0, 1, 1); - a.extend(0, 0, 1); - - let result: FixedSizeBinaryArray = a.into(); - - let expected = FixedSizeBinaryArray::from_iter(vec![Some(&[0, 2]), Some(&[0, 1])], 2); - assert_eq!(result, expected); - } -} diff --git a/src/array/growable/list.rs b/src/array/growable/list.rs index 3c360c5cc9b..94a623c6cf9 100644 --- a/src/array/growable/list.rs +++ b/src/array/growable/list.rs @@ -146,111 +146,3 @@ impl<'a, O: Offset> From> for ListArray { ) } } - -#[cfg(test)] -mod tests { - use super::*; - - use crate::array::{ListArray, MutableListArray, MutablePrimitiveArray, TryExtend}; - - fn create_list_array(data: Vec>>>) -> ListArray { - let mut array = MutableListArray::>::new(); - array.try_extend(data).unwrap(); - array.into() - } - - #[test] - fn basic() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - Some(vec![Some(4), Some(5)]), - Some(vec![Some(6i32), Some(7), Some(8)]), - ]; - - let array = create_list_array(data); - - let mut a = GrowableList::new(vec![&array], false, 0); - a.extend(0, 0, 1); - - let result: ListArray = a.into(); - - let expected = vec![Some(vec![Some(1i32), Some(2), Some(3)])]; - let expected = create_list_array(expected); - - assert_eq!(result, expected) - } - - #[test] - fn null_offset() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(6i32), Some(7), Some(8)]), - ]; - let array = create_list_array(data); - let array = array.slice(1, 2); - - let mut a = GrowableList::new(vec![&array], false, 0); - a.extend(0, 1, 1); - - let result: ListArray = a.into(); - - let expected = vec![Some(vec![Some(6i32), Some(7), Some(8)])]; - let expected = create_list_array(expected); - - assert_eq!(result, expected) - } - - #[test] - fn null_offsets() { - let data = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(6i32), None, Some(8)]), - ]; - let array = create_list_array(data); - let array = array.slice(1, 2); - - let mut a = GrowableList::new(vec![&array], false, 0); - a.extend(0, 1, 1); - - let result: ListArray = a.into(); - - let expected = vec![Some(vec![Some(6i32), None, Some(8)])]; - let expected = create_list_array(expected); - - assert_eq!(result, expected) - } - - #[test] - fn test_from_two_lists() { - let data_1 = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(6i32), None, Some(8)]), - ]; - let array_1 = create_list_array(data_1); - - let data_2 = vec![ - Some(vec![Some(8i32), Some(7), Some(6)]), - Some(vec![Some(5i32), None, Some(4)]), - Some(vec![Some(2i32), Some(1), Some(0)]), - ]; - let array_2 = create_list_array(data_2); - - let mut a = GrowableList::new(vec![&array_1, &array_2], false, 6); - a.extend(0, 0, 2); - a.extend(1, 1, 1); - - let result: ListArray = a.into(); - - let expected = vec![ - Some(vec![Some(1i32), Some(2), Some(3)]), - None, - Some(vec![Some(5i32), None, Some(4)]), - ]; - let expected = create_list_array(expected); - - assert_eq!(result, expected); - } -} diff --git a/src/array/growable/mod.rs b/src/array/growable/mod.rs index dc44a3ef3e5..41e3ca3328a 100644 --- a/src/array/growable/mod.rs +++ b/src/array/growable/mod.rs @@ -239,132 +239,3 @@ pub fn make_growable<'a>( }, } } - -/* -#[cfg(test)] -mod tests { - use std::convert::TryFrom; - - use super::*; - - use crate::{ - array::{ - Array, ArrayDataRef, ArrayRef, BooleanArray, DictionaryArray, - FixedSizeBinaryArray, Int16Array, Int16Type, Int32Array, Int64Array, - Int64Builder, ListBuilder, NullArray, PrimitiveBuilder, StringArray, - StringDictionaryBuilder, StructArray, UInt8Array, - }, - buffer::Buffer, - datatypes::Field, - }; - use crate::{ - array::{ListArray, StringBuilder}, - error::Result, - }; - - fn create_dictionary_array(values: &[&str], keys: &[Option<&str>]) -> ArrayDataRef { - let values = StringArray::from(values.to_vec()); - let mut builder = StringDictionaryBuilder::new_with_dictionary( - PrimitiveBuilder::::new(3), - &values, - ) - .unwrap(); - for key in keys { - if let Some(v) = key { - builder.append(v).unwrap(); - } else { - builder.append_null().unwrap() - } - } - builder.finish().data() - } - - /* - // this is an old test used on a meanwhile removed dead code - // that is still useful when `MutableArrayData` supports fixed-size lists. - #[test] - fn test_fixed_size_list_append() -> Result<()> { - let int_builder = UInt16Builder::new(64); - let mut builder = FixedSizeListBuilder::::new(int_builder, 2); - builder.values().append_slice(&[1, 2])?; - builder.append(true)?; - builder.values().append_slice(&[3, 4])?; - builder.append(false)?; - builder.values().append_slice(&[5, 6])?; - builder.append(true)?; - - let a_builder = UInt16Builder::new(64); - let mut a_builder = FixedSizeListBuilder::::new(a_builder, 2); - a_builder.values().append_slice(&[7, 8])?; - a_builder.append(true)?; - a_builder.values().append_slice(&[9, 10])?; - a_builder.append(true)?; - a_builder.values().append_slice(&[11, 12])?; - a_builder.append(false)?; - a_builder.values().append_slice(&[13, 14])?; - a_builder.append(true)?; - a_builder.values().append_null()?; - a_builder.values().append_null()?; - a_builder.append(true)?; - let a = a_builder.finish(); - - // append array - builder.append_data(&[ - a.data(), - a.slice(1, 3).data(), - a.slice(2, 1).data(), - a.slice(5, 0).data(), - ])?; - let finished = builder.finish(); - - let expected_int_array = UInt16Array::from(vec![ - Some(1), - Some(2), - Some(3), - Some(4), - Some(5), - Some(6), - // append first array - Some(7), - Some(8), - Some(9), - Some(10), - Some(11), - Some(12), - Some(13), - Some(14), - None, - None, - // append slice(1, 3) - Some(9), - Some(10), - Some(11), - Some(12), - Some(13), - Some(14), - // append slice(2, 1) - Some(11), - Some(12), - ]); - let expected_list_data = ArrayData::new( - DataType::FixedSizeList( - Box::new(Field::new("item", DataType::UInt16, true)), - 2, - ), - 12, - None, - None, - 0, - vec![], - vec![expected_int_array.data()], - ); - let expected_list = - FixedSizeListArray::from(Arc::new(expected_list_data) as ArrayDataRef); - assert_eq!(&expected_list.values(), &finished.values()); - assert_eq!(expected_list.len(), finished.len()); - - Ok(()) - } - */ -} -*/ diff --git a/src/array/growable/null.rs b/src/array/growable/null.rs index 6df6123b85d..74c4498a897 100644 --- a/src/array/growable/null.rs +++ b/src/array/growable/null.rs @@ -44,21 +44,3 @@ impl From for NullArray { NullArray::from_data(val.length) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_null() { - let mut mutable = GrowableNull::new(); - - mutable.extend(0, 1, 2); - mutable.extend(1, 0, 1); - - let result: NullArray = mutable.into(); - - let expected = NullArray::from_data(3); - assert_eq!(result, expected); - } -} diff --git a/src/array/growable/primitive.rs b/src/array/growable/primitive.rs index f1e11118208..118bd79e6b3 100644 --- a/src/array/growable/primitive.rs +++ b/src/array/growable/primitive.rs @@ -95,74 +95,3 @@ impl<'a, T: NativeType> From> for PrimitiveArray { PrimitiveArray::::from_data(val.data_type, val.values.into(), val.validity.into()) } } - -#[cfg(test)] -mod tests { - use super::*; - - use crate::array::PrimitiveArray; - use crate::datatypes::DataType; - - /// tests extending from a primitive array w/ offset nor nulls - #[test] - fn test_primitive() { - let b = PrimitiveArray::::from(vec![Some(1), Some(2), Some(3)]).to(DataType::UInt8); - let mut a = GrowablePrimitive::new(vec![&b], false, 3); - a.extend(0, 0, 2); - let result: PrimitiveArray = a.into(); - let expected = PrimitiveArray::::from(vec![Some(1), Some(2)]).to(DataType::UInt8); - assert_eq!(result, expected); - } - - /// tests extending from a primitive array with offset w/ nulls - #[test] - fn test_primitive_offset() { - let b = PrimitiveArray::::from(vec![Some(1), Some(2), Some(3)]).to(DataType::UInt8); - let b = b.slice(1, 2); - let mut a = GrowablePrimitive::new(vec![&b], false, 2); - a.extend(0, 0, 2); - let result: PrimitiveArray = a.into(); - let expected = PrimitiveArray::::from(vec![Some(2), Some(3)]).to(DataType::UInt8); - assert_eq!(result, expected); - } - - /// tests extending from a primitive array with offset and nulls - #[test] - fn test_primitive_null_offset() { - let b = PrimitiveArray::::from(vec![Some(1), None, Some(3)]).to(DataType::UInt8); - let b = b.slice(1, 2); - let mut a = GrowablePrimitive::new(vec![&b], false, 2); - a.extend(0, 0, 2); - let result: PrimitiveArray = a.into(); - let expected = PrimitiveArray::::from(vec![None, Some(3)]).to(DataType::UInt8); - assert_eq!(result, expected); - } - - #[test] - fn test_primitive_null_offset_validity() { - let b = PrimitiveArray::::from(vec![Some(1), Some(2), Some(3)]).to(DataType::UInt8); - let b = b.slice(1, 2); - let mut a = GrowablePrimitive::new(vec![&b], true, 2); - a.extend(0, 0, 2); - a.extend_validity(3); - a.extend(0, 1, 1); - let result: PrimitiveArray = a.into(); - let expected = - PrimitiveArray::::from(vec![Some(2), Some(3), None, None, None, Some(3)]) - .to(DataType::UInt8); - assert_eq!(result, expected); - } - - #[test] - fn test_primitive_joining_arrays() { - let b = PrimitiveArray::::from(vec![Some(1), Some(2), Some(3)]); - let c = PrimitiveArray::::from(vec![Some(4), Some(5), Some(6)]); - let mut a = GrowablePrimitive::new(vec![&b, &c], false, 4); - a.extend(0, 0, 2); - a.extend(1, 1, 2); - let result: PrimitiveArray = a.into(); - - let expected = PrimitiveArray::::from(vec![Some(1), Some(2), Some(5), Some(6)]); - assert_eq!(result, expected); - } -} diff --git a/src/array/growable/structure.rs b/src/array/growable/structure.rs index 0248fe8f328..f0f9053e952 100644 --- a/src/array/growable/structure.rs +++ b/src/array/growable/structure.rs @@ -119,123 +119,3 @@ impl<'a> From> for StructArray { StructArray::from_data(val.arrays[0].fields().to_vec(), values, val.validity.into()) } } - -#[cfg(test)] -mod tests { - use std::iter::FromIterator; - - use crate::array::{PrimitiveArray, Utf8Array}; - use crate::bitmap::Bitmap; - use crate::datatypes::{DataType, Field}; - - use super::*; - - fn some_values() -> (Vec, Vec>) { - let strings: Arc = Arc::new(Utf8Array::::from_iter(vec![ - Some("a"), - Some("aa"), - None, - Some("mark"), - Some("doe"), - ])); - let ints: Arc = Arc::new( - PrimitiveArray::::from(vec![Some(1), Some(2), Some(3), Some(4), Some(5)]) - .to(DataType::Int32), - ); - let fields = vec![ - Field::new("f1", DataType::Utf8, true), - Field::new("f2", DataType::Int32, true), - ]; - (fields, vec![strings, ints]) - } - - #[test] - fn basic() { - let (fields, values) = some_values(); - - let array = StructArray::from_data(fields.clone(), values.clone(), None); - - let mut a = GrowableStruct::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - let result: StructArray = a.into(); - - let expected = StructArray::from_data( - fields, - vec![values[0].slice(1, 2).into(), values[1].slice(1, 2).into()], - None, - ); - assert_eq!(result, expected) - } - - #[test] - fn offset() { - let (fields, values) = some_values(); - - let array = StructArray::from_data(fields.clone(), values.clone(), None).slice(1, 3); - - let mut a = GrowableStruct::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - let result: StructArray = a.into(); - - let expected = StructArray::from_data( - fields, - vec![values[0].slice(2, 2).into(), values[1].slice(2, 2).into()], - None, - ); - - assert_eq!(result, expected); - } - - #[test] - fn nulls() { - let (fields, values) = some_values(); - - let array = StructArray::from_data( - fields.clone(), - values.clone(), - Some(Bitmap::from_u8_slice(&[0b00000010], 5)), - ); - - let mut a = GrowableStruct::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - let result: StructArray = a.into(); - - let expected = StructArray::from_data( - fields, - vec![values[0].slice(1, 2).into(), values[1].slice(1, 2).into()], - Some(Bitmap::from_u8_slice(&[0b00000010], 5).slice(1, 2)), - ); - - assert_eq!(result, expected) - } - - #[test] - fn many() { - let (fields, values) = some_values(); - - let array = StructArray::from_data(fields.clone(), values.clone(), None); - - let mut mutable = GrowableStruct::new(vec![&array, &array], false, 0); - - mutable.extend(0, 1, 2); - mutable.extend(1, 0, 2); - let result: StructArray = mutable.into(); - - let expected_string: Arc = Arc::new(Utf8Array::::from_iter(vec![ - Some("aa"), - None, - Some("a"), - Some("aa"), - ])); - let expected_int: Arc = Arc::new( - PrimitiveArray::::from(vec![Some(2), Some(3), Some(1), Some(2)]) - .to(DataType::Int32), - ); - - let expected = StructArray::from_data(fields, vec![expected_string, expected_int], None); - assert_eq!(result, expected) - } -} diff --git a/src/array/growable/utf8.rs b/src/array/growable/utf8.rs index 26005966519..2d34c64866c 100644 --- a/src/array/growable/utf8.rs +++ b/src/array/growable/utf8.rs @@ -103,92 +103,3 @@ impl<'a, O: Offset> From> for Utf8Array { } } } - -#[cfg(test)] -mod tests { - use std::iter::FromIterator; - - use super::*; - - use crate::array::Utf8Array; - - /// tests extending from a variable-sized (strings and binary) array w/ offset with nulls - #[test] - fn test_variable_sized_validity() { - let array = Utf8Array::::from_iter(vec![Some("a"), Some("bc"), None, Some("defh")]); - - let mut a = GrowableUtf8::new(vec![&array], false, 0); - - a.extend(0, 1, 2); - - let result: Utf8Array = a.into(); - - let expected = Utf8Array::::from_iter(vec![Some("bc"), None]); - assert_eq!(result, expected); - } - - /// tests extending from a variable-sized (strings and binary) array - /// with an offset and nulls - #[test] - fn test_variable_sized_offsets() { - let array = Utf8Array::::from_iter(vec![Some("a"), Some("bc"), None, Some("defh")]); - let array = array.slice(1, 3); - - let mut a = GrowableUtf8::new(vec![&array], false, 0); - - a.extend(0, 0, 3); - - let result: Utf8Array = a.into(); - - let expected = Utf8Array::::from_iter(vec![Some("bc"), None, Some("defh")]); - assert_eq!(result, expected); - } - - #[test] - fn test_string_offsets() { - let array = Utf8Array::::from_iter(vec![Some("a"), Some("bc"), None, Some("defh")]); - let array = array.slice(1, 3); - - let mut a = GrowableUtf8::new(vec![&array], false, 0); - - a.extend(0, 0, 3); - - let result: Utf8Array = a.into(); - - let expected = Utf8Array::::from_iter(vec![Some("bc"), None, Some("defh")]); - assert_eq!(result, expected); - } - - #[test] - fn test_multiple_with_validity() { - let array1 = Utf8Array::::from_slice(&["hello", "world"]); - let array2 = Utf8Array::::from(&[Some("1"), None]); - - let mut a = GrowableUtf8::new(vec![&array1, &array2], false, 5); - - a.extend(0, 0, 2); - a.extend(1, 0, 2); - - let result: Utf8Array = a.into(); - - let expected = - Utf8Array::::from_iter(vec![Some("hello"), Some("world"), Some("1"), None]); - assert_eq!(result, expected); - } - - #[test] - fn test_string_null_offset_validity() { - let array = Utf8Array::::from_iter(vec![Some("a"), Some("bc"), None, Some("defh")]); - let array = array.slice(1, 3); - - let mut a = GrowableUtf8::new(vec![&array], true, 0); - - a.extend(0, 1, 2); - a.extend_validity(1); - - let result: Utf8Array = a.into(); - - let expected = Utf8Array::::from_iter(vec![None, Some("defh"), None]); - assert_eq!(result, expected); - } -} diff --git a/src/array/list/mod.rs b/src/array/list/mod.rs index 44228aab823..8e767dda0a8 100644 --- a/src/array/list/mod.rs +++ b/src/array/list/mod.rs @@ -187,76 +187,3 @@ impl std::fmt::Display for ListArray { display_fmt(self.iter(), head, f, true) } } - -#[cfg(test)] -mod tests { - use crate::array::primitive::PrimitiveArray; - - use super::*; - - #[test] - fn display() { - let values = Buffer::from([1, 2, 3, 4, 5]); - let values = PrimitiveArray::::from_data(DataType::Int32, values, None); - - let data_type = ListArray::::default_datatype(DataType::Int32); - let array = ListArray::::from_data( - data_type, - Buffer::from([0, 2, 2, 3, 5]), - Arc::new(values), - None, - ); - - assert_eq!( - format!("{}", array), - "ListArray[\nInt32[1, 2],\nInt32[],\nInt32[3],\nInt32[4, 5]\n]" - ); - } - - #[test] - #[should_panic( - expected = "The child's datatype must match the inner type of the \'data_type\'" - )] - fn test_nested_panic() { - let values = Buffer::from([1, 2, 3, 4, 5]); - let values = PrimitiveArray::::from_data(DataType::Int32, values, None); - - let data_type = ListArray::::default_datatype(DataType::Int32); - let array = ListArray::::from_data( - data_type.clone(), - Buffer::from([0, 2, 2, 3, 5]), - Arc::new(values), - None, - ); - - // The datatype for the nested array has to be created considering - // the nested structure of the child data - let _ = - ListArray::::from_data(data_type, Buffer::from([0, 2, 4]), Arc::new(array), None); - } - - #[test] - fn test_nested_display() { - let values = Buffer::from([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); - let values = PrimitiveArray::::from_data(DataType::Int32, values, None); - - let data_type = ListArray::::default_datatype(DataType::Int32); - let array = ListArray::::from_data( - data_type, - Buffer::from([0, 2, 4, 7, 7, 8, 10]), - Arc::new(values), - None, - ); - - let data_type = ListArray::::default_datatype(array.data_type().clone()); - let nested = ListArray::::from_data( - data_type, - Buffer::from([0, 2, 5, 6]), - Arc::new(array), - None, - ); - - let expected = "ListArray[\nListArray[\nInt32[1, 2],\nInt32[3, 4]\n],\nListArray[\nInt32[5, 6, 7],\nInt32[],\nInt32[8]\n],\nListArray[\nInt32[9, 10]\n]\n]"; - assert_eq!(format!("{}", nested), expected); - } -} diff --git a/src/array/list/mutable.rs b/src/array/list/mutable.rs index 20378b254f8..3dde5bede06 100644 --- a/src/array/list/mutable.rs +++ b/src/array/list/mutable.rs @@ -190,42 +190,3 @@ impl MutableArray for MutableListArray>::new(); - array.try_extend(data).unwrap(); - let array: ListArray = array.into(); - - let values = PrimitiveArray::::from_data( - DataType::Int32, - Buffer::from([1, 2, 3, 4, 0, 6]), - Some(Bitmap::from([true, true, true, true, false, true])), - ); - - let data_type = ListArray::::default_datatype(DataType::Int32); - let expected = ListArray::::from_data( - data_type, - Buffer::from([0, 3, 3, 6]), - Arc::new(values), - Some(Bitmap::from([true, false, true])), - ); - assert_eq!(expected, array); - } -} diff --git a/src/array/mod.rs b/src/array/mod.rs index 37aa8f7d318..0ad9886275d 100644 --- a/src/array/mod.rs +++ b/src/array/mod.rs @@ -392,12 +392,13 @@ pub mod growable; pub mod ord; pub use display::get_display; +pub use equal::equal; pub use binary::{BinaryArray, MutableBinaryArray}; pub use boolean::{BooleanArray, MutableBooleanArray}; pub use dictionary::{DictionaryArray, DictionaryKey, MutableDictionaryArray}; -pub use fixed_size_binary::FixedSizeBinaryArray; -pub use fixed_size_list::FixedSizeListArray; +pub use fixed_size_binary::{FixedSizeBinaryArray, MutableFixedSizeBinaryArray}; +pub use fixed_size_list::{FixedSizeListArray, MutableFixedSizeListArray}; pub use list::{ListArray, MutableListArray}; pub use null::NullArray; pub use primitive::*; @@ -475,57 +476,6 @@ pub unsafe trait GenericBinaryArray: Array { fn offsets(&self) -> &[O]; } -#[cfg(test)] -mod tests { - use super::*; - use crate::datatypes::*; - - #[test] - fn nulls() { - let datatypes = vec![ - DataType::Int32, - DataType::Float64, - DataType::Utf8, - DataType::Binary, - DataType::List(Box::new(Field::new("a", DataType::Binary, true))), - ]; - let a = datatypes - .into_iter() - .all(|x| new_null_array(x, 10).null_count() == 10); - assert!(a); - } - - #[test] - fn empty() { - let datatypes = vec![ - DataType::Int32, - DataType::Float64, - DataType::Utf8, - DataType::Binary, - DataType::List(Box::new(Field::new("a", DataType::Binary, true))), - DataType::Union(vec![Field::new("a", DataType::Binary, true)], None, true), - DataType::Union(vec![Field::new("a", DataType::Binary, true)], None, false), - ]; - let a = datatypes.into_iter().all(|x| new_empty_array(x).len() == 0); - assert!(a); - } - - #[test] - fn test_clone() { - let datatypes = vec![ - DataType::Int32, - DataType::Float64, - DataType::Utf8, - DataType::Binary, - DataType::List(Box::new(Field::new("a", DataType::Binary, true))), - ]; - let a = datatypes - .into_iter() - .all(|x| clone(new_null_array(x.clone(), 10).as_ref()) == new_null_array(x, 10)); - assert!(a); - } -} - // backward compatibility use std::sync::Arc; pub type ArrayRef = Arc; diff --git a/src/array/ord.rs b/src/array/ord.rs index 654877adeca..1ca236de78f 100644 --- a/src/array/ord.rs +++ b/src/array/ord.rs @@ -205,108 +205,3 @@ pub fn build_compare<'a>(left: &'a dyn Array, right: &'a dyn Array) -> Result Result<()> { - let array = Int32Array::from_slice(&[1, 2]); - - let cmp = build_compare(&array, &array)?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - Ok(()) - } - - #[test] - fn test_i32_i32() -> Result<()> { - let array1 = Int32Array::from_slice(&[1]); - let array2 = Int32Array::from_slice(&[2]); - - let cmp = build_compare(&array1, &array2)?; - - assert_eq!(Ordering::Less, (cmp)(0, 0)); - Ok(()) - } - - #[test] - fn test_f32() -> Result<()> { - let array = &Float32Array::from_slice(&[1.0, 2.0]); - - let cmp = build_compare(array, array)?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - Ok(()) - } - - #[test] - fn test_f64() -> Result<()> { - let array = Float64Array::from_slice(&[1.0, 2.0]); - - let cmp = build_compare(&array, &array)?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - Ok(()) - } - - #[test] - fn test_f64_nan() -> Result<()> { - let array = Float64Array::from_slice(&[1.0, f64::NAN]); - - let cmp = build_compare(&array, &array)?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - Ok(()) - } - - #[test] - fn test_f64_zeros() -> Result<()> { - let array = Float64Array::from_slice(&[-0.0, 0.0]); - - let cmp = build_compare(&array, &array)?; - - // official IEEE 754 (2008 revision) - assert_eq!(Ordering::Less, (cmp)(0, 1)); - assert_eq!(Ordering::Greater, (cmp)(1, 0)); - Ok(()) - } - - #[test] - fn test_dict() -> Result<()> { - let data = vec!["a", "b", "c", "a", "a", "c", "c"]; - - let data = data.into_iter().map(Some); - let mut array = MutableDictionaryArray::>::new(); - array.try_extend(data)?; - let array: DictionaryArray = array.into(); - - let cmp = build_compare(&array, &array)?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - assert_eq!(Ordering::Equal, (cmp)(3, 4)); - assert_eq!(Ordering::Greater, (cmp)(2, 3)); - Ok(()) - } - - #[test] - fn test_dict_1() -> Result<()> { - let data = vec![1, 2, 3, 1, 1, 3, 3]; - - let data = data.into_iter().map(Some); - - let mut array = MutableDictionaryArray::>::new(); - array.try_extend(data)?; - let array = array.into_arc(); - - let cmp = build_compare(array.as_ref(), array.as_ref())?; - - assert_eq!(Ordering::Less, (cmp)(0, 1)); - assert_eq!(Ordering::Equal, (cmp)(3, 4)); - assert_eq!(Ordering::Greater, (cmp)(2, 3)); - Ok(()) - } -} diff --git a/src/array/primitive/display.rs b/src/array/primitive/display.rs index 5c924adacc1..911eb258798 100644 --- a/src/array/primitive/display.rs +++ b/src/array/primitive/display.rs @@ -13,166 +13,3 @@ impl std::fmt::Display for PrimitiveArray { display_fmt(iter, head, f, new_lines) } } - -#[cfg(test)] -mod tests { - use super::super::{DaysMsArray, Int128Array, Int32Array, Int64Array}; - use crate::datatypes::*; - use crate::types::days_ms; - - #[test] - fn display_int32() { - let array = Int32Array::from(&[Some(1), None, Some(2)]); - assert_eq!(format!("{}", array), "Int32[1, , 2]"); - } - - #[test] - fn display_date32() { - let array = Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Date32); - assert_eq!(format!("{}", array), "Date32[1970-01-02, , 1970-01-03]"); - } - - #[test] - fn display_time32s() { - let array = - Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Time32(TimeUnit::Second)); - assert_eq!(format!("{}", array), "Time32(Second)[00:00:01, , 00:00:02]"); - } - - #[test] - fn display_time32ms() { - let array = - Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Time32(TimeUnit::Millisecond)); - assert_eq!( - format!("{}", array), - "Time32(Millisecond)[00:00:00.001, , 00:00:00.002]" - ); - } - - #[test] - fn display_interval_d() { - let array = Int32Array::from(&[Some(1), None, Some(2)]) - .to(DataType::Interval(IntervalUnit::YearMonth)); - assert_eq!(format!("{}", array), "Interval(YearMonth)[1m, , 2m]"); - } - - #[test] - fn display_int64() { - let array = Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Int64); - assert_eq!(format!("{}", array), "Int64[1, , 2]"); - } - - #[test] - fn display_date64() { - let array = Int64Array::from(&[Some(1), None, Some(86400000)]).to(DataType::Date64); - assert_eq!(format!("{}", array), "Date64[1970-01-01, , 1970-01-02]"); - } - - #[test] - fn display_time64us() { - let array = - Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Time64(TimeUnit::Microsecond)); - assert_eq!( - format!("{}", array), - "Time64(Microsecond)[00:00:00.000001, , 00:00:00.000002]" - ); - } - - #[test] - fn display_time64ns() { - let array = - Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Time64(TimeUnit::Nanosecond)); - assert_eq!( - format!("{}", array), - "Time64(Nanosecond)[00:00:00.000000001, , 00:00:00.000000002]" - ); - } - - #[test] - fn display_timestamp_s() { - let array = Int64Array::from(&[Some(1), None, Some(2)]) - .to(DataType::Timestamp(TimeUnit::Second, None)); - assert_eq!( - format!("{}", array), - "Timestamp(Second, None)[1970-01-01 00:00:01, , 1970-01-01 00:00:02]" - ); - } - - #[test] - fn display_timestamp_ms() { - let array = Int64Array::from(&[Some(1), None, Some(2)]) - .to(DataType::Timestamp(TimeUnit::Millisecond, None)); - assert_eq!( - format!("{}", array), - "Timestamp(Millisecond, None)[1970-01-01 00:00:00.001, , 1970-01-01 00:00:00.002]" - ); - } - - #[test] - fn display_timestamp_us() { - let array = Int64Array::from(&[Some(1), None, Some(2)]) - .to(DataType::Timestamp(TimeUnit::Microsecond, None)); - assert_eq!( - format!("{}", array), - "Timestamp(Microsecond, None)[1970-01-01 00:00:00.000001, , 1970-01-01 00:00:00.000002]" - ); - } - - #[test] - fn display_timestamp_ns() { - let array = Int64Array::from(&[Some(1), None, Some(2)]) - .to(DataType::Timestamp(TimeUnit::Nanosecond, None)); - assert_eq!( - format!("{}", array), - "Timestamp(Nanosecond, None)[1970-01-01 00:00:00.000000001, , 1970-01-01 00:00:00.000000002]" - ); - } - - #[test] - fn display_duration_ms() { - let array = Int64Array::from(&[Some(1), None, Some(2)]) - .to(DataType::Duration(TimeUnit::Millisecond)); - assert_eq!(format!("{}", array), "Duration(Millisecond)[1ms, , 2ms]"); - } - - #[test] - fn display_duration_s() { - let array = - Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Second)); - assert_eq!(format!("{}", array), "Duration(Second)[1s, , 2s]"); - } - - #[test] - fn display_duration_us() { - let array = Int64Array::from(&[Some(1), None, Some(2)]) - .to(DataType::Duration(TimeUnit::Microsecond)); - assert_eq!(format!("{}", array), "Duration(Microsecond)[1us, , 2us]"); - } - - #[test] - fn display_duration_ns() { - let array = Int64Array::from(&[Some(1), None, Some(2)]) - .to(DataType::Duration(TimeUnit::Nanosecond)); - assert_eq!(format!("{}", array), "Duration(Nanosecond)[1ns, , 2ns]"); - } - - #[test] - fn display_decimal() { - let array = - Int128Array::from(&[Some(12345), None, Some(23456)]).to(DataType::Decimal(5, 2)); - assert_eq!(format!("{}", array), "Decimal(5, 2)[123.45, , 234.56]"); - } - - #[test] - fn display_decimal1() { - let array = - Int128Array::from(&[Some(12345), None, Some(23456)]).to(DataType::Decimal(5, 1)); - assert_eq!(format!("{}", array), "Decimal(5, 1)[1234.5, , 2345.6]"); - } - - #[test] - fn display_interval_days_ms() { - let array = DaysMsArray::from(&[Some(days_ms::new(1, 1)), None, Some(days_ms::new(2, 2))]); - assert_eq!(format!("{}", array), "Interval(DayTime)[1d1ms, , 2d2ms]"); - } -} diff --git a/src/array/primitive/from_natural.rs b/src/array/primitive/from_natural.rs index 02e030c0af6..f07a219a577 100644 --- a/src/array/primitive/from_natural.rs +++ b/src/array/primitive/from_natural.rs @@ -65,30 +65,3 @@ impl PrimitiveArray { MutablePrimitiveArray::::from_trusted_len_iter_unchecked(iter).into() } } - -#[cfg(test)] -mod tests { - use crate::array::Array; - - use super::*; - use std::iter::FromIterator; - - #[test] - fn bla() { - let data = vec![Some(1), None, Some(10)]; - - let array = PrimitiveArray::from(data.clone()); - assert_eq!(array.len(), 3); - - let array = PrimitiveArray::from_iter(data); - assert_eq!(array.len(), 3); - - let data = vec![1i32, 2, 3]; - - let array = PrimitiveArray::from_values(data.clone()); - assert_eq!(array.len(), 3); - - let array = PrimitiveArray::from_trusted_len_values_iter(data.into_iter()); - assert_eq!(array.len(), 3); - } -} diff --git a/src/array/primitive/mod.rs b/src/array/primitive/mod.rs index 5a394e817c4..49033ba9dec 100644 --- a/src/array/primitive/mod.rs +++ b/src/array/primitive/mod.rs @@ -204,53 +204,3 @@ pub type UInt16Vec = MutablePrimitiveArray; pub type UInt32Vec = MutablePrimitiveArray; /// A type definition [`MutablePrimitiveArray`] for `u64` pub type UInt64Vec = MutablePrimitiveArray; - -#[cfg(test)] -mod tests { - use super::*; - use crate::array::Int32Array; - use std::iter::FromIterator; - - #[test] - fn basics() { - let data = vec![Some(1), None, Some(10)]; - - let array = Int32Array::from_iter(data); - - assert_eq!(array.value(0), 1); - assert_eq!(array.value(1), 0); - assert_eq!(array.value(2), 10); - assert_eq!(array.values().as_slice(), &[1, 0, 10]); - assert_eq!( - array.validity(), - &Some(Bitmap::from_u8_slice(&[0b00000101], 3)) - ); - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(array.is_valid(2)); - - let array2 = PrimitiveArray::::from_data( - DataType::Int32, - array.values().clone(), - array.validity().clone(), - ); - assert_eq!(array, array2); - - let array = array.slice(1, 2); - assert_eq!(array.value(0), 0); - assert_eq!(array.value(1), 10); - assert_eq!(array.values().as_slice(), &[0, 10]); - - unsafe { - assert_eq!(array.value_unchecked(0), 0); - assert_eq!(array.value_unchecked(1), 10); - } - } - - #[test] - fn empty() { - let array = PrimitiveArray::::new_empty(DataType::Int32); - assert_eq!(array.values().len(), 0); - assert_eq!(array.validity(), &None); - } -} diff --git a/src/array/primitive/mutable.rs b/src/array/primitive/mutable.rs index 7a66cdda221..69220c3cfa6 100644 --- a/src/array/primitive/mutable.rs +++ b/src/array/primitive/mutable.rs @@ -543,79 +543,3 @@ where Ok((validity, buffer)) } - -#[cfg(test)] -mod tests { - use crate::bitmap::Bitmap; - - use super::*; - - #[test] - fn push() { - let mut a = MutablePrimitiveArray::::new(); - a.push(Some(1)); - a.push(None); - assert_eq!(a.len(), 2); - assert!(a.is_valid(0)); - assert!(!a.is_valid(1)); - - assert_eq!(a.values(), &MutableBuffer::from([1, 0])); - } - - #[test] - fn set() { - let mut a = MutablePrimitiveArray::::from([Some(1), None]); - - a.set(0, Some(2)); - a.set(1, Some(1)); - - assert_eq!(a.len(), 2); - assert!(a.is_valid(0)); - assert!(a.is_valid(1)); - - assert_eq!(a.values(), &MutableBuffer::from([2, 1])); - } - - #[test] - fn from_iter() { - let a = MutablePrimitiveArray::::from_iter((0..2).map(Some)); - assert_eq!(a.len(), 2); - assert_eq!(a.validity(), &None); - } - - #[test] - fn natural_arc() { - let a = MutablePrimitiveArray::::from_slice(&[0, 1]).into_arc(); - assert_eq!(a.len(), 2); - } - - #[test] - fn only_nulls() { - let mut a = MutablePrimitiveArray::::new(); - a.push(None); - a.push(None); - let a: PrimitiveArray = a.into(); - assert_eq!(a.validity(), &Some(Bitmap::from([false, false]))); - } - - #[test] - fn from_trusted_len() { - let a = - MutablePrimitiveArray::::from_trusted_len_iter(vec![Some(1), None].into_iter()); - let a: PrimitiveArray = a.into(); - assert_eq!(a.validity(), &Some(Bitmap::from([true, false]))); - } - - #[test] - fn extend_trusted_len() { - let mut a = MutablePrimitiveArray::::new(); - a.extend_trusted_len(vec![Some(1), Some(2)].into_iter()); - assert_eq!(a.validity(), &None); - a.extend_trusted_len(vec![None, Some(4)].into_iter()); - assert_eq!( - a.validity(), - &Some(MutableBitmap::from([true, true, false, true])) - ); - assert_eq!(a.values(), &MutableBuffer::::from([1, 2, 0, 4])); - } -} diff --git a/src/array/union/mod.rs b/src/array/union/mod.rs index bccf74812e8..f688656e2f3 100644 --- a/src/array/union/mod.rs +++ b/src/array/union/mod.rs @@ -247,29 +247,3 @@ impl std::fmt::Display for UnionArray { display_fmt(iter, head, f, new_lines) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::{array::*, buffer::Buffer, datatypes::*, error::Result}; - - #[test] - fn display() -> Result<()> { - let fields = vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, true), - ]; - let data_type = DataType::Union(fields, None, true); - let types = Buffer::from(&[0, 0, 1]); - let fields = vec![ - Arc::new(Int32Array::from(&[Some(1), None, Some(2)])) as Arc, - Arc::new(Utf8Array::::from(&[Some("a"), Some("b"), Some("c")])) as Arc, - ]; - - let array = UnionArray::from_data(data_type, types, fields, None); - - assert_eq!(format!("{}", array), "UnionArray[1, , c]"); - - Ok(()) - } -} diff --git a/src/array/utf8/from.rs b/src/array/utf8/from.rs index 036e548699c..faf9125ed07 100644 --- a/src/array/utf8/from.rs +++ b/src/array/utf8/from.rs @@ -253,27 +253,3 @@ impl> FromIterator> for Utf8Array { MutableUtf8Array::from_iter(iter).into() } } - -#[cfg(test)] -mod tests { - use crate::array::Array; - - use super::*; - - #[test] - fn test_from() { - let array = Utf8Array::::from(&[Some("hello"), Some(" "), None]); - - let a = array.validity().as_ref().unwrap(); - assert_eq!(a, &Bitmap::from([true, true, false])); - } - - #[test] - fn test_from_iter_values() { - let b = Utf8Array::::from_iter_values(vec!["a", "b", "cc"]); - - let offsets = Buffer::from(&[0, 1, 2, 4]); - let values = Buffer::from("abcc".as_bytes()); - assert_eq!(b, Utf8Array::::from_data(offsets, values, None)); - } -} diff --git a/src/array/utf8/mod.rs b/src/array/utf8/mod.rs index 11f6ebdd10e..a9ed14c6b64 100644 --- a/src/array/utf8/mod.rs +++ b/src/array/utf8/mod.rs @@ -200,52 +200,3 @@ unsafe impl GenericBinaryArray for Utf8Array { self.offsets() } } - -#[cfg(test)] -mod tests { - use super::*; - use std::iter::FromIterator; - - #[test] - fn basics() { - let data = vec![Some("hello"), None, Some("hello2")]; - - let array = Utf8Array::::from_iter(data); - - assert_eq!(array.value(0), "hello"); - assert_eq!(array.value(1), ""); - assert_eq!(array.value(2), "hello2"); - assert_eq!(unsafe { array.value_unchecked(2) }, "hello2"); - assert_eq!(array.values().as_slice(), b"hellohello2"); - assert_eq!(array.offsets().as_slice(), &[0, 5, 5, 11]); - assert_eq!( - array.validity(), - &Some(Bitmap::from_u8_slice(&[0b00000101], 3)) - ); - assert!(array.is_valid(0)); - assert!(!array.is_valid(1)); - assert!(array.is_valid(2)); - - let array2 = Utf8Array::::from_data( - array.offsets().clone(), - array.values().clone(), - array.validity().clone(), - ); - assert_eq!(array, array2); - - let array = array.slice(1, 2); - assert_eq!(array.value(0), ""); - assert_eq!(array.value(1), "hello2"); - // note how this keeps everything: the offsets were sliced - assert_eq!(array.values().as_slice(), b"hellohello2"); - assert_eq!(array.offsets().as_slice(), &[5, 5, 11]); - } - - #[test] - fn empty() { - let array = Utf8Array::::new_empty(); - assert_eq!(array.values().as_slice(), b""); - assert_eq!(array.offsets().as_slice(), &[0]); - assert_eq!(array.validity(), &None); - } -} diff --git a/src/array/utf8/mutable.rs b/src/array/utf8/mutable.rs index c8d73ac0fdf..4ba5038eb7f 100644 --- a/src/array/utf8/mutable.rs +++ b/src/array/utf8/mutable.rs @@ -213,27 +213,3 @@ impl> TryExtend> for MutableUtf8Array { iter.try_for_each(|x| self.try_push(x)) } } - -#[cfg(test)] -mod tests { - use crate::bitmap::Bitmap; - - use super::*; - - #[test] - fn test_capacities() { - let b = MutableUtf8Array::::with_capacities(1, 10); - - assert_eq!(b.values.capacity(), 64); - assert_eq!(b.offsets.capacity(), 16); // 64 bytes - } - - #[test] - fn push_null() { - let mut array = MutableUtf8Array::::new(); - array.push::<&str>(None); - - let array: Utf8Array = array.into(); - assert_eq!(array.validity(), &Some(Bitmap::from([false]))); - } -} diff --git a/src/bitmap/bitmap_ops.rs b/src/bitmap/bitmap_ops.rs index 67b036ff004..d092a24aef2 100644 --- a/src/bitmap/bitmap_ops.rs +++ b/src/bitmap/bitmap_ops.rs @@ -177,96 +177,3 @@ impl Not for &Bitmap { unary(self, |a| !a) } } - -#[cfg(test)] -mod test { - use super::*; - - use crate::bitmap::Bitmap; - use crate::buffer::MutableBuffer; - - fn create_bitmap>(bytes: P, len: usize) -> Bitmap { - let buffer = MutableBuffer::::from(bytes.as_ref()); - Bitmap::from_u8_buffer(buffer, len) - } - - #[test] - fn test_eq() { - let lhs = create_bitmap([0b01101010], 8); - let rhs = create_bitmap([0b01001110], 8); - assert!(!eq(&lhs, &rhs)); - assert!(eq(&lhs, &lhs)); - } - - #[test] - fn test_eq_len() { - let lhs = create_bitmap([0b01101010], 6); - let rhs = create_bitmap([0b00101010], 6); - assert!(eq(&lhs, &rhs)); - let rhs = create_bitmap([0b00001010], 6); - assert!(!eq(&lhs, &rhs)); - } - - #[test] - fn test_eq_slice() { - let lhs = create_bitmap([0b10101010], 8).slice(1, 7); - let rhs = create_bitmap([0b10101011], 8).slice(1, 7); - assert!(eq(&lhs, &rhs)); - - let lhs = create_bitmap([0b10101010], 8).slice(2, 6); - let rhs = create_bitmap([0b10101110], 8).slice(2, 6); - assert!(!eq(&lhs, &rhs)); - } - - #[test] - fn test_and() { - let lhs = create_bitmap([0b01101010], 8); - let rhs = create_bitmap([0b01001110], 8); - let expected = create_bitmap([0b01001010], 8); - assert_eq!(&lhs & &rhs, expected); - } - - #[test] - fn test_or_large() { - let input: &[u8] = &[ - 0b00000000, 0b00000001, 0b00000010, 0b00000100, 0b00001000, 0b00010000, 0b00100000, - 0b01000010, 0b11111111, - ]; - let input1: &[u8] = &[ - 0b00000000, 0b00000001, 0b10000000, 0b10000000, 0b10000000, 0b10000000, 0b10000000, - 0b10000000, 0b11111111, - ]; - let expected: &[u8] = &[ - 0b00000000, 0b00000001, 0b10000010, 0b10000100, 0b10001000, 0b10010000, 0b10100000, - 0b11000010, 0b11111111, - ]; - - let lhs = create_bitmap(input, 62); - let rhs = create_bitmap(input1, 62); - let expected = create_bitmap(expected, 62); - assert_eq!(&lhs | &rhs, expected); - } - - #[test] - fn test_and_offset() { - let lhs = create_bitmap([0b01101011], 8).slice(1, 7); - let rhs = create_bitmap([0b01001111], 8).slice(1, 7); - let expected = create_bitmap([0b01001010], 8).slice(1, 7); - assert_eq!(&lhs & &rhs, expected); - } - - #[test] - fn test_or() { - let lhs = create_bitmap([0b01101010], 8); - let rhs = create_bitmap([0b01001110], 8); - let expected = create_bitmap([0b01101110], 8); - assert_eq!(&lhs | &rhs, expected); - } - - #[test] - fn test_not() { - let lhs = create_bitmap([0b01101010], 6); - let expected = create_bitmap([0b00010101], 6); - assert_eq!(!&lhs, expected); - } -} diff --git a/src/io/csv/read/deserialize.rs b/src/io/csv/read/deserialize.rs index 84fc1bead6e..272b042122e 100644 --- a/src/io/csv/read/deserialize.rs +++ b/src/io/csv/read/deserialize.rs @@ -197,85 +197,3 @@ where RecordBatch::try_new(schema, columns) } - -#[cfg(test)] -mod tests { - use super::super::*; - use super::*; - - fn test(input: &str, data_type: DataType) -> Result> { - let reader = std::io::Cursor::new(input); - let mut reader = ReaderBuilder::new().has_headers(false).from_reader(reader); - - let mut rows = vec![ByteRecord::default(); 10]; - let rows_read = read_rows(&mut reader, 0, &mut rows)?; - deserialize_column(&rows[..rows_read], 0, data_type, 0) - } - - #[test] - fn int32() -> Result<()> { - let result = test("1,\n,\n3,", DataType::Int32)?; - let expected = Int32Array::from(&[Some(1), None, Some(3)]); - assert_eq!(expected, result.as_ref()); - Ok(()) - } - - #[test] - fn date32() -> Result<()> { - let result = test("1970-01-01,\n2020-03-15,\n1945-05-08,\n", DataType::Date32)?; - let expected = Int32Array::from(&[Some(0), Some(18336), Some(-9004)]).to(DataType::Date32); - assert_eq!(expected, result.as_ref()); - Ok(()) - } - - #[test] - fn date64() -> Result<()> { - let input = "1970-01-01T00:00:00,\n \ - 2018-11-13T17:11:10,\n \ - 2018-11-13T17:11:10.011,\n \ - 1900-02-28T12:34:56,\n"; - - let result = test(input, DataType::Date64)?; - let expected = Int64Array::from(&[ - Some(0), - Some(1542129070000), - Some(1542129070011), - Some(-2203932304000), - ]) - .to(DataType::Date64); - assert_eq!(expected, result.as_ref()); - Ok(()) - } - - #[test] - fn boolean() -> Result<()> { - let input = vec!["true", "True", "False", "F", "t"]; - let input = input.join("\n"); - - let expected = BooleanArray::from(&[Some(true), Some(true), Some(false), None, None]); - - let result = test(&input, DataType::Boolean)?; - - assert_eq!(expected, result.as_ref()); - Ok(()) - } - - #[test] - fn float32() -> Result<()> { - let input = vec!["12.34", "12", "0.0", "inf", "-inf", "dd"]; - let input = input.join("\n"); - - let expected = Float32Array::from(&[ - Some(12.34), - Some(12.0), - Some(0.0), - Some(f32::INFINITY), - Some(f32::NEG_INFINITY), - None, - ]); - - let result = test(&input, DataType::Float32)?; - assert_eq!(expected, result.as_ref()); - Ok(()) - } -} diff --git a/src/io/csv/read/infer_schema.rs b/src/io/csv/read/infer_schema.rs index be48e3ef677..e382d8a1bf9 100644 --- a/src/io/csv/read/infer_schema.rs +++ b/src/io/csv/read/infer_schema.rs @@ -92,51 +92,3 @@ pub fn infer_schema DataType>( Ok(Schema::new(fields)) } - -#[cfg(test)] -mod tests { - use super::*; - - use std::io::Cursor; - - use crate::datatypes::{DataType, Field, Schema}; - use crate::error::Result; - - use super::super::{infer, ReaderBuilder}; - - #[test] - fn basics() -> Result<()> { - let file = Cursor::new("1,2,3\na,b,c\na,,c"); - let mut reader = ReaderBuilder::new().from_reader(file); - - let schema = infer_schema(&mut reader, Some(10), false, &infer)?; - - assert_eq!( - schema, - Schema::new(vec![ - Field::new("column_1", DataType::Utf8, true), - Field::new("column_2", DataType::Utf8, true), - Field::new("column_3", DataType::Utf8, true), - ]) - ); - Ok(()) - } - - #[test] - fn ints() -> Result<()> { - let file = Cursor::new("1,2,3\n1,a,5\n2,,4"); - let mut reader = ReaderBuilder::new().from_reader(file); - - let schema = infer_schema(&mut reader, Some(10), false, &infer)?; - - assert_eq!( - schema, - Schema::new(vec![ - Field::new("column_1", DataType::Int64, true), - Field::new("column_2", DataType::Utf8, true), - Field::new("column_3", DataType::Int64, true), - ]) - ); - Ok(()) - } -} diff --git a/src/io/csv/read/reader.rs b/src/io/csv/read/reader.rs index f6412aa6049..7b16566d251 100644 --- a/src/io/csv/read/reader.rs +++ b/src/io/csv/read/reader.rs @@ -85,54 +85,3 @@ pub fn infer(string: &str) -> DataType { DataType::Utf8 } } - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use super::super::{deserialize_batch, deserialize_column, infer_schema, ReaderBuilder}; - use super::*; - - use crate::array::{Float64Array, Utf8Array}; - - #[test] - fn test_read() -> Result<()> { - let mut reader = ReaderBuilder::new().from_path("test/data/uk_cities_with_headers.csv")?; - - let schema = Arc::new(infer_schema(&mut reader, None, true, &infer)?); - - let mut rows = vec![ByteRecord::default(); 100]; - let rows_read = read_rows(&mut reader, 0, &mut rows)?; - - let batch = deserialize_batch( - &rows[..rows_read], - schema.fields(), - None, - 0, - deserialize_column, - )?; - - let batch_schema = batch.schema(); - - assert_eq!(&schema, batch_schema); - assert_eq!(37, batch.num_rows()); - assert_eq!(3, batch.num_columns()); - - let lat = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - assert!((57.653484 - lat.value(0)).abs() < f64::EPSILON); - - let city = batch - .column(0) - .as_any() - .downcast_ref::>() - .unwrap(); - - assert_eq!("Elgin, Scotland, the UK", city.value(0)); - assert_eq!("Aberdeen, Aberdeen City, UK", city.value(13)); - Ok(()) - } -} diff --git a/src/io/csv/write/mod.rs b/src/io/csv/write/mod.rs index 507bfc6ae75..61c37424a31 100644 --- a/src/io/csv/write/mod.rs +++ b/src/io/csv/write/mod.rs @@ -72,110 +72,3 @@ pub fn write_header(writer: &mut Writer, schema: &Schema) -> Result writer.write_record(&fields)?; Ok(()) } - -#[cfg(test)] -mod tests { - use super::WriterBuilder; - - use super::*; - - use crate::array::*; - use crate::datatypes::*; - - use std::io::Cursor; - use std::sync::Arc; - - fn data() -> RecordBatch { - let schema = Schema::new(vec![ - Field::new("c1", DataType::Utf8, false), - Field::new("c2", DataType::Float64, true), - Field::new("c3", DataType::UInt32, false), - Field::new("c4", DataType::Boolean, true), - Field::new("c5", DataType::Timestamp(TimeUnit::Millisecond, None), true), - Field::new("c6", DataType::Time32(TimeUnit::Second), false), - ]); - - let c1 = Utf8Array::::from_slice([ - "Lorem ipsum dolor sit amet", - "consectetur adipiscing elit", - "sed do eiusmod tempor", - ]); - let c2 = Float64Array::from([Some(123.564532), None, Some(-556132.25)]); - let c3 = UInt32Array::from_slice(&[3, 2, 1]); - let c4 = BooleanArray::from(&[Some(true), Some(false), None]); - let c5 = PrimitiveArray::::from([None, Some(1555584887378), Some(1555555555555)]) - .to(DataType::Timestamp(TimeUnit::Millisecond, None)); - let c6 = PrimitiveArray::::from_slice(&[1234, 24680, 85563]) - .to(DataType::Time32(TimeUnit::Second)); - - RecordBatch::try_new( - Arc::new(schema), - vec![ - Arc::new(c1), - Arc::new(c2), - Arc::new(c3), - Arc::new(c4), - Arc::new(c5), - Arc::new(c6), - ], - ) - .unwrap() - } - - #[test] - fn test_write_csv() -> Result<()> { - let batch = data(); - - let write = Cursor::new(Vec::::new()); - let mut writer = WriterBuilder::new().from_writer(write); - - write_header(&mut writer, batch.schema())?; - let batches = vec![&batch, &batch]; - let options = SerializeOptions::default(); - batches - .iter() - .try_for_each(|batch| write_batch(&mut writer, batch, &options))?; - - // check - let buffer = writer.into_inner().unwrap().into_inner(); - assert_eq!( - r#"c1,c2,c3,c4,c5,c6 -Lorem ipsum dolor sit amet,123.564532,3,true,,00:20:34 -consectetur adipiscing elit,,2,false,2019-04-18T10:54:47.378000000,06:51:20 -sed do eiusmod tempor,-556132.25,1,,2019-04-18T02:45:55.555000000,23:46:03 -Lorem ipsum dolor sit amet,123.564532,3,true,,00:20:34 -consectetur adipiscing elit,,2,false,2019-04-18T10:54:47.378000000,06:51:20 -sed do eiusmod tempor,-556132.25,1,,2019-04-18T02:45:55.555000000,23:46:03 -"# - .to_string(), - String::from_utf8(buffer).unwrap(), - ); - Ok(()) - } - - #[test] - fn test_write_csv_custom_options() -> Result<()> { - let batch = data(); - - let write = Cursor::new(Vec::::new()); - let mut writer = WriterBuilder::new().delimiter(b'|').from_writer(write); - - let options = SerializeOptions { - time_format: "%r".to_string(), - ..Default::default() - }; - write_batch(&mut writer, &batch, &options)?; - - // check - let buffer = writer.into_inner().unwrap().into_inner(); - assert_eq!( - r#"Lorem ipsum dolor sit amet|123.564532|3|true||12:20:34 AM -consectetur adipiscing elit||2|false|2019-04-18T10:54:47.378000000|06:51:20 AM -sed do eiusmod tempor|-556132.25|1||2019-04-18T02:45:55.555000000|11:46:03 PM -"# - .to_string(), - String::from_utf8(buffer).unwrap(), - ); - Ok(()) - } -} diff --git a/src/io/json/mod.rs b/src/io/json/mod.rs index 7d8e2136dcc..8870b04dd40 100644 --- a/src/io/json/mod.rs +++ b/src/io/json/mod.rs @@ -30,60 +30,3 @@ impl From for ArrowError { ArrowError::External("".to_string(), Box::new(error)) } } - -#[cfg(test)] -mod tests { - use super::*; - - use serde_json::Value; - use std::fs::{read_to_string, File}; - - fn test_write_for_file(test_file: &str) { - let builder = ReaderBuilder::new() - .infer_schema(None) - .with_batch_size(1024); - let mut reader: Reader = builder - .build::(File::open(test_file).unwrap()) - .unwrap(); - let batch = reader.next().unwrap().unwrap(); - - let mut buf = Vec::new(); - { - let mut writer = LineDelimitedWriter::new(&mut buf); - writer.write_batches(&[batch]).unwrap(); - } - - let result = String::from_utf8(buf).unwrap(); - let expected = read_to_string(test_file).unwrap(); - for (r, e) in result.lines().zip(expected.lines()) { - let mut result_json = serde_json::from_str::(r).unwrap(); - let expected_json = serde_json::from_str::(e).unwrap(); - if let Value::Object(e) = &expected_json { - // remove null value from object to make comparision consistent: - if let Value::Object(r) = result_json { - result_json = Value::Object( - r.into_iter() - .filter(|(k, v)| e.contains_key(k) || *v != Value::Null) - .collect(), - ); - } - assert_eq!(result_json, expected_json); - } - } - } - - #[test] - fn write_basic_rows() { - test_write_for_file("test/data/basic.json"); - } - - #[test] - fn write_arrays() { - test_write_for_file("test/data/arrays.json"); - } - - #[test] - fn write_basic_nulls() { - test_write_for_file("test/data/basic_nulls.json"); - } -} diff --git a/src/record_batch.rs b/src/record_batch.rs index 3f81a979c47..07e512cd06c 100644 --- a/src/record_batch.rs +++ b/src/record_batch.rs @@ -384,117 +384,3 @@ pub trait RecordBatchReader: Iterator> { self.next().transpose() } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn basic() { - let schema = Schema::new(vec![ - Field::new("a", DataType::Int32, false), - Field::new("b", DataType::Utf8, false), - ]); - - let a = Int32Array::from_slice(&[1, 2, 3, 4, 5]); - let b = Utf8Array::::from_slice(&["a", "b", "c", "d", "e"]); - - let record_batch = - RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap(); - check_batch(record_batch) - } - - fn check_batch(record_batch: RecordBatch) { - assert_eq!(5, record_batch.num_rows()); - assert_eq!(2, record_batch.num_columns()); - assert_eq!(&DataType::Int32, record_batch.schema().field(0).data_type()); - assert_eq!(&DataType::Utf8, record_batch.schema().field(1).data_type()); - assert_eq!(5, record_batch.column(0).len()); - assert_eq!(5, record_batch.column(1).len()); - } - - #[test] - fn try_from_iter() { - let a: ArrayRef = Arc::new(Int32Array::from(vec![ - Some(1), - Some(2), - None, - Some(4), - Some(5), - ])); - let b: ArrayRef = Arc::new(Utf8Array::::from_slice(&["a", "b", "c", "d", "e"])); - - let record_batch = - RecordBatch::try_from_iter(vec![("a", a), ("b", b)]).expect("valid conversion"); - - let expected_schema = Schema::new(vec![ - Field::new("a", DataType::Int32, true), - Field::new("b", DataType::Utf8, false), - ]); - assert_eq!(record_batch.schema().as_ref(), &expected_schema); - check_batch(record_batch); - } - - #[test] - fn try_from_iter_with_nullable() { - let a: ArrayRef = Arc::new(Int32Array::from_slice(&[1, 2, 3, 4, 5])); - let b: ArrayRef = Arc::new(Utf8Array::::from_slice(&["a", "b", "c", "d", "e"])); - - // Note there are no nulls in a or b, but we specify that b is nullable - let record_batch = - RecordBatch::try_from_iter_with_nullable(vec![("a", a, false), ("b", b, true)]) - .expect("valid conversion"); - - let expected_schema = Schema::new(vec![ - Field::new("a", DataType::Int32, false), - Field::new("b", DataType::Utf8, true), - ]); - assert_eq!(record_batch.schema().as_ref(), &expected_schema); - check_batch(record_batch); - } - - #[test] - fn type_mismatch() { - let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); - - let a = Int64Array::from_slice(&[1, 2, 3, 4, 5]); - - let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]); - assert!(batch.is_err()); - } - - #[test] - fn number_of_fields_mismatch() { - let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); - - let a = Int32Array::from_slice(&[1, 2, 3, 4, 5]); - let b = Int32Array::from_slice(&[1, 2, 3, 4, 5]); - - let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]); - assert!(batch.is_err()); - } - - #[test] - fn from_struct_array() { - let boolean = Arc::new(BooleanArray::from_slice(&[false, false, true, true])) as ArrayRef; - let int = Arc::new(Int32Array::from_slice(&[42, 28, 19, 31])) as ArrayRef; - let struct_array = StructArray::from_data( - vec![ - Field::new("b", DataType::Boolean, false), - Field::new("c", DataType::Int32, false), - ], - vec![boolean.clone(), int.clone()], - None, - ); - - let batch = RecordBatch::from(&struct_array); - assert_eq!(2, batch.num_columns()); - assert_eq!(4, batch.num_rows()); - assert_eq!( - struct_array.data_type(), - &DataType::Struct(batch.schema().fields().to_vec()) - ); - assert_eq!(boolean.as_ref(), batch.column(0).as_ref()); - assert_eq!(int.as_ref(), batch.column(1).as_ref()); - } -} diff --git a/tests/it/array/binary/mod.rs b/tests/it/array/binary/mod.rs new file mode 100644 index 00000000000..1f48730acd6 --- /dev/null +++ b/tests/it/array/binary/mod.rs @@ -0,0 +1,55 @@ +use arrow2::{ + array::{Array, BinaryArray}, + bitmap::Bitmap, +}; + +#[test] +fn basics() { + let data = vec![Some(b"hello".to_vec()), None, Some(b"hello2".to_vec())]; + + let array: BinaryArray = data.into_iter().collect(); + + assert_eq!(array.value(0), b"hello"); + assert_eq!(array.value(1), b""); + assert_eq!(array.value(2), b"hello2"); + assert_eq!(unsafe { array.value_unchecked(2) }, b"hello2"); + assert_eq!(array.values().as_slice(), b"hellohello2"); + assert_eq!(array.offsets().as_slice(), &[0, 5, 5, 11]); + assert_eq!( + array.validity(), + &Some(Bitmap::from_u8_slice(&[0b00000101], 3)) + ); + assert!(array.is_valid(0)); + assert!(!array.is_valid(1)); + assert!(array.is_valid(2)); + + let array2 = BinaryArray::::from_data( + array.offsets().clone(), + array.values().clone(), + array.validity().clone(), + ); + assert_eq!(array, array2); + + let array = array.slice(1, 2); + assert_eq!(array.value(0), b""); + assert_eq!(array.value(1), b"hello2"); + // note how this keeps everything: the offsets were sliced + assert_eq!(array.values().as_slice(), b"hellohello2"); + assert_eq!(array.offsets().as_slice(), &[5, 5, 11]); +} + +#[test] +fn empty() { + let array = BinaryArray::::new_empty(); + assert_eq!(array.values().as_slice(), b""); + assert_eq!(array.offsets().as_slice(), &[0]); + assert_eq!(array.validity(), &None); +} + +#[test] +fn from() { + let array = BinaryArray::::from(&[Some(b"hello".as_ref()), Some(b" ".as_ref()), None]); + + let a = array.validity().as_ref().unwrap(); + assert_eq!(a, &Bitmap::from([true, true, false])); +} diff --git a/tests/it/array/boolean/mod.rs b/tests/it/array/boolean/mod.rs new file mode 100644 index 00000000000..ce2cb815d13 --- /dev/null +++ b/tests/it/array/boolean/mod.rs @@ -0,0 +1,53 @@ +use arrow2::{ + array::{Array, BooleanArray}, + bitmap::Bitmap, +}; + +mod mutable; + +#[test] +fn basics() { + let data = vec![Some(true), None, Some(false)]; + + let array: BooleanArray = data.into_iter().collect(); + + assert!(array.value(0)); + assert!(!array.value(1)); + assert!(!array.value(2)); + assert_eq!(array.values(), &Bitmap::from_u8_slice(&[0b00000001], 3)); + assert_eq!( + array.validity(), + &Some(Bitmap::from_u8_slice(&[0b00000101], 3)) + ); + assert!(array.is_valid(0)); + assert!(!array.is_valid(1)); + assert!(array.is_valid(2)); + + let array2 = BooleanArray::from_data(array.values().clone(), array.validity().clone()); + assert_eq!(array, array2); + + let array = array.slice(1, 2); + assert!(!array.value(0)); + assert!(!array.value(1)); +} + +#[test] +fn empty() { + let array = BooleanArray::new_empty(); + assert_eq!(array.values().len(), 0); + assert_eq!(array.validity(), &None); +} + +#[test] +fn from_trusted_len_iter() { + let iter = std::iter::repeat(true).take(2).map(Some); + let a = BooleanArray::from_trusted_len_iter(iter); + assert_eq!(a.len(), 2); +} + +#[test] +fn from_iter() { + let iter = std::iter::repeat(true).take(2).map(Some); + let a: BooleanArray = iter.collect(); + assert_eq!(a.len(), 2); +} diff --git a/tests/it/array/boolean/mutable.rs b/tests/it/array/boolean/mutable.rs new file mode 100644 index 00000000000..9408c1e89de --- /dev/null +++ b/tests/it/array/boolean/mutable.rs @@ -0,0 +1,30 @@ +use arrow2::{ + array::{MutableArray, MutableBooleanArray}, + bitmap::MutableBitmap, +}; + +#[test] +fn push() { + let mut a = MutableBooleanArray::new(); + a.push(Some(true)); + a.push(None); + assert_eq!(a.len(), 2); + assert!(a.is_valid(0)); + assert!(!a.is_valid(1)); + + assert_eq!(a.values(), &MutableBitmap::from([true, false])); +} + +#[test] +fn from_trusted_len_iter() { + let iter = std::iter::repeat(true).take(2).map(Some); + let a = MutableBooleanArray::from_trusted_len_iter(iter); + assert_eq!(a.len(), 2); +} + +#[test] +fn from_iter() { + let iter = std::iter::repeat(true).take(2).map(Some); + let a: MutableBooleanArray = iter.collect(); + assert_eq!(a.len(), 2); +} diff --git a/tests/it/array/dictionary/mod.rs b/tests/it/array/dictionary/mod.rs new file mode 100644 index 00000000000..f7a9b61988d --- /dev/null +++ b/tests/it/array/dictionary/mod.rs @@ -0,0 +1 @@ +mod mutable; diff --git a/tests/it/array/dictionary/mutable.rs b/tests/it/array/dictionary/mutable.rs new file mode 100644 index 00000000000..924ba90646b --- /dev/null +++ b/tests/it/array/dictionary/mutable.rs @@ -0,0 +1,40 @@ +use arrow2::array::*; +use arrow2::error::Result; + +#[test] +fn primitive() -> Result<()> { + let data = vec![Some(1), Some(2), Some(1)]; + + let mut a = MutableDictionaryArray::>::new(); + a.try_extend(data)?; + assert_eq!(a.len(), 3); + assert_eq!(a.values().len(), 2); + Ok(()) +} + +#[test] +fn utf8_natural() -> Result<()> { + let data = vec![Some("a"), Some("b"), Some("a")]; + + let mut a = MutableDictionaryArray::>::new(); + a.try_extend(data)?; + + assert_eq!(a.len(), 3); + assert_eq!(a.values().len(), 2); + Ok(()) +} + +#[test] +fn binary_natural() -> Result<()> { + let data = vec![ + Some("a".as_bytes()), + Some("b".as_bytes()), + Some("a".as_bytes()), + ]; + + let mut a = MutableDictionaryArray::>::new(); + a.try_extend(data)?; + assert_eq!(a.len(), 3); + assert_eq!(a.values().len(), 2); + Ok(()) +} diff --git a/tests/it/array/equal/boolean.rs b/tests/it/array/equal/boolean.rs new file mode 100644 index 00000000000..1fc9082de55 --- /dev/null +++ b/tests/it/array/equal/boolean.rs @@ -0,0 +1,53 @@ +use arrow2::array::*; + +use super::test_equal; + +#[test] +fn test_boolean_equal() { + let a = BooleanArray::from_slice([false, false, true]); + let b = BooleanArray::from_slice([false, false, true]); + test_equal(&a, &b, true); + + let b = BooleanArray::from_slice([false, false, false]); + test_equal(&a, &b, false); +} + +#[test] +fn test_boolean_equal_null() { + let a = BooleanArray::from(vec![Some(false), None, None, Some(true)]); + let b = BooleanArray::from(vec![Some(false), None, None, Some(true)]); + test_equal(&a, &b, true); + + let b = BooleanArray::from(vec![None, None, None, Some(true)]); + test_equal(&a, &b, false); + + let b = BooleanArray::from(vec![Some(true), None, None, Some(true)]); + test_equal(&a, &b, false); +} + +#[test] +fn test_boolean_equal_offset() { + let a = BooleanArray::from_slice(vec![false, true, false, true, false, false, true]); + let b = BooleanArray::from_slice(vec![true, false, false, false, true, false, true, true]); + test_equal(&a, &b, false); + + let a_slice = a.slice(2, 3); + let b_slice = b.slice(3, 3); + test_equal(&a_slice, &b_slice, true); + + let a_slice = a.slice(3, 4); + let b_slice = b.slice(4, 4); + test_equal(&a_slice, &b_slice, false); + + // Elements fill in `u8`'s exactly. + let mut vector = vec![false, false, true, true, true, true, true, true]; + let a = BooleanArray::from_slice(vector.clone()); + let b = BooleanArray::from_slice(vector.clone()); + test_equal(&a, &b, true); + + // Elements fill in `u8`s + suffix bits. + vector.push(true); + let a = BooleanArray::from_slice(vector.clone()); + let b = BooleanArray::from_slice(vector); + test_equal(&a, &b, true); +} diff --git a/tests/it/array/equal/dictionary.rs b/tests/it/array/equal/dictionary.rs new file mode 100644 index 00000000000..c93a62a64c6 --- /dev/null +++ b/tests/it/array/equal/dictionary.rs @@ -0,0 +1,58 @@ +use std::sync::Arc; + +use arrow2::array::*; + +use super::test_equal; + +fn create_dictionary_array(values: &[&str], keys: &[Option]) -> DictionaryArray { + let keys = Int16Array::from(keys); + let values = Utf8Array::::from_slice(values); + + DictionaryArray::from_data(keys, Arc::new(values)) +} + +#[test] +fn dictionary_equal() { + // (a, b, c), (0, 1, 0, 2) => (a, b, a, c) + let a = create_dictionary_array(&["a", "b", "c"], &[Some(0), Some(1), Some(0), Some(2)]); + // different representation (values and keys are swapped), same result + let b = create_dictionary_array(&["a", "c", "b"], &[Some(0), Some(2), Some(0), Some(1)]); + test_equal(&a, &b, true); + + // different len + let b = create_dictionary_array(&["a", "c", "b"], &[Some(0), Some(2), Some(1)]); + test_equal(&a, &b, false); + + // different key + let b = create_dictionary_array(&["a", "c", "b"], &[Some(0), Some(2), Some(0), Some(0)]); + test_equal(&a, &b, false); + + // different values, same keys + let b = create_dictionary_array(&["a", "b", "d"], &[Some(0), Some(1), Some(0), Some(2)]); + test_equal(&a, &b, false); +} + +#[test] +fn dictionary_equal_null() { + // (a, b, c), (1, 2, 1, 3) => (a, b, a, c) + let a = create_dictionary_array(&["a", "b", "c"], &[Some(0), None, Some(0), Some(2)]); + + // equal to self + test_equal(&a, &a, true); + + // different representation (values and keys are swapped), same result + let b = create_dictionary_array(&["a", "c", "b"], &[Some(0), None, Some(0), Some(1)]); + test_equal(&a, &b, true); + + // different null position + let b = create_dictionary_array(&["a", "c", "b"], &[Some(0), Some(2), Some(0), None]); + test_equal(&a, &b, false); + + // different key + let b = create_dictionary_array(&["a", "c", "b"], &[Some(0), None, Some(0), Some(0)]); + test_equal(&a, &b, false); + + // different values, same keys + let b = create_dictionary_array(&["a", "b", "d"], &[Some(0), None, Some(0), Some(2)]); + test_equal(&a, &b, false); +} diff --git a/tests/it/array/equal/fixed_size_list.rs b/tests/it/array/equal/fixed_size_list.rs new file mode 100644 index 00000000000..9e26f7523e3 --- /dev/null +++ b/tests/it/array/equal/fixed_size_list.rs @@ -0,0 +1,85 @@ +use arrow2::{ + array::{FixedSizeListArray, MutableFixedSizeListArray, MutablePrimitiveArray}, + datatypes::DataType, +}; + +use super::test_equal; + +/// Create a fixed size list of 2 value lengths +fn create_fixed_size_list_array, T: AsRef<[Option]>>( + data: T, +) -> FixedSizeListArray { + let data = data.as_ref().iter().map(|x| { + Some(match x { + Some(x) => x.as_ref().iter().map(|x| Some(*x)).collect::>(), + None => std::iter::repeat(None).take(3).collect::>(), + }) + }); + + MutableFixedSizeListArray::>::try_from_iter(data, 3, DataType::Int32) + .unwrap() + .into() +} + +#[test] +fn test_fixed_size_list_equal() { + let a = create_fixed_size_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 6])]); + let b = create_fixed_size_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 6])]); + test_equal(&a, &b, true); + + let b = create_fixed_size_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 7])]); + test_equal(&a, &b, false); +} + +// Test the case where null_count > 0 +#[test] +fn test_fixed_list_null() { + let a = + create_fixed_size_list_array(&[Some(&[1, 2, 3]), None, None, Some(&[4, 5, 6]), None, None]); + /* + let b = create_fixed_size_list_array(&[ + Some(&[1, 2, 3]), + None, + None, + Some(&[4, 5, 6]), + None, + None, + ]); + test_equal(&a, &b, true); + + let b = create_fixed_size_list_array(&[ + Some(&[1, 2, 3]), + None, + Some(&[7, 8, 9]), + Some(&[4, 5, 6]), + None, + None, + ]); + test_equal(&a, &b, false); + */ + + let b = + create_fixed_size_list_array(&[Some(&[1, 2, 3]), None, None, Some(&[3, 6, 9]), None, None]); + test_equal(&a, &b, false); +} + +#[test] +fn test_fixed_list_offsets() { + // Test the case where offset != 0 + let a = + create_fixed_size_list_array(&[Some(&[1, 2, 3]), None, None, Some(&[4, 5, 6]), None, None]); + let b = + create_fixed_size_list_array(&[Some(&[1, 2, 3]), None, None, Some(&[3, 6, 9]), None, None]); + + let a_slice = a.slice(0, 3); + let b_slice = b.slice(0, 3); + test_equal(&a_slice, &b_slice, true); + + let a_slice = a.slice(0, 5); + let b_slice = b.slice(0, 5); + test_equal(&a_slice, &b_slice, false); + + let a_slice = a.slice(4, 1); + let b_slice = b.slice(4, 1); + test_equal(&a_slice, &b_slice, true); +} diff --git a/tests/it/array/equal/list.rs b/tests/it/array/equal/list.rs new file mode 100644 index 00000000000..d0fa8098b74 --- /dev/null +++ b/tests/it/array/equal/list.rs @@ -0,0 +1,93 @@ +use std::sync::Arc; + +use arrow2::array::{Int32Array, ListArray, MutableListArray, MutablePrimitiveArray, TryExtend}; +use arrow2::bitmap::Bitmap; +use arrow2::buffer::Buffer; +use arrow2::datatypes::DataType; + +use super::test_equal; + +fn create_list_array, T: AsRef<[Option]>>(data: T) -> ListArray { + let iter = data.as_ref().iter().map(|x| { + x.as_ref() + .map(|x| x.as_ref().iter().map(|x| Some(*x)).collect::>()) + }); + let mut array = MutableListArray::>::new(); + array.try_extend(iter).unwrap(); + array.into() +} + +#[test] +fn test_list_equal() { + let a = create_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 6])]); + let b = create_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 6])]); + test_equal(&a, &b, true); + + let b = create_list_array(&[Some(&[1, 2, 3]), Some(&[4, 5, 7])]); + test_equal(&a, &b, false); +} + +// Test the case where null_count > 0 +#[test] +fn test_list_null() { + let a = create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 4]), None, None]); + let b = create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 4]), None, None]); + test_equal(&a, &b, true); + + let b = create_list_array(&[ + Some(&[1, 2]), + None, + Some(&[5, 6]), + Some(&[3, 4]), + None, + None, + ]); + test_equal(&a, &b, false); + + let b = create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 5]), None, None]); + test_equal(&a, &b, false); +} + +// Test the case where offset != 0 +#[test] +fn test_list_offsets() { + let a = create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 4]), None, None]); + let b = create_list_array(&[Some(&[1, 2]), None, None, Some(&[3, 5]), None, None]); + + let a_slice = a.slice(0, 3); + let b_slice = b.slice(0, 3); + test_equal(&a_slice, &b_slice, true); + + let a_slice = a.slice(0, 5); + let b_slice = b.slice(0, 5); + test_equal(&a_slice, &b_slice, false); + + let a_slice = a.slice(4, 1); + let b_slice = b.slice(4, 1); + test_equal(&a_slice, &b_slice, true); +} + +#[test] +fn test_bla() { + let offsets = Buffer::from([0, 3, 3, 6]); + let data_type = ListArray::::default_datatype(DataType::Int32); + let values = Arc::new(Int32Array::from([ + Some(1), + Some(2), + Some(3), + Some(4), + None, + Some(6), + ])); + let validity = Bitmap::from([true, false, true]); + let lhs = ListArray::::from_data(data_type, offsets, values, Some(validity)); + let lhs = lhs.slice(1, 2); + + let offsets = Buffer::from([0, 0, 3]); + let data_type = ListArray::::default_datatype(DataType::Int32); + let values = Arc::new(Int32Array::from([Some(4), None, Some(6)])); + let validity = Bitmap::from([false, true]); + let rhs = ListArray::::from_data(data_type, offsets, values, Some(validity)); + + assert_eq!(lhs, rhs); +} diff --git a/tests/it/array/equal/mod.rs b/tests/it/array/equal/mod.rs new file mode 100644 index 00000000000..a5d1884cf24 --- /dev/null +++ b/tests/it/array/equal/mod.rs @@ -0,0 +1,50 @@ +use arrow2::array::*; + +mod dictionary; +mod fixed_size_list; +mod list; +mod primitive; +mod utf8; + +pub fn test_equal(lhs: &dyn Array, rhs: &dyn Array, expected: bool) { + // equality is symmetric + assert!(equal(lhs, lhs), "\n{:?}\n{:?}", lhs, lhs); + assert!(equal(rhs, rhs), "\n{:?}\n{:?}", rhs, rhs); + + assert_eq!(equal(lhs, rhs), expected, "\n{:?}\n{:?}", lhs, rhs); + assert_eq!(equal(rhs, lhs), expected, "\n{:?}\n{:?}", rhs, lhs); +} + +#[allow(clippy::type_complexity)] +fn binary_cases() -> Vec<(Vec>, Vec>, bool)> { + let base = vec![ + Some("hello".to_owned()), + None, + None, + Some("world".to_owned()), + None, + None, + ]; + let not_base = vec![ + Some("hello".to_owned()), + Some("foo".to_owned()), + None, + Some("world".to_owned()), + None, + None, + ]; + vec![ + ( + vec![Some("hello".to_owned()), Some("world".to_owned())], + vec![Some("hello".to_owned()), Some("world".to_owned())], + true, + ), + ( + vec![Some("hello".to_owned()), Some("world".to_owned())], + vec![Some("hello".to_owned()), Some("arrow".to_owned())], + false, + ), + (base.clone(), base.clone(), true), + (base, not_base, false), + ] +} diff --git a/tests/it/array/equal/primitive.rs b/tests/it/array/equal/primitive.rs new file mode 100644 index 00000000000..cee9bed9b62 --- /dev/null +++ b/tests/it/array/equal/primitive.rs @@ -0,0 +1,90 @@ +use arrow2::array::*; + +use super::test_equal; + +#[test] +fn test_primitive() { + let cases = vec![ + ( + vec![Some(1), Some(2), Some(3)], + vec![Some(1), Some(2), Some(3)], + true, + ), + ( + vec![Some(1), Some(2), Some(3)], + vec![Some(1), Some(2), Some(4)], + false, + ), + ( + vec![Some(1), Some(2), None], + vec![Some(1), Some(2), None], + true, + ), + ( + vec![Some(1), None, Some(3)], + vec![Some(1), Some(2), None], + false, + ), + ( + vec![Some(1), None, None], + vec![Some(1), Some(2), None], + false, + ), + ]; + + for (lhs, rhs, expected) in cases { + let lhs = Int32Array::from(&lhs); + let rhs = Int32Array::from(&rhs); + test_equal(&lhs, &rhs, expected); + } +} + +#[test] +fn test_primitive_slice() { + let cases = vec![ + ( + vec![Some(1), Some(2), Some(3)], + (0, 1), + vec![Some(1), Some(2), Some(3)], + (0, 1), + true, + ), + ( + vec![Some(1), Some(2), Some(3)], + (1, 1), + vec![Some(1), Some(2), Some(3)], + (2, 1), + false, + ), + ( + vec![Some(1), Some(2), None], + (1, 1), + vec![Some(1), None, Some(2)], + (2, 1), + true, + ), + ( + vec![None, Some(2), None], + (1, 1), + vec![None, None, Some(2)], + (2, 1), + true, + ), + ( + vec![Some(1), None, Some(2), None, Some(3)], + (2, 2), + vec![None, Some(2), None, Some(3)], + (1, 2), + true, + ), + ]; + + for (lhs, slice_lhs, rhs, slice_rhs, expected) in cases { + let lhs = Int32Array::from(&lhs); + let lhs = lhs.slice(slice_lhs.0, slice_lhs.1); + let rhs = Int32Array::from(&rhs); + let rhs = rhs.slice(slice_rhs.0, slice_rhs.1); + + test_equal(&lhs, &rhs, expected); + } +} diff --git a/tests/it/array/equal/utf8.rs b/tests/it/array/equal/utf8.rs new file mode 100644 index 00000000000..2be9ebef83c --- /dev/null +++ b/tests/it/array/equal/utf8.rs @@ -0,0 +1,25 @@ +use arrow2::array::*; + +use super::{binary_cases, test_equal}; + +fn test_generic_string_equal() { + let cases = binary_cases(); + + for (lhs, rhs, expected) in cases { + let lhs = lhs.iter().map(|x| x.as_deref()).collect::>(); + let rhs = rhs.iter().map(|x| x.as_deref()).collect::>(); + let lhs = Utf8Array::::from(&lhs); + let rhs = Utf8Array::::from(&rhs); + test_equal(&lhs, &rhs, expected); + } +} + +#[test] +fn utf8_equal() { + test_generic_string_equal::() +} + +#[test] +fn large_utf8_equal() { + test_generic_string_equal::() +} diff --git a/tests/it/array/fixed_size_binary/mod.rs b/tests/it/array/fixed_size_binary/mod.rs new file mode 100644 index 00000000000..f7a9b61988d --- /dev/null +++ b/tests/it/array/fixed_size_binary/mod.rs @@ -0,0 +1 @@ +mod mutable; diff --git a/tests/it/array/fixed_size_binary/mutable.rs b/tests/it/array/fixed_size_binary/mutable.rs new file mode 100644 index 00000000000..064fcadddac --- /dev/null +++ b/tests/it/array/fixed_size_binary/mutable.rs @@ -0,0 +1,21 @@ +use arrow2::array::*; +use arrow2::bitmap::Bitmap; + +#[test] +fn basic() { + let array = MutableFixedSizeBinaryArray::try_from_iter( + vec![Some(b"ab"), Some(b"bc"), None, Some(b"fh")], + 2, + ) + .unwrap(); + assert_eq!(array.len(), 4); +} + +#[test] +fn push_null() { + let mut array = MutableFixedSizeBinaryArray::new(2); + array.push::<&[u8]>(None); + + let array: FixedSizeBinaryArray = array.into(); + assert_eq!(array.validity(), &Some(Bitmap::from([false]))); +} diff --git a/tests/it/array/fixed_size_list/mod.rs b/tests/it/array/fixed_size_list/mod.rs new file mode 100644 index 00000000000..f7a9b61988d --- /dev/null +++ b/tests/it/array/fixed_size_list/mod.rs @@ -0,0 +1 @@ +mod mutable; diff --git a/tests/it/array/fixed_size_list/mutable.rs b/tests/it/array/fixed_size_list/mutable.rs new file mode 100644 index 00000000000..31e2f11c164 --- /dev/null +++ b/tests/it/array/fixed_size_list/mutable.rs @@ -0,0 +1,32 @@ +use arrow2::array::*; +use arrow2::datatypes::DataType; + +#[test] +fn primitive() { + let data = vec![ + Some(vec![Some(1i32), Some(2), Some(3)]), + Some(vec![None, None, None]), + Some(vec![Some(4), None, Some(6)]), + ]; + + let list: FixedSizeListArray = + MutableFixedSizeListArray::>::try_from_iter( + data, + 3, + DataType::Int32, + ) + .unwrap() + .into(); + + let a = list.value(0); + let a = a.as_any().downcast_ref::().unwrap(); + + let expected = Int32Array::from(vec![Some(1i32), Some(2), Some(3)]); + assert_eq!(a, &expected); + + let a = list.value(1); + let a = a.as_any().downcast_ref::().unwrap(); + + let expected = Int32Array::from(vec![None, None, None]); + assert_eq!(a, &expected) +} diff --git a/tests/it/array/growable/binary.rs b/tests/it/array/growable/binary.rs new file mode 100644 index 00000000000..4451e141c9a --- /dev/null +++ b/tests/it/array/growable/binary.rs @@ -0,0 +1,82 @@ +use arrow2::array::{ + growable::{Growable, GrowableBinary}, + BinaryArray, +}; + +#[test] +fn no_offsets() { + let array = BinaryArray::::from(&[Some("a"), Some("bc"), None, Some("defh")]); + + let mut a = GrowableBinary::new(vec![&array], false, 0); + + a.extend(0, 1, 2); + + let result: BinaryArray = a.into(); + + let expected = BinaryArray::::from(&[Some("bc"), None]); + assert_eq!(result, expected); +} + +/// tests extending from a variable-sized (strings and binary) array +/// with an offset and nulls +#[test] +fn with_offsets() { + let array = BinaryArray::::from(&[Some("a"), Some("bc"), None, Some("defh")]); + let array = array.slice(1, 3); + + let mut a = GrowableBinary::new(vec![&array], false, 0); + + a.extend(0, 0, 3); + + let result: BinaryArray = a.into(); + + let expected = BinaryArray::::from(&[Some("bc"), None, Some("defh")]); + assert_eq!(result, expected); +} + +#[test] +fn test_string_offsets() { + let array = BinaryArray::::from(&[Some("a"), Some("bc"), None, Some("defh")]); + let array = array.slice(1, 3); + + let mut a = GrowableBinary::new(vec![&array], false, 0); + + a.extend(0, 0, 3); + + let result: BinaryArray = a.into(); + + let expected = BinaryArray::::from(&[Some("bc"), None, Some("defh")]); + assert_eq!(result, expected); +} + +#[test] +fn test_multiple_with_validity() { + let array1 = BinaryArray::::from_slice(&[b"hello", b"world"]); + let array2 = BinaryArray::::from(&[Some("1"), None]); + + let mut a = GrowableBinary::new(vec![&array1, &array2], false, 5); + + a.extend(0, 0, 2); + a.extend(1, 0, 2); + + let result: BinaryArray = a.into(); + + let expected = BinaryArray::::from(&[Some("hello"), Some("world"), Some("1"), None]); + assert_eq!(result, expected); +} + +#[test] +fn test_string_null_offset_validity() { + let array = BinaryArray::::from(&[Some("a"), Some("bc"), None, Some("defh")]); + let array = array.slice(1, 3); + + let mut a = GrowableBinary::new(vec![&array], true, 0); + + a.extend(0, 1, 2); + a.extend_validity(1); + + let result: BinaryArray = a.into(); + + let expected = BinaryArray::::from(&[None, Some("defh"), None]); + assert_eq!(result, expected); +} diff --git a/tests/it/array/growable/boolean.rs b/tests/it/array/growable/boolean.rs new file mode 100644 index 00000000000..780c677cce4 --- /dev/null +++ b/tests/it/array/growable/boolean.rs @@ -0,0 +1,16 @@ +use arrow2::array::growable::{Growable, GrowableBoolean}; +use arrow2::array::BooleanArray; + +#[test] +fn test_bool() { + let array = BooleanArray::from(vec![Some(false), Some(true), None, Some(false)]); + + let mut a = GrowableBoolean::new(vec![&array], false, 0); + + a.extend(0, 1, 2); + + let result: BooleanArray = a.into(); + + let expected = BooleanArray::from(vec![Some(true), None]); + assert_eq!(result, expected); +} diff --git a/tests/it/array/growable/dictionary.rs b/tests/it/array/growable/dictionary.rs new file mode 100644 index 00000000000..61bacafd9be --- /dev/null +++ b/tests/it/array/growable/dictionary.rs @@ -0,0 +1,64 @@ +use std::sync::Arc; + +use arrow2::array::growable::{Growable, GrowableDictionary}; +use arrow2::array::*; +use arrow2::error::Result; + +#[test] +fn test_single() -> Result<()> { + let original_data = vec![Some("a"), Some("b"), Some("a")]; + + let data = original_data.clone(); + let mut array = MutableDictionaryArray::>::new(); + array.try_extend(data)?; + let array = array.into(); + + // same values, less keys + let expected = DictionaryArray::::from_data( + PrimitiveArray::from(vec![Some(1), Some(0)]), + Arc::new(Utf8Array::::from(&original_data)), + ); + + let mut growable = GrowableDictionary::new(&[&array], false, 0); + + growable.extend(0, 1, 2); + + let result: DictionaryArray = growable.into(); + + assert_eq!(result, expected); + Ok(()) +} + +#[test] +fn test_multi() -> Result<()> { + let mut original_data1 = vec![Some("a"), Some("b"), None, Some("a")]; + let original_data2 = vec![Some("c"), Some("b"), None, Some("a")]; + + let data1 = original_data1.clone(); + let data2 = original_data2.clone(); + + let mut array1 = MutableDictionaryArray::>::new(); + array1.try_extend(data1)?; + let array1: DictionaryArray = array1.into(); + + let mut array2 = MutableDictionaryArray::>::new(); + array2.try_extend(data2)?; + let array2: DictionaryArray = array2.into(); + + // same values, less keys + original_data1.extend(original_data2.iter().cloned()); + let expected = DictionaryArray::::from_data( + PrimitiveArray::from(vec![Some(1), None, Some(3), None]), + Arc::new(Utf8Array::::from_slice(&["a", "b", "c", "b", "a"])), + ); + + let mut growable = GrowableDictionary::new(&[&array1, &array2], false, 0); + + growable.extend(0, 1, 2); + growable.extend(1, 1, 2); + + let result: DictionaryArray = growable.into(); + + assert_eq!(result, expected); + Ok(()) +} diff --git a/tests/it/array/growable/fixed_binary.rs b/tests/it/array/growable/fixed_binary.rs new file mode 100644 index 00000000000..e5936a1f096 --- /dev/null +++ b/tests/it/array/growable/fixed_binary.rs @@ -0,0 +1,89 @@ +use arrow2::array::{ + growable::{Growable, GrowableFixedSizeBinary}, + FixedSizeBinaryArray, +}; + +/// tests extending from a variable-sized (strings and binary) array w/ offset with nulls +#[test] +fn basic() { + let array = + FixedSizeBinaryArray::from_iter(vec![Some(b"ab"), Some(b"bc"), None, Some(b"de")], 2); + + let mut a = GrowableFixedSizeBinary::new(vec![&array], false, 0); + + a.extend(0, 1, 2); + + let result: FixedSizeBinaryArray = a.into(); + + let expected = FixedSizeBinaryArray::from_iter(vec![Some("bc"), None], 2); + assert_eq!(result, expected); +} + +/// tests extending from a variable-sized (strings and binary) array +/// with an offset and nulls +#[test] +fn offsets() { + let array = + FixedSizeBinaryArray::from_iter(vec![Some(b"ab"), Some(b"bc"), None, Some(b"fh")], 2); + let array = array.slice(1, 3); + + let mut a = GrowableFixedSizeBinary::new(vec![&array], false, 0); + + a.extend(0, 0, 3); + + let result: FixedSizeBinaryArray = a.into(); + + let expected = FixedSizeBinaryArray::from_iter(vec![Some(b"bc"), None, Some(b"fh")], 2); + assert_eq!(result, expected); +} + +#[test] +fn multiple_with_validity() { + let array1 = FixedSizeBinaryArray::from_iter(vec![Some("hello"), Some("world")], 5); + let array2 = FixedSizeBinaryArray::from_iter(vec![Some("12345"), None], 5); + + let mut a = GrowableFixedSizeBinary::new(vec![&array1, &array2], false, 5); + + a.extend(0, 0, 2); + a.extend(1, 0, 2); + + let result: FixedSizeBinaryArray = a.into(); + + let expected = + FixedSizeBinaryArray::from_iter(vec![Some("hello"), Some("world"), Some("12345"), None], 5); + assert_eq!(result, expected); +} + +#[test] +fn null_offset_validity() { + let array = FixedSizeBinaryArray::from_iter(vec![Some("aa"), Some("bc"), None, Some("fh")], 2); + let array = array.slice(1, 3); + + let mut a = GrowableFixedSizeBinary::new(vec![&array], true, 0); + + a.extend(0, 1, 2); + a.extend_validity(1); + + let result: FixedSizeBinaryArray = a.into(); + + let expected = FixedSizeBinaryArray::from_iter(vec![None, Some("fh"), None], 2); + assert_eq!(result, expected); +} + +#[test] +fn sized_offsets() { + let array = + FixedSizeBinaryArray::from_iter(vec![Some(&[0, 0]), Some(&[0, 1]), Some(&[0, 2])], 2); + let array = array.slice(1, 2); + // = [[0, 1], [0, 2]] due to the offset = 1 + + let mut a = GrowableFixedSizeBinary::new(vec![&array], false, 0); + + a.extend(0, 1, 1); + a.extend(0, 0, 1); + + let result: FixedSizeBinaryArray = a.into(); + + let expected = FixedSizeBinaryArray::from_iter(vec![Some(&[0, 2]), Some(&[0, 1])], 2); + assert_eq!(result, expected); +} diff --git a/tests/it/array/growable/list.rs b/tests/it/array/growable/list.rs new file mode 100644 index 00000000000..ef183654579 --- /dev/null +++ b/tests/it/array/growable/list.rs @@ -0,0 +1,105 @@ +use arrow2::array::{ + growable::{Growable, GrowableList}, + ListArray, MutableListArray, MutablePrimitiveArray, TryExtend, +}; + +fn create_list_array(data: Vec>>>) -> ListArray { + let mut array = MutableListArray::>::new(); + array.try_extend(data).unwrap(); + array.into() +} + +#[test] +fn basic() { + let data = vec![ + Some(vec![Some(1i32), Some(2), Some(3)]), + Some(vec![Some(4), Some(5)]), + Some(vec![Some(6i32), Some(7), Some(8)]), + ]; + + let array = create_list_array(data); + + let mut a = GrowableList::new(vec![&array], false, 0); + a.extend(0, 0, 1); + + let result: ListArray = a.into(); + + let expected = vec![Some(vec![Some(1i32), Some(2), Some(3)])]; + let expected = create_list_array(expected); + + assert_eq!(result, expected) +} + +#[test] +fn null_offset() { + let data = vec![ + Some(vec![Some(1i32), Some(2), Some(3)]), + None, + Some(vec![Some(6i32), Some(7), Some(8)]), + ]; + let array = create_list_array(data); + let array = array.slice(1, 2); + + let mut a = GrowableList::new(vec![&array], false, 0); + a.extend(0, 1, 1); + + let result: ListArray = a.into(); + + let expected = vec![Some(vec![Some(6i32), Some(7), Some(8)])]; + let expected = create_list_array(expected); + + assert_eq!(result, expected) +} + +#[test] +fn null_offsets() { + let data = vec![ + Some(vec![Some(1i32), Some(2), Some(3)]), + None, + Some(vec![Some(6i32), None, Some(8)]), + ]; + let array = create_list_array(data); + let array = array.slice(1, 2); + + let mut a = GrowableList::new(vec![&array], false, 0); + a.extend(0, 1, 1); + + let result: ListArray = a.into(); + + let expected = vec![Some(vec![Some(6i32), None, Some(8)])]; + let expected = create_list_array(expected); + + assert_eq!(result, expected) +} + +#[test] +fn test_from_two_lists() { + let data_1 = vec![ + Some(vec![Some(1i32), Some(2), Some(3)]), + None, + Some(vec![Some(6i32), None, Some(8)]), + ]; + let array_1 = create_list_array(data_1); + + let data_2 = vec![ + Some(vec![Some(8i32), Some(7), Some(6)]), + Some(vec![Some(5i32), None, Some(4)]), + Some(vec![Some(2i32), Some(1), Some(0)]), + ]; + let array_2 = create_list_array(data_2); + + let mut a = GrowableList::new(vec![&array_1, &array_2], false, 6); + a.extend(0, 0, 2); + a.extend(1, 1, 1); + + let result: ListArray = a.into(); + + let expected = vec![ + Some(vec![Some(1i32), Some(2), Some(3)]), + None, + Some(vec![Some(5i32), None, Some(4)]), + ]; + let expected = create_list_array(expected); + + assert_eq!(result, expected); +} diff --git a/tests/it/array/growable/mod.rs b/tests/it/array/growable/mod.rs new file mode 100644 index 00000000000..48ac9b440e0 --- /dev/null +++ b/tests/it/array/growable/mod.rs @@ -0,0 +1,138 @@ +mod binary; +mod boolean; +mod dictionary; +mod fixed_binary; +mod list; +mod null; +mod primitive; +mod struct_; +mod utf8; + +/* +#[cfg(test)] +mod tests { + use std::convert::TryFrom; + + use super::*; + + use crate::{ + array::{ + Array, ArrayDataRef, ArrayRef, BooleanArray, DictionaryArray, + FixedSizeBinaryArray, Int16Array, Int16Type, Int32Array, Int64Array, + Int64Builder, ListBuilder, NullArray, PrimitiveBuilder, StringArray, + StringDictionaryBuilder, StructArray, UInt8Array, + }, + buffer::Buffer, + datatypes::Field, + }; + use crate::{ + array::{ListArray, StringBuilder}, + error::Result, + }; + + fn create_dictionary_array(values: &[&str], keys: &[Option<&str>]) -> ArrayDataRef { + let values = StringArray::from(values.to_vec()); + let mut builder = StringDictionaryBuilder::new_with_dictionary( + PrimitiveBuilder::::new(3), + &values, + ) + .unwrap(); + for key in keys { + if let Some(v) = key { + builder.append(v).unwrap(); + } else { + builder.append_null().unwrap() + } + } + builder.finish().data() + } + + /* + // this is an old test used on a meanwhile removed dead code + // that is still useful when `MutableArrayData` supports fixed-size lists. + #[test] + fn test_fixed_size_list_append() -> Result<()> { + let int_builder = UInt16Builder::new(64); + let mut builder = FixedSizeListBuilder::::new(int_builder, 2); + builder.values().append_slice(&[1, 2])?; + builder.append(true)?; + builder.values().append_slice(&[3, 4])?; + builder.append(false)?; + builder.values().append_slice(&[5, 6])?; + builder.append(true)?; + + let a_builder = UInt16Builder::new(64); + let mut a_builder = FixedSizeListBuilder::::new(a_builder, 2); + a_builder.values().append_slice(&[7, 8])?; + a_builder.append(true)?; + a_builder.values().append_slice(&[9, 10])?; + a_builder.append(true)?; + a_builder.values().append_slice(&[11, 12])?; + a_builder.append(false)?; + a_builder.values().append_slice(&[13, 14])?; + a_builder.append(true)?; + a_builder.values().append_null()?; + a_builder.values().append_null()?; + a_builder.append(true)?; + let a = a_builder.finish(); + + // append array + builder.append_data(&[ + a.data(), + a.slice(1, 3).data(), + a.slice(2, 1).data(), + a.slice(5, 0).data(), + ])?; + let finished = builder.finish(); + + let expected_int_array = UInt16Array::from(vec![ + Some(1), + Some(2), + Some(3), + Some(4), + Some(5), + Some(6), + // append first array + Some(7), + Some(8), + Some(9), + Some(10), + Some(11), + Some(12), + Some(13), + Some(14), + None, + None, + // append slice(1, 3) + Some(9), + Some(10), + Some(11), + Some(12), + Some(13), + Some(14), + // append slice(2, 1) + Some(11), + Some(12), + ]); + let expected_list_data = ArrayData::new( + DataType::FixedSizeList( + Box::new(Field::new("item", DataType::UInt16, true)), + 2, + ), + 12, + None, + None, + 0, + vec![], + vec![expected_int_array.data()], + ); + let expected_list = + FixedSizeListArray::from(Arc::new(expected_list_data) as ArrayDataRef); + assert_eq!(&expected_list.values(), &finished.values()); + assert_eq!(expected_list.len(), finished.len()); + + Ok(()) + } + */ +} +*/ diff --git a/tests/it/array/growable/null.rs b/tests/it/array/growable/null.rs new file mode 100644 index 00000000000..934f6a24c93 --- /dev/null +++ b/tests/it/array/growable/null.rs @@ -0,0 +1,15 @@ +use arrow2::array::growable::{Growable, GrowableNull}; +use arrow2::array::*; + +#[test] +fn null() { + let mut mutable = GrowableNull::new(); + + mutable.extend(0, 1, 2); + mutable.extend(1, 0, 1); + + let result: NullArray = mutable.into(); + + let expected = NullArray::from_data(3); + assert_eq!(result, expected); +} diff --git a/tests/it/array/growable/primitive.rs b/tests/it/array/growable/primitive.rs new file mode 100644 index 00000000000..aec18582063 --- /dev/null +++ b/tests/it/array/growable/primitive.rs @@ -0,0 +1,65 @@ +use arrow2::array::{ + growable::{Growable, GrowablePrimitive}, + PrimitiveArray, +}; + +/// tests extending from a primitive array w/ offset nor nulls +#[test] +fn basics() { + let b = PrimitiveArray::::from(vec![Some(1), Some(2), Some(3)]); + let mut a = GrowablePrimitive::new(vec![&b], false, 3); + a.extend(0, 0, 2); + let result: PrimitiveArray = a.into(); + let expected = PrimitiveArray::::from(vec![Some(1), Some(2)]); + assert_eq!(result, expected); +} + +/// tests extending from a primitive array with offset w/ nulls +#[test] +fn offset() { + let b = PrimitiveArray::::from(vec![Some(1), Some(2), Some(3)]); + let b = b.slice(1, 2); + let mut a = GrowablePrimitive::new(vec![&b], false, 2); + a.extend(0, 0, 2); + let result: PrimitiveArray = a.into(); + let expected = PrimitiveArray::::from(vec![Some(2), Some(3)]); + assert_eq!(result, expected); +} + +/// tests extending from a primitive array with offset and nulls +#[test] +fn null_offset() { + let b = PrimitiveArray::::from(vec![Some(1), None, Some(3)]); + let b = b.slice(1, 2); + let mut a = GrowablePrimitive::new(vec![&b], false, 2); + a.extend(0, 0, 2); + let result: PrimitiveArray = a.into(); + let expected = PrimitiveArray::::from(vec![None, Some(3)]); + assert_eq!(result, expected); +} + +#[test] +fn null_offset_validity() { + let b = PrimitiveArray::::from(&[Some(1), Some(2), Some(3)]); + let b = b.slice(1, 2); + let mut a = GrowablePrimitive::new(vec![&b], true, 2); + a.extend(0, 0, 2); + a.extend_validity(3); + a.extend(0, 1, 1); + let result: PrimitiveArray = a.into(); + let expected = PrimitiveArray::::from(&[Some(2), Some(3), None, None, None, Some(3)]); + assert_eq!(result, expected); +} + +#[test] +fn joining_arrays() { + let b = PrimitiveArray::::from(&[Some(1), Some(2), Some(3)]); + let c = PrimitiveArray::::from(&[Some(4), Some(5), Some(6)]); + let mut a = GrowablePrimitive::new(vec![&b, &c], false, 4); + a.extend(0, 0, 2); + a.extend(1, 1, 2); + let result: PrimitiveArray = a.into(); + + let expected = PrimitiveArray::::from(&[Some(1), Some(2), Some(5), Some(6)]); + assert_eq!(result, expected); +} diff --git a/tests/it/array/growable/struct_.rs b/tests/it/array/growable/struct_.rs new file mode 100644 index 00000000000..ed5e715c46e --- /dev/null +++ b/tests/it/array/growable/struct_.rs @@ -0,0 +1,119 @@ +use std::sync::Arc; + +use arrow2::array::{ + growable::{Growable, GrowableStruct}, + Array, PrimitiveArray, StructArray, Utf8Array, +}; +use arrow2::bitmap::Bitmap; +use arrow2::datatypes::{DataType, Field}; + +fn some_values() -> (Vec, Vec>) { + let strings: Arc = Arc::new(Utf8Array::::from(&[ + Some("a"), + Some("aa"), + None, + Some("mark"), + Some("doe"), + ])); + let ints: Arc = Arc::new(PrimitiveArray::::from(&[ + Some(1), + Some(2), + Some(3), + Some(4), + Some(5), + ])); + let fields = vec![ + Field::new("f1", DataType::Utf8, true), + Field::new("f2", DataType::Int32, true), + ]; + (fields, vec![strings, ints]) +} + +#[test] +fn basic() { + let (fields, values) = some_values(); + + let array = StructArray::from_data(fields.clone(), values.clone(), None); + + let mut a = GrowableStruct::new(vec![&array], false, 0); + + a.extend(0, 1, 2); + let result: StructArray = a.into(); + + let expected = StructArray::from_data( + fields, + vec![values[0].slice(1, 2).into(), values[1].slice(1, 2).into()], + None, + ); + assert_eq!(result, expected) +} + +#[test] +fn offset() { + let (fields, values) = some_values(); + + let array = StructArray::from_data(fields.clone(), values.clone(), None).slice(1, 3); + + let mut a = GrowableStruct::new(vec![&array], false, 0); + + a.extend(0, 1, 2); + let result: StructArray = a.into(); + + let expected = StructArray::from_data( + fields, + vec![values[0].slice(2, 2).into(), values[1].slice(2, 2).into()], + None, + ); + + assert_eq!(result, expected); +} + +#[test] +fn nulls() { + let (fields, values) = some_values(); + + let array = StructArray::from_data( + fields.clone(), + values.clone(), + Some(Bitmap::from_u8_slice(&[0b00000010], 5)), + ); + + let mut a = GrowableStruct::new(vec![&array], false, 0); + + a.extend(0, 1, 2); + let result: StructArray = a.into(); + + let expected = StructArray::from_data( + fields, + vec![values[0].slice(1, 2).into(), values[1].slice(1, 2).into()], + Some(Bitmap::from_u8_slice(&[0b00000010], 5).slice(1, 2)), + ); + + assert_eq!(result, expected) +} + +#[test] +fn many() { + let (fields, values) = some_values(); + + let array = StructArray::from_data(fields.clone(), values.clone(), None); + + let mut mutable = GrowableStruct::new(vec![&array, &array], false, 0); + + mutable.extend(0, 1, 2); + mutable.extend(1, 0, 2); + let result: StructArray = mutable.into(); + + let expected_string: Arc = Arc::new(Utf8Array::::from(&[ + Some("aa"), + None, + Some("a"), + Some("aa"), + ])); + let expected_int: Arc = Arc::new( + PrimitiveArray::::from(vec![Some(2), Some(3), Some(1), Some(2)]).to(DataType::Int32), + ); + + let expected = StructArray::from_data(fields, vec![expected_string, expected_int], None); + assert_eq!(result, expected) +} diff --git a/tests/it/array/growable/utf8.rs b/tests/it/array/growable/utf8.rs new file mode 100644 index 00000000000..feb4465a6f8 --- /dev/null +++ b/tests/it/array/growable/utf8.rs @@ -0,0 +1,83 @@ +use arrow2::array::{ + growable::{Growable, GrowableUtf8}, + Utf8Array, +}; + +/// tests extending from a variable-sized (strings and binary) array w/ offset with nulls +#[test] +fn validity() { + let array = Utf8Array::::from(&[Some("a"), Some("bc"), None, Some("defh")]); + + let mut a = GrowableUtf8::new(vec![&array], false, 0); + + a.extend(0, 1, 2); + + let result: Utf8Array = a.into(); + + let expected = Utf8Array::::from(&[Some("bc"), None]); + assert_eq!(result, expected); +} + +/// tests extending from a variable-sized (strings and binary) array +/// with an offset and nulls +#[test] +fn offsets() { + let array = Utf8Array::::from(&[Some("a"), Some("bc"), None, Some("defh")]); + let array = array.slice(1, 3); + + let mut a = GrowableUtf8::new(vec![&array], false, 0); + + a.extend(0, 0, 3); + + let result: Utf8Array = a.into(); + + let expected = Utf8Array::::from(&[Some("bc"), None, Some("defh")]); + assert_eq!(result, expected); +} + +#[test] +fn offsets2() { + let array = Utf8Array::::from(&[Some("a"), Some("bc"), None, Some("defh")]); + let array = array.slice(1, 3); + + let mut a = GrowableUtf8::new(vec![&array], false, 0); + + a.extend(0, 0, 3); + + let result: Utf8Array = a.into(); + + let expected = Utf8Array::::from(&[Some("bc"), None, Some("defh")]); + assert_eq!(result, expected); +} + +#[test] +fn multiple_with_validity() { + let array1 = Utf8Array::::from_slice(&["hello", "world"]); + let array2 = Utf8Array::::from(&[Some("1"), None]); + + let mut a = GrowableUtf8::new(vec![&array1, &array2], false, 5); + + a.extend(0, 0, 2); + a.extend(1, 0, 2); + + let result: Utf8Array = a.into(); + + let expected = Utf8Array::::from(&[Some("hello"), Some("world"), Some("1"), None]); + assert_eq!(result, expected); +} + +#[test] +fn null_offset_validity() { + let array = Utf8Array::::from(&[Some("a"), Some("bc"), None, Some("defh")]); + let array = array.slice(1, 3); + + let mut a = GrowableUtf8::new(vec![&array], true, 0); + + a.extend(0, 1, 2); + a.extend_validity(1); + + let result: Utf8Array = a.into(); + + let expected = Utf8Array::::from(&[None, Some("defh"), None]); + assert_eq!(result, expected); +} diff --git a/tests/it/array/growable/utils.rs b/tests/it/array/growable/utils.rs new file mode 100644 index 00000000000..1b6f5ff55e1 --- /dev/null +++ b/tests/it/array/growable/utils.rs @@ -0,0 +1,75 @@ +use crate::{ + array::{Array, Offset}, + bitmap::{Bitmap, MutableBitmap}, + buffer::MutableBuffer, +}; + +pub(super) fn extend_offsets( + buffer: &mut MutableBuffer, + last_offset: &mut T, + offsets: &[T], +) { + buffer.reserve(offsets.len() - 1); + offsets.windows(2).for_each(|offsets| { + // compute the new offset + let length = offsets[1] - offsets[0]; + *last_offset += length; + buffer.push(*last_offset); + }); +} + +pub(super) type ExtendNullBits<'a> = Box; + +pub(super) fn build_extend_null_bits(array: &dyn Array, use_validity: bool) -> ExtendNullBits { + if let Some(bitmap) = array.validity() { + Box::new(move |validity, start, len| { + assert!(start + len <= bitmap.len()); + unsafe { + let iter = (start..start + len).map(|i| bitmap.get_bit_unchecked(i)); + validity.extend_from_trusted_len_iter_unchecked(iter); + }; + }) + } else if use_validity { + Box::new(|validity, _, len| { + let iter = (0..len).map(|_| true); + unsafe { + validity.extend_from_trusted_len_iter_unchecked(iter); + }; + }) + } else { + Box::new(|_, _, _| {}) + } +} + +#[inline] +pub(super) fn extend_validity( + mutable_validity: &mut MutableBitmap, + validity: &Option, + start: usize, + len: usize, + use_validity: bool, +) { + if let Some(bitmap) = validity { + assert!(start + len <= bitmap.len()); + unsafe { + let iter = (start..start + len).map(|i| bitmap.get_bit_unchecked(i)); + mutable_validity.extend_from_trusted_len_iter_unchecked(iter); + }; + } else if use_validity { + mutable_validity.extend_constant(len, true); + }; +} + +#[inline] +pub(super) fn extend_offset_values( + buffer: &mut MutableBuffer, + offsets: &[O], + values: &[u8], + start: usize, + len: usize, +) { + let start_values = offsets[start].to_usize(); + let end_values = offsets[start + len].to_usize(); + let new_values = &values[start_values..end_values]; + buffer.extend_from_slice(new_values); +} diff --git a/tests/it/array/list/mod.rs b/tests/it/array/list/mod.rs new file mode 100644 index 00000000000..b3717465d95 --- /dev/null +++ b/tests/it/array/list/mod.rs @@ -0,0 +1,66 @@ +use std::sync::Arc; + +use arrow2::array::*; +use arrow2::buffer::Buffer; +use arrow2::datatypes::DataType; + +mod mutable; + +#[test] +fn display() { + let values = Buffer::from([1, 2, 3, 4, 5]); + let values = PrimitiveArray::::from_data(DataType::Int32, values, None); + + let data_type = ListArray::::default_datatype(DataType::Int32); + let array = ListArray::::from_data( + data_type, + Buffer::from([0, 2, 2, 3, 5]), + Arc::new(values), + None, + ); + + assert_eq!( + format!("{}", array), + "ListArray[\nInt32[1, 2],\nInt32[],\nInt32[3],\nInt32[4, 5]\n]" + ); +} + +#[test] +#[should_panic(expected = "The child's datatype must match the inner type of the \'data_type\'")] +fn test_nested_panic() { + let values = Buffer::from([1, 2, 3, 4, 5]); + let values = PrimitiveArray::::from_data(DataType::Int32, values, None); + + let data_type = ListArray::::default_datatype(DataType::Int32); + let array = ListArray::::from_data( + data_type.clone(), + Buffer::from([0, 2, 2, 3, 5]), + Arc::new(values), + None, + ); + + // The datatype for the nested array has to be created considering + // the nested structure of the child data + let _ = ListArray::::from_data(data_type, Buffer::from([0, 2, 4]), Arc::new(array), None); +} + +#[test] +fn test_nested_display() { + let values = Buffer::from([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + let values = PrimitiveArray::::from_data(DataType::Int32, values, None); + + let data_type = ListArray::::default_datatype(DataType::Int32); + let array = ListArray::::from_data( + data_type, + Buffer::from([0, 2, 4, 7, 7, 8, 10]), + Arc::new(values), + None, + ); + + let data_type = ListArray::::default_datatype(array.data_type().clone()); + let nested = + ListArray::::from_data(data_type, Buffer::from([0, 2, 5, 6]), Arc::new(array), None); + + let expected = "ListArray[\nListArray[\nInt32[1, 2],\nInt32[3, 4]\n],\nListArray[\nInt32[5, 6, 7],\nInt32[],\nInt32[8]\n],\nListArray[\nInt32[9, 10]\n]\n]"; + assert_eq!(format!("{}", nested), expected); +} diff --git a/tests/it/array/list/mutable.rs b/tests/it/array/list/mutable.rs new file mode 100644 index 00000000000..d05ad6f34ca --- /dev/null +++ b/tests/it/array/list/mutable.rs @@ -0,0 +1,31 @@ +use std::sync::Arc; + +use arrow2::{array::*, bitmap::Bitmap, buffer::Buffer, datatypes::DataType}; + +#[test] +fn basics() { + let data = vec![ + Some(vec![Some(1i32), Some(2), Some(3)]), + None, + Some(vec![Some(4), None, Some(6)]), + ]; + + let mut array = MutableListArray::>::new(); + array.try_extend(data).unwrap(); + let array: ListArray = array.into(); + + let values = PrimitiveArray::::from_data( + DataType::Int32, + Buffer::from([1, 2, 3, 4, 0, 6]), + Some(Bitmap::from([true, true, true, true, false, true])), + ); + + let data_type = ListArray::::default_datatype(DataType::Int32); + let expected = ListArray::::from_data( + data_type, + Buffer::from([0, 3, 3, 6]), + Arc::new(values), + Some(Bitmap::from([true, false, true])), + ); + assert_eq!(expected, array); +} diff --git a/tests/it/array/mod.rs b/tests/it/array/mod.rs new file mode 100644 index 00000000000..046e0a38079 --- /dev/null +++ b/tests/it/array/mod.rs @@ -0,0 +1,60 @@ +mod binary; +mod boolean; +mod dictionary; +mod equal; +mod fixed_size_binary; +mod fixed_size_list; +mod growable; +mod list; +mod ord; +mod primitive; +mod union; +mod utf8; + +use arrow2::array::{clone, new_empty_array, new_null_array}; +use arrow2::datatypes::{DataType, Field}; + +#[test] +fn nulls() { + let datatypes = vec![ + DataType::Int32, + DataType::Float64, + DataType::Utf8, + DataType::Binary, + DataType::List(Box::new(Field::new("a", DataType::Binary, true))), + ]; + let a = datatypes + .into_iter() + .all(|x| new_null_array(x, 10).null_count() == 10); + assert!(a); +} + +#[test] +fn empty() { + let datatypes = vec![ + DataType::Int32, + DataType::Float64, + DataType::Utf8, + DataType::Binary, + DataType::List(Box::new(Field::new("a", DataType::Binary, true))), + DataType::Union(vec![Field::new("a", DataType::Binary, true)], None, true), + DataType::Union(vec![Field::new("a", DataType::Binary, true)], None, false), + ]; + let a = datatypes.into_iter().all(|x| new_empty_array(x).len() == 0); + assert!(a); +} + +#[test] +fn test_clone() { + let datatypes = vec![ + DataType::Int32, + DataType::Float64, + DataType::Utf8, + DataType::Binary, + DataType::List(Box::new(Field::new("a", DataType::Binary, true))), + ]; + let a = datatypes + .into_iter() + .all(|x| clone(new_null_array(x.clone(), 10).as_ref()) == new_null_array(x, 10)); + assert!(a); +} diff --git a/tests/it/array/ord.rs b/tests/it/array/ord.rs new file mode 100644 index 00000000000..cf2723436d0 --- /dev/null +++ b/tests/it/array/ord.rs @@ -0,0 +1,103 @@ +use std::cmp::Ordering; + +use arrow2::array::ord::build_compare; +use arrow2::array::*; +use arrow2::error::Result; + +#[test] +fn i32() -> Result<()> { + let array = Int32Array::from_slice(&[1, 2]); + + let cmp = build_compare(&array, &array)?; + + assert_eq!(Ordering::Less, (cmp)(0, 1)); + Ok(()) +} + +#[test] +fn i32_i32() -> Result<()> { + let array1 = Int32Array::from_slice(&[1]); + let array2 = Int32Array::from_slice(&[2]); + + let cmp = build_compare(&array1, &array2)?; + + assert_eq!(Ordering::Less, (cmp)(0, 0)); + Ok(()) +} + +#[test] +fn f32() -> Result<()> { + let array = &Float32Array::from_slice(&[1.0, 2.0]); + + let cmp = build_compare(array, array)?; + + assert_eq!(Ordering::Less, (cmp)(0, 1)); + Ok(()) +} + +#[test] +fn f64() -> Result<()> { + let array = Float64Array::from_slice(&[1.0, 2.0]); + + let cmp = build_compare(&array, &array)?; + + assert_eq!(Ordering::Less, (cmp)(0, 1)); + Ok(()) +} + +#[test] +fn f64_nan() -> Result<()> { + let array = Float64Array::from_slice(&[1.0, f64::NAN]); + + let cmp = build_compare(&array, &array)?; + + assert_eq!(Ordering::Less, (cmp)(0, 1)); + Ok(()) +} + +#[test] +fn f64_zeros() -> Result<()> { + let array = Float64Array::from_slice(&[-0.0, 0.0]); + + let cmp = build_compare(&array, &array)?; + + // official IEEE 754 (2008 revision) + assert_eq!(Ordering::Less, (cmp)(0, 1)); + assert_eq!(Ordering::Greater, (cmp)(1, 0)); + Ok(()) +} + +#[test] +fn dict_utf8() -> Result<()> { + let data = vec!["a", "b", "c", "a", "a", "c", "c"]; + + let data = data.into_iter().map(Some); + let mut array = MutableDictionaryArray::>::new(); + array.try_extend(data)?; + let array: DictionaryArray = array.into(); + + let cmp = build_compare(&array, &array)?; + + assert_eq!(Ordering::Less, (cmp)(0, 1)); + assert_eq!(Ordering::Equal, (cmp)(3, 4)); + assert_eq!(Ordering::Greater, (cmp)(2, 3)); + Ok(()) +} + +#[test] +fn dict_i32() -> Result<()> { + let data = vec![1, 2, 3, 1, 1, 3, 3]; + + let data = data.into_iter().map(Some); + + let mut array = MutableDictionaryArray::>::new(); + array.try_extend(data)?; + let array = array.into_arc(); + + let cmp = build_compare(array.as_ref(), array.as_ref())?; + + assert_eq!(Ordering::Less, (cmp)(0, 1)); + assert_eq!(Ordering::Equal, (cmp)(3, 4)); + assert_eq!(Ordering::Greater, (cmp)(2, 3)); + Ok(()) +} diff --git a/tests/it/array/primitive/mod.rs b/tests/it/array/primitive/mod.rs new file mode 100644 index 00000000000..57173a6415e --- /dev/null +++ b/tests/it/array/primitive/mod.rs @@ -0,0 +1,222 @@ +use arrow2::{array::*, bitmap::Bitmap, datatypes::*, types::days_ms}; +use std::iter::FromIterator; + +mod mutable; + +#[test] +fn basics() { + let data = vec![Some(1), None, Some(10)]; + + let array = Int32Array::from_iter(data); + + assert_eq!(array.value(0), 1); + assert_eq!(array.value(1), 0); + assert_eq!(array.value(2), 10); + assert_eq!(array.values().as_slice(), &[1, 0, 10]); + assert_eq!( + array.validity(), + &Some(Bitmap::from_u8_slice(&[0b00000101], 3)) + ); + assert!(array.is_valid(0)); + assert!(!array.is_valid(1)); + assert!(array.is_valid(2)); + + let array2 = Int32Array::from_data( + DataType::Int32, + array.values().clone(), + array.validity().clone(), + ); + assert_eq!(array, array2); + + let array = array.slice(1, 2); + assert_eq!(array.value(0), 0); + assert_eq!(array.value(1), 10); + assert_eq!(array.values().as_slice(), &[0, 10]); + + unsafe { + assert_eq!(array.value_unchecked(0), 0); + assert_eq!(array.value_unchecked(1), 10); + } +} + +#[test] +fn empty() { + let array = Int32Array::new_empty(DataType::Int32); + assert_eq!(array.values().len(), 0); + assert_eq!(array.validity(), &None); +} + +#[test] +fn from() { + let data = vec![Some(1), None, Some(10)]; + + let array = PrimitiveArray::from(data.clone()); + assert_eq!(array.len(), 3); + + let array = PrimitiveArray::from_iter(data.clone()); + assert_eq!(array.len(), 3); + + let array = PrimitiveArray::from_trusted_len_iter(data.into_iter()); + assert_eq!(array.len(), 3); + + let data = vec![1i32, 2, 3]; + + let array = PrimitiveArray::from_values(data.clone()); + assert_eq!(array.len(), 3); + + let array = PrimitiveArray::from_trusted_len_values_iter(data.into_iter()); + assert_eq!(array.len(), 3); +} + +#[test] +fn display_int32() { + let array = Int32Array::from(&[Some(1), None, Some(2)]); + assert_eq!(format!("{}", array), "Int32[1, , 2]"); +} + +#[test] +fn display_date32() { + let array = Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Date32); + assert_eq!(format!("{}", array), "Date32[1970-01-02, , 1970-01-03]"); +} + +#[test] +fn display_time32s() { + let array = Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Time32(TimeUnit::Second)); + assert_eq!(format!("{}", array), "Time32(Second)[00:00:01, , 00:00:02]"); +} + +#[test] +fn display_time32ms() { + let array = + Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Time32(TimeUnit::Millisecond)); + assert_eq!( + format!("{}", array), + "Time32(Millisecond)[00:00:00.001, , 00:00:00.002]" + ); +} + +#[test] +fn display_interval_d() { + let array = + Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Interval(IntervalUnit::YearMonth)); + assert_eq!(format!("{}", array), "Interval(YearMonth)[1m, , 2m]"); +} + +#[test] +fn display_int64() { + let array = Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Int64); + assert_eq!(format!("{}", array), "Int64[1, , 2]"); +} + +#[test] +fn display_date64() { + let array = Int64Array::from(&[Some(1), None, Some(86400000)]).to(DataType::Date64); + assert_eq!(format!("{}", array), "Date64[1970-01-01, , 1970-01-02]"); +} + +#[test] +fn display_time64us() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Time64(TimeUnit::Microsecond)); + assert_eq!( + format!("{}", array), + "Time64(Microsecond)[00:00:00.000001, , 00:00:00.000002]" + ); +} + +#[test] +fn display_time64ns() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Time64(TimeUnit::Nanosecond)); + assert_eq!( + format!("{}", array), + "Time64(Nanosecond)[00:00:00.000000001, , 00:00:00.000000002]" + ); +} + +#[test] +fn display_timestamp_s() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Timestamp(TimeUnit::Second, None)); + assert_eq!( + format!("{}", array), + "Timestamp(Second, None)[1970-01-01 00:00:01, , 1970-01-01 00:00:02]" + ); +} + +#[test] +fn display_timestamp_ms() { + let array = Int64Array::from(&[Some(1), None, Some(2)]) + .to(DataType::Timestamp(TimeUnit::Millisecond, None)); + assert_eq!( + format!("{}", array), + "Timestamp(Millisecond, None)[1970-01-01 00:00:00.001, , 1970-01-01 00:00:00.002]" + ); +} + +#[test] +fn display_timestamp_us() { + let array = Int64Array::from(&[Some(1), None, Some(2)]) + .to(DataType::Timestamp(TimeUnit::Microsecond, None)); + assert_eq!( + format!("{}", array), + "Timestamp(Microsecond, None)[1970-01-01 00:00:00.000001, , 1970-01-01 00:00:00.000002]" + ); +} + +#[test] +fn display_timestamp_ns() { + let array = Int64Array::from(&[Some(1), None, Some(2)]) + .to(DataType::Timestamp(TimeUnit::Nanosecond, None)); + assert_eq!( + format!("{}", array), + "Timestamp(Nanosecond, None)[1970-01-01 00:00:00.000000001, , 1970-01-01 00:00:00.000000002]" + ); +} + +#[test] +fn display_duration_ms() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Millisecond)); + assert_eq!(format!("{}", array), "Duration(Millisecond)[1ms, , 2ms]"); +} + +#[test] +fn display_duration_s() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Second)); + assert_eq!(format!("{}", array), "Duration(Second)[1s, , 2s]"); +} + +#[test] +fn display_duration_us() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Microsecond)); + assert_eq!(format!("{}", array), "Duration(Microsecond)[1us, , 2us]"); +} + +#[test] +fn display_duration_ns() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Nanosecond)); + assert_eq!(format!("{}", array), "Duration(Nanosecond)[1ns, , 2ns]"); +} + +#[test] +fn display_decimal() { + let array = Int128Array::from(&[Some(12345), None, Some(23456)]).to(DataType::Decimal(5, 2)); + assert_eq!(format!("{}", array), "Decimal(5, 2)[123.45, , 234.56]"); +} + +#[test] +fn display_decimal1() { + let array = Int128Array::from(&[Some(12345), None, Some(23456)]).to(DataType::Decimal(5, 1)); + assert_eq!(format!("{}", array), "Decimal(5, 1)[1234.5, , 2345.6]"); +} + +#[test] +fn display_interval_days_ms() { + let array = DaysMsArray::from(&[Some(days_ms::new(1, 1)), None, Some(days_ms::new(2, 2))]); + assert_eq!(format!("{}", array), "Interval(DayTime)[1d1ms, , 2d2ms]"); +} diff --git a/tests/it/array/primitive/mutable.rs b/tests/it/array/primitive/mutable.rs new file mode 100644 index 00000000000..8dbff298239 --- /dev/null +++ b/tests/it/array/primitive/mutable.rs @@ -0,0 +1,74 @@ +use arrow2::{ + array::*, + bitmap::{Bitmap, MutableBitmap}, + buffer::MutableBuffer, +}; +use std::iter::FromIterator; + +#[test] +fn push() { + let mut a = MutablePrimitiveArray::::new(); + a.push(Some(1)); + a.push(None); + assert_eq!(a.len(), 2); + assert!(a.is_valid(0)); + assert!(!a.is_valid(1)); + + assert_eq!(a.values(), &MutableBuffer::from([1, 0])); +} + +#[test] +fn set() { + let mut a = MutablePrimitiveArray::::from([Some(1), None]); + + a.set(0, Some(2)); + a.set(1, Some(1)); + + assert_eq!(a.len(), 2); + assert!(a.is_valid(0)); + assert!(a.is_valid(1)); + + assert_eq!(a.values(), &MutableBuffer::from([2, 1])); +} + +#[test] +fn from_iter() { + let a = MutablePrimitiveArray::::from_iter((0..2).map(Some)); + assert_eq!(a.len(), 2); + assert_eq!(a.validity(), &None); +} + +#[test] +fn natural_arc() { + let a = MutablePrimitiveArray::::from_slice(&[0, 1]).into_arc(); + assert_eq!(a.len(), 2); +} + +#[test] +fn only_nulls() { + let mut a = MutablePrimitiveArray::::new(); + a.push(None); + a.push(None); + let a: PrimitiveArray = a.into(); + assert_eq!(a.validity(), &Some(Bitmap::from([false, false]))); +} + +#[test] +fn from_trusted_len() { + let a = MutablePrimitiveArray::::from_trusted_len_iter(vec![Some(1), None].into_iter()); + let a: PrimitiveArray = a.into(); + assert_eq!(a.validity(), &Some(Bitmap::from([true, false]))); +} + +#[test] +fn extend_trusted_len() { + let mut a = MutablePrimitiveArray::::new(); + a.extend_trusted_len(vec![Some(1), Some(2)].into_iter()); + assert_eq!(a.validity(), &None); + a.extend_trusted_len(vec![None, Some(4)].into_iter()); + assert_eq!( + a.validity(), + &Some(MutableBitmap::from([true, true, false, true])) + ); + assert_eq!(a.values(), &MutableBuffer::::from([1, 2, 0, 4])); +} diff --git a/tests/it/array/union.rs b/tests/it/array/union.rs new file mode 100644 index 00000000000..ecc7f46fe5f --- /dev/null +++ b/tests/it/array/union.rs @@ -0,0 +1,23 @@ +use std::sync::Arc; + +use arrow2::{array::*, buffer::Buffer, datatypes::*, error::Result}; + +#[test] +fn display() -> Result<()> { + let fields = vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Utf8, true), + ]; + let data_type = DataType::Union(fields, None, true); + let types = Buffer::from(&[0, 0, 1]); + let fields = vec![ + Arc::new(Int32Array::from(&[Some(1), None, Some(2)])) as Arc, + Arc::new(Utf8Array::::from(&[Some("a"), Some("b"), Some("c")])) as Arc, + ]; + + let array = UnionArray::from_data(data_type, types, fields, None); + + assert_eq!(format!("{}", array), "UnionArray[1, , c]"); + + Ok(()) +} diff --git a/tests/it/array/utf8/mod.rs b/tests/it/array/utf8/mod.rs new file mode 100644 index 00000000000..6e8ad20949f --- /dev/null +++ b/tests/it/array/utf8/mod.rs @@ -0,0 +1,63 @@ +use arrow2::{array::*, bitmap::Bitmap, buffer::Buffer}; + +mod mutable; + +#[test] +fn basics() { + let data = vec![Some("hello"), None, Some("hello2")]; + + let array: Utf8Array = data.into_iter().collect(); + + assert_eq!(array.value(0), "hello"); + assert_eq!(array.value(1), ""); + assert_eq!(array.value(2), "hello2"); + assert_eq!(unsafe { array.value_unchecked(2) }, "hello2"); + assert_eq!(array.values().as_slice(), b"hellohello2"); + assert_eq!(array.offsets().as_slice(), &[0, 5, 5, 11]); + assert_eq!( + array.validity(), + &Some(Bitmap::from_u8_slice(&[0b00000101], 3)) + ); + assert!(array.is_valid(0)); + assert!(!array.is_valid(1)); + assert!(array.is_valid(2)); + + let array2 = Utf8Array::::from_data( + array.offsets().clone(), + array.values().clone(), + array.validity().clone(), + ); + assert_eq!(array, array2); + + let array = array.slice(1, 2); + assert_eq!(array.value(0), ""); + assert_eq!(array.value(1), "hello2"); + // note how this keeps everything: the offsets were sliced + assert_eq!(array.values().as_slice(), b"hellohello2"); + assert_eq!(array.offsets().as_slice(), &[5, 5, 11]); +} + +#[test] +fn empty() { + let array = Utf8Array::::new_empty(); + assert_eq!(array.values().as_slice(), b""); + assert_eq!(array.offsets().as_slice(), &[0]); + assert_eq!(array.validity(), &None); +} + +#[test] +fn from() { + let array = Utf8Array::::from(&[Some("hello"), Some(" "), None]); + + let a = array.validity().as_ref().unwrap(); + assert_eq!(a, &Bitmap::from([true, true, false])); +} + +#[test] +fn from_iter_values() { + let b = Utf8Array::::from_iter_values(vec!["a", "b", "cc"]); + + let offsets = Buffer::from(&[0, 1, 2, 4]); + let values = Buffer::from("abcc".as_bytes()); + assert_eq!(b, Utf8Array::::from_data(offsets, values, None)); +} diff --git a/tests/it/array/utf8/mutable.rs b/tests/it/array/utf8/mutable.rs new file mode 100644 index 00000000000..1aedcf16d33 --- /dev/null +++ b/tests/it/array/utf8/mutable.rs @@ -0,0 +1,19 @@ +use arrow2::array::{Array, MutableUtf8Array, Utf8Array}; +use arrow2::bitmap::Bitmap; + +#[test] +fn capacities() { + let b = MutableUtf8Array::::with_capacities(1, 10); + + assert_eq!(b.values().capacity(), 64); + assert_eq!(b.offsets().capacity(), 16); // 64 bytes +} + +#[test] +fn push_null() { + let mut array = MutableUtf8Array::::new(); + array.push::<&str>(None); + + let array: Utf8Array = array.into(); + assert_eq!(array.validity(), &Some(Bitmap::from([false]))); +} diff --git a/tests/it/bitmap/mod.rs b/tests/it/bitmap/mod.rs index 85a86d9c1fe..39466bec2cb 100644 --- a/tests/it/bitmap/mod.rs +++ b/tests/it/bitmap/mod.rs @@ -1,3 +1,89 @@ mod immutable; mod mutable; mod utils; + +use arrow2::{bitmap::Bitmap, buffer::MutableBuffer}; + +fn create_bitmap>(bytes: P, len: usize) -> Bitmap { + let buffer = MutableBuffer::::from(bytes.as_ref()); + Bitmap::from_u8_buffer(buffer, len) +} + +#[test] +fn eq() { + let lhs = create_bitmap([0b01101010], 8); + let rhs = create_bitmap([0b01001110], 8); + assert!(lhs != rhs); +} + +#[test] +fn eq_len() { + let lhs = create_bitmap([0b01101010], 6); + let rhs = create_bitmap([0b00101010], 6); + assert!(lhs == rhs); + let rhs = create_bitmap([0b00001010], 6); + assert!(lhs != rhs); +} + +#[test] +fn eq_slice() { + let lhs = create_bitmap([0b10101010], 8).slice(1, 7); + let rhs = create_bitmap([0b10101011], 8).slice(1, 7); + assert!(lhs == rhs); + + let lhs = create_bitmap([0b10101010], 8).slice(2, 6); + let rhs = create_bitmap([0b10101110], 8).slice(2, 6); + assert!(lhs != rhs); +} + +#[test] +fn and() { + let lhs = create_bitmap([0b01101010], 8); + let rhs = create_bitmap([0b01001110], 8); + let expected = create_bitmap([0b01001010], 8); + assert_eq!(&lhs & &rhs, expected); +} + +#[test] +fn or_large() { + let input: &[u8] = &[ + 0b00000000, 0b00000001, 0b00000010, 0b00000100, 0b00001000, 0b00010000, 0b00100000, + 0b01000010, 0b11111111, + ]; + let input1: &[u8] = &[ + 0b00000000, 0b00000001, 0b10000000, 0b10000000, 0b10000000, 0b10000000, 0b10000000, + 0b10000000, 0b11111111, + ]; + let expected: &[u8] = &[ + 0b00000000, 0b00000001, 0b10000010, 0b10000100, 0b10001000, 0b10010000, 0b10100000, + 0b11000010, 0b11111111, + ]; + + let lhs = create_bitmap(input, 62); + let rhs = create_bitmap(input1, 62); + let expected = create_bitmap(expected, 62); + assert_eq!(&lhs | &rhs, expected); +} + +#[test] +fn and_offset() { + let lhs = create_bitmap([0b01101011], 8).slice(1, 7); + let rhs = create_bitmap([0b01001111], 8).slice(1, 7); + let expected = create_bitmap([0b01001010], 8).slice(1, 7); + assert_eq!(&lhs & &rhs, expected); +} + +#[test] +fn or() { + let lhs = create_bitmap([0b01101010], 8); + let rhs = create_bitmap([0b01001110], 8); + let expected = create_bitmap([0b01101110], 8); + assert_eq!(&lhs | &rhs, expected); +} + +#[test] +fn not() { + let lhs = create_bitmap([0b01101010], 6); + let expected = create_bitmap([0b00010101], 6); + assert_eq!(!&lhs, expected); +} diff --git a/tests/it/ffi.rs b/tests/it/ffi.rs index ec45c79a7f5..fba1a179861 100644 --- a/tests/it/ffi.rs +++ b/tests/it/ffi.rs @@ -30,44 +30,44 @@ fn test_round_trip(expected: impl Array + Clone + 'static) -> Result<()> { } #[test] -fn test_u32() -> Result<()> { +fn u32() -> Result<()> { let data = Int32Array::from(&[Some(2), None, Some(1), None]); test_release(data) } #[test] -fn test_u64() -> Result<()> { +fn u64() -> Result<()> { let data = UInt64Array::from(&[Some(2), None, Some(1), None]); test_round_trip(data) } #[test] -fn test_i64() -> Result<()> { +fn i64() -> Result<()> { let data = Int64Array::from(&[Some(2), None, Some(1), None]); test_round_trip(data) } #[test] -fn test_utf8() -> Result<()> { +fn utf8() -> Result<()> { let data = Utf8Array::::from(&vec![Some("a"), None, Some("bb"), None]); test_round_trip(data) } #[test] -fn test_large_utf8() -> Result<()> { +fn large_utf8() -> Result<()> { let data = Utf8Array::::from(&vec![Some("a"), None, Some("bb"), None]); test_round_trip(data) } #[test] -fn test_binary() -> Result<()> { +fn binary() -> Result<()> { let data = BinaryArray::::from(&vec![Some(b"a".as_ref()), None, Some(b"bb".as_ref()), None]); test_round_trip(data) } #[test] -fn test_timestamp_tz() -> Result<()> { +fn timestamp_tz() -> Result<()> { let data = Int64Array::from(&vec![Some(2), None, None]).to(DataType::Timestamp( TimeUnit::Second, Some("UTC".to_string()), @@ -76,14 +76,14 @@ fn test_timestamp_tz() -> Result<()> { } #[test] -fn test_large_binary() -> Result<()> { +fn large_binary() -> Result<()> { let data = BinaryArray::::from(&vec![Some(b"a".as_ref()), None, Some(b"bb".as_ref()), None]); test_round_trip(data) } #[test] -fn test_list() -> Result<()> { +fn list() -> Result<()> { let data = vec![ Some(vec![Some(1i32), Some(2), Some(3)]), None, @@ -99,7 +99,7 @@ fn test_list() -> Result<()> { } #[test] -fn test_list_list() -> Result<()> { +fn list_list() -> Result<()> { let data = vec![ Some(vec![ Some(vec![None]), @@ -120,7 +120,7 @@ fn test_list_list() -> Result<()> { } #[test] -fn test_dict() -> Result<()> { +fn dict() -> Result<()> { let data = vec![Some("a"), Some("a"), None, Some("b")]; let mut array = MutableDictionaryArray::>::new(); diff --git a/tests/it/io/csv/mod.rs b/tests/it/io/csv/mod.rs new file mode 100644 index 00000000000..f7cdac66078 --- /dev/null +++ b/tests/it/io/csv/mod.rs @@ -0,0 +1,2 @@ +mod read; +mod write; diff --git a/tests/it/io/csv/read.rs b/tests/it/io/csv/read.rs new file mode 100644 index 00000000000..2a5ee355f54 --- /dev/null +++ b/tests/it/io/csv/read.rs @@ -0,0 +1,160 @@ +use std::io::Cursor; +use std::sync::Arc; + +use arrow2::array::*; +use arrow2::datatypes::*; +use arrow2::error::Result; +use arrow2::io::csv::read::*; + +#[test] +fn read() -> Result<()> { + let mut reader = ReaderBuilder::new().from_path("test/data/uk_cities_with_headers.csv")?; + + let schema = Arc::new(infer_schema(&mut reader, None, true, &infer)?); + + let mut rows = vec![ByteRecord::default(); 100]; + let rows_read = read_rows(&mut reader, 0, &mut rows)?; + + let batch = deserialize_batch( + &rows[..rows_read], + schema.fields(), + None, + 0, + deserialize_column, + )?; + + let batch_schema = batch.schema(); + + assert_eq!(&schema, batch_schema); + assert_eq!(37, batch.num_rows()); + assert_eq!(3, batch.num_columns()); + + let lat = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert!((57.653484 - lat.value(0)).abs() < f64::EPSILON); + + let city = batch + .column(0) + .as_any() + .downcast_ref::>() + .unwrap(); + + assert_eq!("Elgin, Scotland, the UK", city.value(0)); + assert_eq!("Aberdeen, Aberdeen City, UK", city.value(13)); + Ok(()) +} + +#[test] +fn infer_basics() -> Result<()> { + let file = Cursor::new("1,2,3\na,b,c\na,,c"); + let mut reader = ReaderBuilder::new().from_reader(file); + + let schema = infer_schema(&mut reader, Some(10), false, &infer)?; + + assert_eq!( + schema, + Schema::new(vec![ + Field::new("column_1", DataType::Utf8, true), + Field::new("column_2", DataType::Utf8, true), + Field::new("column_3", DataType::Utf8, true), + ]) + ); + Ok(()) +} + +#[test] +fn infer_ints() -> Result<()> { + let file = Cursor::new("1,2,3\n1,a,5\n2,,4"); + let mut reader = ReaderBuilder::new().from_reader(file); + + let schema = infer_schema(&mut reader, Some(10), false, &infer)?; + + assert_eq!( + schema, + Schema::new(vec![ + Field::new("column_1", DataType::Int64, true), + Field::new("column_2", DataType::Utf8, true), + Field::new("column_3", DataType::Int64, true), + ]) + ); + Ok(()) +} + +fn test_deserialize(input: &str, data_type: DataType) -> Result> { + let reader = std::io::Cursor::new(input); + let mut reader = ReaderBuilder::new().has_headers(false).from_reader(reader); + + let mut rows = vec![ByteRecord::default(); 10]; + let rows_read = read_rows(&mut reader, 0, &mut rows)?; + deserialize_column(&rows[..rows_read], 0, data_type, 0) +} + +#[test] +fn int32() -> Result<()> { + let result = test_deserialize("1,\n,\n3,", DataType::Int32)?; + let expected = Int32Array::from(&[Some(1), None, Some(3)]); + assert_eq!(expected, result.as_ref()); + Ok(()) +} + +#[test] +fn date32() -> Result<()> { + let result = test_deserialize("1970-01-01,\n2020-03-15,\n1945-05-08,\n", DataType::Date32)?; + let expected = Int32Array::from(&[Some(0), Some(18336), Some(-9004)]).to(DataType::Date32); + assert_eq!(expected, result.as_ref()); + Ok(()) +} + +#[test] +fn date64() -> Result<()> { + let input = "1970-01-01T00:00:00,\n \ + 2018-11-13T17:11:10,\n \ + 2018-11-13T17:11:10.011,\n \ + 1900-02-28T12:34:56,\n"; + + let result = test_deserialize(input, DataType::Date64)?; + let expected = Int64Array::from(&[ + Some(0), + Some(1542129070000), + Some(1542129070011), + Some(-2203932304000), + ]) + .to(DataType::Date64); + assert_eq!(expected, result.as_ref()); + Ok(()) +} + +#[test] +fn boolean() -> Result<()> { + let input = vec!["true", "True", "False", "F", "t"]; + let input = input.join("\n"); + + let expected = BooleanArray::from(&[Some(true), Some(true), Some(false), None, None]); + + let result = test_deserialize(&input, DataType::Boolean)?; + + assert_eq!(expected, result.as_ref()); + Ok(()) +} + +#[test] +fn float32() -> Result<()> { + let input = vec!["12.34", "12", "0.0", "inf", "-inf", "dd"]; + let input = input.join("\n"); + + let expected = Float32Array::from(&[ + Some(12.34), + Some(12.0), + Some(0.0), + Some(f32::INFINITY), + Some(f32::NEG_INFINITY), + None, + ]); + + let result = test_deserialize(&input, DataType::Float32)?; + assert_eq!(expected, result.as_ref()); + Ok(()) +} diff --git a/tests/it/io/csv/write.rs b/tests/it/io/csv/write.rs new file mode 100644 index 00000000000..bd41d2e7998 --- /dev/null +++ b/tests/it/io/csv/write.rs @@ -0,0 +1,102 @@ +use std::io::Cursor; +use std::sync::Arc; + +use arrow2::array::*; +use arrow2::datatypes::*; +use arrow2::error::Result; +use arrow2::io::csv::write::*; +use arrow2::record_batch::RecordBatch; + +fn data() -> RecordBatch { + let schema = Schema::new(vec![ + Field::new("c1", DataType::Utf8, false), + Field::new("c2", DataType::Float64, true), + Field::new("c3", DataType::UInt32, false), + Field::new("c4", DataType::Boolean, true), + Field::new("c5", DataType::Timestamp(TimeUnit::Millisecond, None), true), + Field::new("c6", DataType::Time32(TimeUnit::Second), false), + ]); + + let c1 = Utf8Array::::from_slice([ + "Lorem ipsum dolor sit amet", + "consectetur adipiscing elit", + "sed do eiusmod tempor", + ]); + let c2 = Float64Array::from([Some(123.564532), None, Some(-556132.25)]); + let c3 = UInt32Array::from_slice(&[3, 2, 1]); + let c4 = BooleanArray::from(&[Some(true), Some(false), None]); + let c5 = PrimitiveArray::::from([None, Some(1555584887378), Some(1555555555555)]) + .to(DataType::Timestamp(TimeUnit::Millisecond, None)); + let c6 = PrimitiveArray::::from_slice(&[1234, 24680, 85563]) + .to(DataType::Time32(TimeUnit::Second)); + + RecordBatch::try_new( + Arc::new(schema), + vec![ + Arc::new(c1), + Arc::new(c2), + Arc::new(c3), + Arc::new(c4), + Arc::new(c5), + Arc::new(c6), + ], + ) + .unwrap() +} + +#[test] +fn write_csv() -> Result<()> { + let batch = data(); + + let write = Cursor::new(Vec::::new()); + let mut writer = WriterBuilder::new().from_writer(write); + + write_header(&mut writer, batch.schema())?; + let batches = vec![&batch, &batch]; + let options = SerializeOptions::default(); + batches + .iter() + .try_for_each(|batch| write_batch(&mut writer, batch, &options))?; + + // check + let buffer = writer.into_inner().unwrap().into_inner(); + assert_eq!( + r#"c1,c2,c3,c4,c5,c6 +Lorem ipsum dolor sit amet,123.564532,3,true,,00:20:34 +consectetur adipiscing elit,,2,false,2019-04-18T10:54:47.378000000,06:51:20 +sed do eiusmod tempor,-556132.25,1,,2019-04-18T02:45:55.555000000,23:46:03 +Lorem ipsum dolor sit amet,123.564532,3,true,,00:20:34 +consectetur adipiscing elit,,2,false,2019-04-18T10:54:47.378000000,06:51:20 +sed do eiusmod tempor,-556132.25,1,,2019-04-18T02:45:55.555000000,23:46:03 +"# + .to_string(), + String::from_utf8(buffer).unwrap(), + ); + Ok(()) +} + +#[test] +fn write_csv_custom_options() -> Result<()> { + let batch = data(); + + let write = Cursor::new(Vec::::new()); + let mut writer = WriterBuilder::new().delimiter(b'|').from_writer(write); + + let options = SerializeOptions { + time_format: "%r".to_string(), + ..Default::default() + }; + write_batch(&mut writer, &batch, &options)?; + + // check + let buffer = writer.into_inner().unwrap().into_inner(); + assert_eq!( + r#"Lorem ipsum dolor sit amet|123.564532|3|true||12:20:34 AM +consectetur adipiscing elit||2|false|2019-04-18T10:54:47.378000000|06:51:20 AM +sed do eiusmod tempor|-556132.25|1||2019-04-18T02:45:55.555000000|11:46:03 PM +"# + .to_string(), + String::from_utf8(buffer).unwrap(), + ); + Ok(()) +} diff --git a/tests/it/io/ipc/write/file.rs b/tests/it/io/ipc/write/file.rs index b1ae89a727b..7a463938044 100644 --- a/tests/it/io/ipc/write/file.rs +++ b/tests/it/io/ipc/write/file.rs @@ -8,7 +8,7 @@ use arrow2::record_batch::RecordBatch; use crate::io::ipc::common::read_gzip_json; -fn test_round_trip(batch: RecordBatch) -> Result<()> { +fn round_trip(batch: RecordBatch) -> Result<()> { let mut result = Vec::::new(); // write IPC version 5 @@ -178,7 +178,7 @@ fn write_sliced_utf8() -> Result<()> { use std::sync::Arc; let array = Arc::new(Utf8Array::::from_slice(["aa", "bb"]).slice(1, 1)) as Arc; let batch = RecordBatch::try_from_iter(vec![("a", array)]).unwrap(); - test_round_trip(batch) + round_trip(batch) } #[test] @@ -193,5 +193,5 @@ fn write_sliced_list() -> Result<()> { array.try_extend(data).unwrap(); let array = array.into_arc().slice(1, 2).into(); let batch = RecordBatch::try_from_iter(vec![("a", array)]).unwrap(); - test_round_trip(batch) + round_trip(batch) } diff --git a/tests/it/io/json/mod.rs b/tests/it/io/json/mod.rs index f7cdac66078..63c11ec1647 100644 --- a/tests/it/io/json/mod.rs +++ b/tests/it/io/json/mod.rs @@ -1,2 +1,58 @@ mod read; mod write; + +use serde_json::Value; +use std::fs::{read_to_string, File}; + +use arrow2::io::json::LineDelimitedWriter; +use arrow2::io::json::Reader; +use arrow2::io::json::ReaderBuilder; + +fn test_write_for_file(test_file: &str) { + let builder = ReaderBuilder::new() + .infer_schema(None) + .with_batch_size(1024); + let mut reader: Reader = builder + .build::(File::open(test_file).unwrap()) + .unwrap(); + let batch = reader.next().unwrap().unwrap(); + + let mut buf = Vec::new(); + { + let mut writer = LineDelimitedWriter::new(&mut buf); + writer.write_batches(&[batch]).unwrap(); + } + + let result = String::from_utf8(buf).unwrap(); + let expected = read_to_string(test_file).unwrap(); + for (r, e) in result.lines().zip(expected.lines()) { + let mut result_json = serde_json::from_str::(r).unwrap(); + let expected_json = serde_json::from_str::(e).unwrap(); + if let Value::Object(e) = &expected_json { + // remove null value from object to make comparision consistent: + if let Value::Object(r) = result_json { + result_json = Value::Object( + r.into_iter() + .filter(|(k, v)| e.contains_key(k) || *v != Value::Null) + .collect(), + ); + } + assert_eq!(result_json, expected_json); + } + } +} + +#[test] +fn write_basic_rows() { + test_write_for_file("test/data/basic.json"); +} + +#[test] +fn write_arrays() { + test_write_for_file("test/data/arrays.json"); +} + +#[test] +fn write_basic_nulls() { + test_write_for_file("test/data/basic_nulls.json"); +} diff --git a/tests/it/io/json/read.rs b/tests/it/io/json/read.rs index 78ef6be9c69..de3bd737623 100644 --- a/tests/it/io/json/read.rs +++ b/tests/it/io/json/read.rs @@ -11,7 +11,7 @@ use arrow2::datatypes::*; use arrow2::{bitmap::Bitmap, buffer::Buffer, error::Result, io::json::*}; #[test] -fn test_json_basic() { +fn json_basic() { let builder = ReaderBuilder::new().infer_schema(None).with_batch_size(64); let mut reader: Reader = builder .build::(File::open("test/data/basic.json").unwrap()) @@ -69,7 +69,7 @@ fn test_json_basic() { } #[test] -fn test_json_basic_with_nulls() { +fn json_basic_with_nulls() { let builder = ReaderBuilder::new().infer_schema(None).with_batch_size(64); let mut reader: Reader = builder .build::(File::open("test/data/basic_nulls.json").unwrap()) @@ -128,7 +128,7 @@ fn test_json_basic_with_nulls() { } #[test] -fn test_json_basic_schema() { +fn json_basic_schema() { let schema = Arc::new(Schema::new(vec![ Field::new("a", DataType::Int32, false), Field::new("b", DataType::Float32, false), @@ -178,7 +178,7 @@ fn test_json_basic_schema() { } #[test] -fn test_json_basic_schema_projection() { +fn json_basic_schema_projection() { // We test implicit and explicit projection: // Implicit: omitting fields from a schema // Explicit: supplying a vec of fields to take @@ -218,7 +218,7 @@ fn test_json_basic_schema_projection() { } #[test] -fn test_json_arrays() { +fn json_arrays() { let builder = ReaderBuilder::new().infer_schema(None).with_batch_size(64); let mut reader: Reader = builder .build::(File::open("test/data/arrays.json").unwrap()) @@ -278,7 +278,7 @@ fn test_json_arrays() { } #[test] -fn test_invalid_json_infer_schema() { +fn invalid_json_infer_schema() { let re = infer_json_schema_from_seekable( &mut BufReader::new(File::open("test/data/uk_cities_with_headers.csv").unwrap()), None, @@ -290,7 +290,7 @@ fn test_invalid_json_infer_schema() { } #[test] -fn test_invalid_json_read_record() { +fn invalid_json_read_record() { let schema = Arc::new(Schema::new(vec![Field::new( "a", DataType::Struct(vec![Field::new("a", DataType::Utf8, true)]), @@ -307,7 +307,7 @@ fn test_invalid_json_read_record() { } #[test] -fn test_mixed_json_arrays() { +fn mixed_json_arrays() { let builder = ReaderBuilder::new().infer_schema(None).with_batch_size(64); let mut reader: Reader = builder .build::(File::open("test/data/mixed_arrays.json").unwrap()) @@ -382,7 +382,7 @@ fn test_mixed_json_arrays() { } #[test] -fn test_nested_struct_json_arrays() { +fn nested_struct_json_arrays() { let d_field = Field::new("d", DataType::Utf8, true); let c_field = Field::new("c", DataType::Struct(vec![d_field.clone()]), true); let a_field = Field::new( @@ -417,7 +417,7 @@ fn test_nested_struct_json_arrays() { } #[test] -fn test_nested_list_json_arrays() { +fn nested_list_json_arrays() { let d_field = Field::new("d", DataType::Utf8, true); let c_field = Field::new("c", DataType::Struct(vec![d_field.clone()]), true); let b_field = Field::new("b", DataType::Boolean, true); @@ -478,7 +478,7 @@ fn test_nested_list_json_arrays() { } #[test] -fn test_dictionary_from_json_basic_with_nulls() -> Result<()> { +fn dictionary_from_json_basic_with_nulls() -> Result<()> { let schema = Arc::new(Schema::new(vec![Field::new( "d", DataType::Dictionary(Box::new(DataType::Int16), Box::new(DataType::Utf8)), @@ -527,7 +527,7 @@ fn test_dictionary_from_json_basic_with_nulls() -> Result<()> { } #[test] -fn test_skip_empty_lines() { +fn skip_empty_lines() { let builder = ReaderBuilder::new().infer_schema(None).with_batch_size(64); let json_content = " {\"a\": 1} @@ -547,7 +547,7 @@ fn test_skip_empty_lines() { } #[test] -fn test_row_type_validation() { +fn row_type_validation() { let builder = ReaderBuilder::new().infer_schema(None).with_batch_size(64); let json_content = " [1, \"hello\"] @@ -560,7 +560,7 @@ fn test_row_type_validation() { } #[test] -fn test_list_of_string_dictionary_from_json_with_nulls() -> Result<()> { +fn list_of_string_dictionary_from_json_with_nulls() -> Result<()> { let data_type = DataType::List(Box::new(Field::new( "item", DataType::Dictionary(Box::new(DataType::UInt64), Box::new(DataType::Utf8)), @@ -611,7 +611,7 @@ fn test_list_of_string_dictionary_from_json_with_nulls() -> Result<()> { } #[test] -fn test_with_multiple_batches() { +fn with_multiple_batches() { let builder = ReaderBuilder::new() .infer_schema(Some(4)) .with_batch_size(5); @@ -628,7 +628,7 @@ fn test_with_multiple_batches() { } #[test] -fn test_json_infer_schema() { +fn json_infer_schema() { let schema = Schema::new(vec![ Field::new("a", DataType::Int64, true), Field::new( diff --git a/tests/it/io/mod.rs b/tests/it/io/mod.rs index d5c4636d733..9ec0112e906 100644 --- a/tests/it/io/mod.rs +++ b/tests/it/io/mod.rs @@ -9,3 +9,6 @@ mod ipc; #[cfg(feature = "io_parquet")] mod parquet; + +#[cfg(feature = "io_csv")] +mod csv; diff --git a/tests/it/io/parquet/write.rs b/tests/it/io/parquet/write.rs index 3e3b1d14618..cc2a13253f7 100644 --- a/tests/it/io/parquet/write.rs +++ b/tests/it/io/parquet/write.rs @@ -68,7 +68,7 @@ fn round_trip( } #[test] -fn test_int64_optional_v1() -> Result<()> { +fn int64_optional_v1() -> Result<()> { round_trip( 0, true, @@ -80,7 +80,7 @@ fn test_int64_optional_v1() -> Result<()> { } #[test] -fn test_int64_required_v1() -> Result<()> { +fn int64_required_v1() -> Result<()> { round_trip( 0, false, @@ -92,7 +92,7 @@ fn test_int64_required_v1() -> Result<()> { } #[test] -fn test_int64_optional_v2() -> Result<()> { +fn int64_optional_v2() -> Result<()> { round_trip( 0, true, @@ -104,7 +104,7 @@ fn test_int64_optional_v2() -> Result<()> { } #[test] -fn test_int64_optional_v2_compressed() -> Result<()> { +fn int64_optional_v2_compressed() -> Result<()> { round_trip( 0, true, @@ -116,7 +116,7 @@ fn test_int64_optional_v2_compressed() -> Result<()> { } #[test] -fn test_utf8_optional_v1() -> Result<()> { +fn utf8_optional_v1() -> Result<()> { round_trip( 2, true, @@ -128,7 +128,7 @@ fn test_utf8_optional_v1() -> Result<()> { } #[test] -fn test_utf8_required_v1() -> Result<()> { +fn utf8_required_v1() -> Result<()> { round_trip( 2, false, @@ -140,7 +140,7 @@ fn test_utf8_required_v1() -> Result<()> { } #[test] -fn test_utf8_optional_v2() -> Result<()> { +fn utf8_optional_v2() -> Result<()> { round_trip( 2, true, @@ -152,7 +152,7 @@ fn test_utf8_optional_v2() -> Result<()> { } #[test] -fn test_utf8_required_v2() -> Result<()> { +fn utf8_required_v2() -> Result<()> { round_trip( 2, false, @@ -164,7 +164,7 @@ fn test_utf8_required_v2() -> Result<()> { } #[test] -fn test_utf8_optional_v2_compressed() -> Result<()> { +fn utf8_optional_v2_compressed() -> Result<()> { round_trip( 2, true, @@ -176,7 +176,7 @@ fn test_utf8_optional_v2_compressed() -> Result<()> { } #[test] -fn test_utf8_required_v2_compressed() -> Result<()> { +fn utf8_required_v2_compressed() -> Result<()> { round_trip( 2, false, @@ -188,7 +188,7 @@ fn test_utf8_required_v2_compressed() -> Result<()> { } #[test] -fn test_bool_optional_v1() -> Result<()> { +fn bool_optional_v1() -> Result<()> { round_trip( 3, true, @@ -200,7 +200,7 @@ fn test_bool_optional_v1() -> Result<()> { } #[test] -fn test_bool_required_v1() -> Result<()> { +fn bool_required_v1() -> Result<()> { round_trip( 3, false, @@ -212,7 +212,7 @@ fn test_bool_required_v1() -> Result<()> { } #[test] -fn test_bool_optional_v2_uncompressed() -> Result<()> { +fn bool_optional_v2_uncompressed() -> Result<()> { round_trip( 3, true, @@ -224,7 +224,7 @@ fn test_bool_optional_v2_uncompressed() -> Result<()> { } #[test] -fn test_bool_required_v2_uncompressed() -> Result<()> { +fn bool_required_v2_uncompressed() -> Result<()> { round_trip( 3, false, @@ -236,7 +236,7 @@ fn test_bool_required_v2_uncompressed() -> Result<()> { } #[test] -fn test_bool_required_v2_compressed() -> Result<()> { +fn bool_required_v2_compressed() -> Result<()> { round_trip( 3, false, @@ -248,7 +248,7 @@ fn test_bool_required_v2_compressed() -> Result<()> { } #[test] -fn test_list_int64_optional_v2() -> Result<()> { +fn list_int64_optional_v2() -> Result<()> { round_trip( 0, true, @@ -260,7 +260,7 @@ fn test_list_int64_optional_v2() -> Result<()> { } #[test] -fn test_list_int64_optional_v1() -> Result<()> { +fn list_int64_optional_v1() -> Result<()> { round_trip( 0, true, @@ -272,7 +272,7 @@ fn test_list_int64_optional_v1() -> Result<()> { } #[test] -fn test_list_bool_optional_v2() -> Result<()> { +fn list_bool_optional_v2() -> Result<()> { round_trip( 4, true, @@ -284,7 +284,7 @@ fn test_list_bool_optional_v2() -> Result<()> { } #[test] -fn test_list_bool_optional_v1() -> Result<()> { +fn list_bool_optional_v1() -> Result<()> { round_trip( 4, true, @@ -296,7 +296,7 @@ fn test_list_bool_optional_v1() -> Result<()> { } #[test] -fn test_list_utf8_optional_v2() -> Result<()> { +fn list_utf8_optional_v2() -> Result<()> { round_trip( 5, true, @@ -308,7 +308,7 @@ fn test_list_utf8_optional_v2() -> Result<()> { } #[test] -fn test_list_utf8_optional_v1() -> Result<()> { +fn list_utf8_optional_v1() -> Result<()> { round_trip( 5, true, @@ -320,7 +320,7 @@ fn test_list_utf8_optional_v1() -> Result<()> { } #[test] -fn test_list_large_binary_optional_v2() -> Result<()> { +fn list_large_binary_optional_v2() -> Result<()> { round_trip( 6, true, @@ -332,7 +332,7 @@ fn test_list_large_binary_optional_v2() -> Result<()> { } #[test] -fn test_list_large_binary_optional_v1() -> Result<()> { +fn list_large_binary_optional_v1() -> Result<()> { round_trip( 6, true, @@ -344,7 +344,7 @@ fn test_list_large_binary_optional_v1() -> Result<()> { } #[test] -fn test_utf8_optional_v2_delta() -> Result<()> { +fn utf8_optional_v2_delta() -> Result<()> { round_trip( 2, true, @@ -356,7 +356,7 @@ fn test_utf8_optional_v2_delta() -> Result<()> { } #[test] -fn test_i32_optional_v2_dict() -> Result<()> { +fn i32_optional_v2_dict() -> Result<()> { round_trip( 6, true, diff --git a/tests/it/io/print.rs b/tests/it/io/print.rs index ad15ee6dcd1..bb1618d3327 100644 --- a/tests/it/io/print.rs +++ b/tests/it/io/print.rs @@ -6,7 +6,7 @@ use arrow2::{ }; #[test] -fn test_write() -> Result<()> { +fn write_basics() -> Result<()> { // define a schema. let schema = Arc::new(Schema::new(vec![ Field::new("a", DataType::Utf8, true), @@ -48,7 +48,7 @@ fn test_write() -> Result<()> { } #[test] -fn test_write_null() -> Result<()> { +fn write_null() -> Result<()> { let schema = Arc::new(Schema::new(vec![ Field::new("a", DataType::Utf8, true), Field::new("b", DataType::Int32, true), @@ -85,7 +85,7 @@ fn test_write_null() -> Result<()> { } #[test] -fn test_write_dictionary() -> Result<()> { +fn write_dictionary() -> Result<()> { // define a schema. let field_type = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)); let schema = Arc::new(Schema::new(vec![Field::new("d1", field_type, true)])); @@ -140,7 +140,7 @@ macro_rules! check_datetime { } #[test] -fn test_write_timestamp_second() { +fn write_timestamp_second() { let expected = vec![ "+---------------------+", "| f |", @@ -158,7 +158,7 @@ fn test_write_timestamp_second() { } #[test] -fn test_write_timestamp_second_with_tz() { +fn write_timestamp_second_with_tz() { let expected = vec![ "+-------------------------+", "| f |", @@ -176,7 +176,7 @@ fn test_write_timestamp_second_with_tz() { } #[test] -fn test_write_timestamp_millisecond() { +fn write_timestamp_millisecond() { let expected = vec![ "+-------------------------+", "| f |", @@ -194,7 +194,7 @@ fn test_write_timestamp_millisecond() { } #[test] -fn test_write_timestamp_microsecond() { +fn write_timestamp_microsecond() { let expected = vec![ "+----------------------------+", "| f |", @@ -212,7 +212,7 @@ fn test_write_timestamp_microsecond() { } #[test] -fn test_write_timestamp_nanosecond() { +fn write_timestamp_nanosecond() { let expected = vec![ "+-------------------------------+", "| f |", @@ -230,7 +230,7 @@ fn test_write_timestamp_nanosecond() { } #[test] -fn test_write_date_32() { +fn write_date_32() { let expected = vec![ "+------------+", "| f |", @@ -243,7 +243,7 @@ fn test_write_date_32() { } #[test] -fn test_write_date_64() { +fn write_date_64() { let expected = vec![ "+------------+", "| f |", @@ -256,7 +256,7 @@ fn test_write_date_64() { } #[test] -fn test_write_time_32_second() { +fn write_time_32_second() { let expected = vec![ "+----------+", "| f |", @@ -269,7 +269,7 @@ fn test_write_time_32_second() { } #[test] -fn test_write_time_32_millisecond() { +fn write_time_32_millisecond() { let expected = vec![ "+--------------+", "| f |", @@ -287,7 +287,7 @@ fn test_write_time_32_millisecond() { } #[test] -fn test_write_time_64_microsecond() { +fn write_time_64_microsecond() { let expected = vec![ "+-----------------+", "| f |", @@ -305,7 +305,7 @@ fn test_write_time_64_microsecond() { } #[test] -fn test_write_time_64_nanosecond() { +fn write_time_64_nanosecond() { let expected = vec![ "+--------------------+", "| f |", @@ -323,7 +323,7 @@ fn test_write_time_64_nanosecond() { } #[test] -fn test_write_struct() -> Result<()> { +fn write_struct() -> Result<()> { let fields = vec![ Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), @@ -361,7 +361,7 @@ fn test_write_struct() -> Result<()> { } #[test] -fn test_write_union() -> Result<()> { +fn write_union() -> Result<()> { let fields = vec![ Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), diff --git a/tests/it/main.rs b/tests/it/main.rs index c7074cf22d0..9cd03e6b78a 100644 --- a/tests/it/main.rs +++ b/tests/it/main.rs @@ -1,4 +1,5 @@ mod alloc; +mod array; mod bitmap; mod buffer; mod ffi; diff --git a/tests/it/record_batch.rs b/tests/it/record_batch.rs new file mode 100644 index 00000000000..b9ca197c060 --- /dev/null +++ b/tests/it/record_batch.rs @@ -0,0 +1,112 @@ +use arrow2::array::*; +use arrow2::datatypes::*; +use arrow2::record_batch::RecordBatch; + +#[test] +fn basic() { + let schema = Schema::new(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Utf8, false), + ]); + + let a = Int32Array::from_slice(&[1, 2, 3, 4, 5]); + let b = Utf8Array::::from_slice(&["a", "b", "c", "d", "e"]); + + let record_batch = + RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap(); + check_batch(record_batch) +} + +fn check_batch(record_batch: RecordBatch) { + assert_eq!(5, record_batch.num_rows()); + assert_eq!(2, record_batch.num_columns()); + assert_eq!(&DataType::Int32, record_batch.schema().field(0).data_type()); + assert_eq!(&DataType::Utf8, record_batch.schema().field(1).data_type()); + assert_eq!(5, record_batch.column(0).len()); + assert_eq!(5, record_batch.column(1).len()); +} + +#[test] +fn try_from_iter() { + let a: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(1), + Some(2), + None, + Some(4), + Some(5), + ])); + let b: ArrayRef = Arc::new(Utf8Array::::from_slice(&["a", "b", "c", "d", "e"])); + + let record_batch = + RecordBatch::try_from_iter(vec![("a", a), ("b", b)]).expect("valid conversion"); + + let expected_schema = Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Utf8, false), + ]); + assert_eq!(record_batch.schema().as_ref(), &expected_schema); + check_batch(record_batch); +} + +#[test] +fn try_from_iter_with_nullable() { + let a: ArrayRef = Arc::new(Int32Array::from_slice(&[1, 2, 3, 4, 5])); + let b: ArrayRef = Arc::new(Utf8Array::::from_slice(&["a", "b", "c", "d", "e"])); + + // Note there are no nulls in a or b, but we specify that b is nullable + let record_batch = + RecordBatch::try_from_iter_with_nullable(vec![("a", a, false), ("b", b, true)]) + .expect("valid conversion"); + + let expected_schema = Schema::new(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Utf8, true), + ]); + assert_eq!(record_batch.schema().as_ref(), &expected_schema); + check_batch(record_batch); +} + +#[test] +fn type_mismatch() { + let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); + + let a = Int64Array::from_slice(&[1, 2, 3, 4, 5]); + + let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]); + assert!(batch.is_err()); +} + +#[test] +fn number_of_fields_mismatch() { + let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); + + let a = Int32Array::from_slice(&[1, 2, 3, 4, 5]); + let b = Int32Array::from_slice(&[1, 2, 3, 4, 5]); + + let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]); + assert!(batch.is_err()); +} + +#[test] +fn from_struct_array() { + let boolean = Arc::new(BooleanArray::from_slice(&[false, false, true, true])) as ArrayRef; + let int = Arc::new(Int32Array::from_slice(&[42, 28, 19, 31])) as ArrayRef; + let struct_array = StructArray::from_data( + vec![ + Field::new("b", DataType::Boolean, false), + Field::new("c", DataType::Int32, false), + ], + vec![boolean.clone(), int.clone()], + None, + ); + + let batch = RecordBatch::from(&struct_array); + assert_eq!(2, batch.num_columns()); + assert_eq!(4, batch.num_rows()); + assert_eq!( + struct_array.data_type(), + &DataType::Struct(batch.schema().fields().to_vec()) + ); + assert_eq!(boolean.as_ref(), batch.column(0).as_ref()); + assert_eq!(int.as_ref(), batch.column(1).as_ref()); +}