Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
More tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
jorgecarleitao committed Oct 23, 2021
1 parent 104ecd6 commit 5510fe5
Show file tree
Hide file tree
Showing 12 changed files with 100 additions and 62 deletions.
9 changes: 5 additions & 4 deletions src/array/binary/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,13 @@ unsafe impl<O: Offset> ToFfi for BinaryArray<O> {
.map(|x| x.offset())
.unwrap_or_default();

assert!(self.offsets.offset() >= offset);
let offsets = std::ptr::NonNull::new(unsafe {
self.offsets.as_ptr().offset(-(offset as isize)) as *mut u8
});

vec![
self.validity.as_ref().map(|x| x.as_ptr()),
std::ptr::NonNull::new(unsafe {
self.offsets.as_ptr().offset(-(offset as isize)) as *mut u8
}),
offsets,
std::ptr::NonNull::new(self.values.as_ptr() as *mut u8),
]
}
Expand Down
6 changes: 6 additions & 0 deletions src/array/boolean/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ impl BooleanArray {
if data_type.to_physical_type() != PhysicalType::Boolean {
panic!("BooleanArray can only be initialized with DataType::Boolean")
}
if matches!(&validity, Some(bitmap) if bitmap.offset() != values.offset()) {
panic!("validity's bitmap offset must equal values offset")
}
Self {
data_type,
values,
Expand Down Expand Up @@ -91,6 +94,9 @@ impl BooleanArray {
if matches!(&validity, Some(bitmap) if bitmap.len() != self.len()) {
panic!("validity should be as least as large as the array")
}
if matches!(&validity, Some(bitmap) if bitmap.offset() != self.values.offset()) {
panic!("validity's bitmap offset must equal values offset")
}
let mut arr = self.clone();
arr.validity = validity;
arr
Expand Down
23 changes: 23 additions & 0 deletions src/array/fixed_size_binary/ffi.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
use crate::array::ffi::ToFfi;

use super::FixedSizeBinaryArray;

unsafe impl ToFfi for FixedSizeBinaryArray {
fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
let offset = self
.validity
.as_ref()
.map(|x| x.offset())
.unwrap_or_default();
let offset = offset * self.size();

// note that this may point to _before_ the pointer. This is fine because the C data interface
// requires users to only access past the offset
let values =
std::ptr::NonNull::new(
unsafe { self.values.as_ptr().offset(-(offset as isize)) } as *mut u8
);

vec![self.validity.as_ref().map(|x| x.as_ptr()), values]
}
}
23 changes: 2 additions & 21 deletions src/array/fixed_size_binary/mod.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
use crate::{bitmap::Bitmap, buffer::Buffer, datatypes::DataType, error::Result};

use super::{display_fmt, display_helper, ffi::ToFfi, Array};
use super::{display_fmt, display_helper, Array};

mod ffi;
mod iterator;
mod mutable;
pub use mutable::*;
Expand Down Expand Up @@ -179,26 +180,6 @@ impl std::fmt::Display for FixedSizeBinaryArray {
}
}

unsafe impl ToFfi for FixedSizeBinaryArray {
fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
let offset = self
.validity
.as_ref()
.map(|x| x.offset())
.unwrap_or_default();
let offset = offset * self.size();

assert!(self.values.offset() >= offset);

vec![
self.validity.as_ref().map(|x| x.as_ptr()),
std::ptr::NonNull::new(
unsafe { self.values.as_ptr().offset(-(offset as isize)) } as *mut u8
),
]
}
}

impl FixedSizeBinaryArray {
/// Creates a [`FixedSizeBinaryArray`] from an fallible iterator of optional `[u8]`.
pub fn try_from_iter<P: AsRef<[u8]>, I: IntoIterator<Item = Option<P>>>(
Expand Down
16 changes: 16 additions & 0 deletions src/array/fixed_size_list/ffi.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
use std::sync::Arc;

use crate::array::ffi::ToFfi;
use crate::array::Array;

use super::FixedSizeListArray;

unsafe impl ToFfi for FixedSizeListArray {
fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
vec![self.validity.as_ref().map(|x| x.as_ptr())]
}

fn children(&self) -> Vec<Arc<dyn Array>> {
vec![self.values().clone()]
}
}
13 changes: 2 additions & 11 deletions src/array/fixed_size_list/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ use crate::{
datatypes::{DataType, Field},
};

use super::{display_fmt, ffi::ToFfi, new_empty_array, new_null_array, Array};
use super::{display_fmt, new_empty_array, new_null_array, Array};

mod ffi;
mod iterator;
pub use iterator::*;
mod mutable;
Expand Down Expand Up @@ -191,13 +192,3 @@ impl std::fmt::Display for FixedSizeListArray {
display_fmt(self.iter(), "FixedSizeListArray", f, true)
}
}

unsafe impl ToFfi for FixedSizeListArray {
fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
vec![self.validity.as_ref().map(|x| x.as_ptr())]
}

fn children(&self) -> Vec<Arc<dyn Array>> {
vec![self.values().clone()]
}
}
12 changes: 6 additions & 6 deletions src/array/list/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ unsafe impl<O: Offset> ToFfi for ListArray<O> {
.map(|x| x.offset())
.unwrap_or_default();

assert!(self.offsets.offset() >= offset);

vec![
self.validity.as_ref().map(|x| x.as_ptr()),
// note that this may point to _before_ the pointer. This is fine because the C data interface
// requires users to only access past the offset
let offsets =
std::ptr::NonNull::new(
unsafe { self.offsets.as_ptr().offset(-(offset as isize)) } as *mut u8
),
]
);

vec![self.validity.as_ref().map(|x| x.as_ptr()), offsets]
}

fn children(&self) -> Vec<Arc<dyn Array>> {
Expand Down
15 changes: 11 additions & 4 deletions src/array/map/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,17 @@ use super::MapArray;

unsafe impl ToFfi for MapArray {
fn buffers(&self) -> Vec<Option<std::ptr::NonNull<u8>>> {
vec![
self.validity.as_ref().map(|x| x.as_ptr()),
std::ptr::NonNull::new(self.offsets.as_ptr() as *mut u8),
]
let offset = self
.validity
.as_ref()
.map(|x| x.offset())
.unwrap_or_default();

let offsets = std::ptr::NonNull::new(unsafe {
self.offsets.as_ptr().offset(-(offset as isize)) as *mut u8
});

vec![self.validity.as_ref().map(|x| x.as_ptr()), offsets]
}

fn children(&self) -> Vec<Arc<dyn Array>> {
Expand Down
5 changes: 2 additions & 3 deletions src/array/specification.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,11 @@ pub fn check_offsets_and_utf8<O: Offset>(offsets: &[O], values: &[u8]) {
}

/// # Panics iff:
/// * the `offsets` is empty, or
/// * the `offsets` is not monotonically increasing, or
/// * any offset is larger or equal to `values_len`.
pub fn check_offsets<O: Offset>(offsets: &[O], values_len: usize) {
if offsets.is_empty() {
return;
}
assert!(!offsets.is_empty());

let mut last = offsets[0];
// assert monotonicity
Expand Down
9 changes: 5 additions & 4 deletions src/array/utf8/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,13 @@ unsafe impl<O: Offset> ToFfi for Utf8Array<O> {
.map(|x| x.offset())
.unwrap_or_default();

assert!(self.offsets.offset() >= offset);
let offsets = std::ptr::NonNull::new(unsafe {
self.offsets.as_ptr().offset(-(offset as isize)) as *mut u8
});

vec![
self.validity.as_ref().map(|x| x.as_ptr()),
std::ptr::NonNull::new(unsafe {
self.offsets.as_ptr().offset(-(offset as isize)) as *mut u8
}),
offsets,
std::ptr::NonNull::new(self.values.as_ptr() as *mut u8),
]
}
Expand Down
6 changes: 0 additions & 6 deletions src/buffer/immutable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -120,12 +120,6 @@ impl<T: NativeType> Buffer<T> {
pub fn as_ptr(&self) -> *const T {
unsafe { self.data.ptr().as_ptr().add(self.offset) }
}

/// Returns the offset
#[inline]
pub(crate) fn offset(&self) -> usize {
self.offset
}
}

impl<T: NativeType> Buffer<T> {
Expand Down
25 changes: 22 additions & 3 deletions tests/it/ffi.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
use arrow2::array::*;
use arrow2::bitmap::Bitmap;
use arrow2::datatypes::{DataType, Field, TimeUnit};
use arrow2::{error::Result, ffi};
use std::collections::BTreeMap;
use std::sync::Arc;

fn test_round_trip(expected: impl Array + Clone + 'static) -> Result<()> {
let array: Arc<dyn Array> = Arc::new(expected.clone());
fn _test_round_trip(array: Arc<dyn Array>, expected: Box<dyn Array>) -> Result<()> {
let field = Field::new("a", array.data_type().clone(), true);
let expected = Box::new(expected) as Box<dyn Array>;

let array_ptr = Box::new(ffi::Ffi_ArrowArray::empty());
let schema_ptr = Box::new(ffi::Ffi_ArrowSchema::empty());
Expand All @@ -32,6 +31,15 @@ fn test_round_trip(expected: impl Array + Clone + 'static) -> Result<()> {
Ok(())
}

fn test_round_trip(expected: impl Array + Clone + 'static) -> Result<()> {
let array: Arc<dyn Array> = Arc::new(expected.clone());
let expected = Box::new(expected) as Box<dyn Array>;
_test_round_trip(array.clone(), clone(expected.as_ref()))?;

// sliced
_test_round_trip(array.slice(1, 2).into(), expected.slice(1, 2))
}

fn test_round_trip_schema(field: Field) -> Result<()> {
// create a `ArrowArray` from the data.
let schema_ptr = Box::new(ffi::Ffi_ArrowSchema::empty());
Expand Down Expand Up @@ -138,6 +146,17 @@ fn list_list() -> Result<()> {
test_round_trip(array)
}

#[test]
fn struct_() -> Result<()> {
let data_type = DataType::Struct(vec![Field::new("a", DataType::Int32, true)]);
let values = vec![Arc::new(Int32Array::from([Some(1), None, Some(3)])) as Arc<dyn Array>];
let validity = Bitmap::from([true, false, true]);

let array = StructArray::from_data(data_type, values, validity.into());

test_round_trip(array)
}

#[test]
fn dict() -> Result<()> {
let data = vec![Some("a"), Some("a"), None, Some("b")];
Expand Down

0 comments on commit 5510fe5

Please sign in to comment.