From 104ecd689475aa100e690baa0f3d00a85eaad879 Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Tue, 19 Oct 2021 16:22:40 +0000 Subject: [PATCH] Simplified offset handling. --- .../tests/test_sql.py | 71 +++++++++++++++---- src/array/README.md | 11 --- src/array/binary/ffi.rs | 16 +++-- src/array/binary/mod.rs | 4 -- src/array/boolean/ffi.rs | 11 +-- src/array/boolean/mod.rs | 3 - src/array/dictionary/ffi.rs | 5 -- src/array/dictionary/mod.rs | 4 -- src/array/ffi.rs | 3 - src/array/fixed_size_binary/mod.rs | 20 +++--- src/array/fixed_size_list/mod.rs | 7 -- src/array/list/ffi.rs | 16 +++-- src/array/list/mod.rs | 3 - src/array/map/ffi.rs | 4 -- src/array/map/mod.rs | 3 - src/array/null.rs | 15 +--- src/array/primitive/ffi.rs | 2 - src/array/struct_.rs | 5 -- src/array/union/ffi.rs | 4 -- src/array/union/mod.rs | 5 -- src/array/utf8/ffi.rs | 15 ++-- src/array/utf8/mod.rs | 4 -- src/buffer/immutable.rs | 12 +++- src/buffer/mutable.rs | 2 +- src/ffi/ffi.rs | 12 ++-- 25 files changed, 128 insertions(+), 129 deletions(-) diff --git a/arrow-pyarrow-integration-testing/tests/test_sql.py b/arrow-pyarrow-integration-testing/tests/test_sql.py index 7a5363cb291..27da0e63515 100644 --- a/arrow-pyarrow-integration-testing/tests/test_sql.py +++ b/arrow-pyarrow-integration-testing/tests/test_sql.py @@ -39,27 +39,46 @@ def tearDown(self): # No leak of C++ memory self.assertEqual(self.old_allocated_cpp, pyarrow.total_allocated_bytes()) + def test_primitive(self): + a = pyarrow.array([0, None, 2, 3, 4]) + b = arrow_pyarrow_integration_testing.round_trip_array(a) + + b.validate(full=True) + assert a.to_pylist() == b.to_pylist() + assert a.type == b.type + def test_primitive_sliced(self): - a = pyarrow.array([0, None, 2, 3, 4]).slice(1, 1) + a = pyarrow.array([0, None, 2, 3, 4]).slice(1, 2) b = arrow_pyarrow_integration_testing.round_trip_array(a) b.validate(full=True) - print(a.to_pylist()) - print(b.to_pylist()) assert a.to_pylist() == b.to_pylist() assert a.type == b.type - def test_string_roundtrip(self): - """ - Python -> Rust -> Python - """ + def test_boolean(self): + a = pyarrow.array([True, None, False, True, False]) + b = arrow_pyarrow_integration_testing.round_trip_array(a) + + b.validate(full=True) + assert a.to_pylist() == b.to_pylist() + assert a.type == b.type + + def test_boolean_sliced(self): + a = pyarrow.array([True, None, False, True, False]).slice(1, 2) + b = arrow_pyarrow_integration_testing.round_trip_array(a) + + b.validate(full=True) + assert a.to_pylist() == b.to_pylist() + assert a.type == b.type + + def test_string(self): a = pyarrow.array(["a", None, "ccc"]) b = arrow_pyarrow_integration_testing.round_trip_array(a) c = pyarrow.array(["a", None, "ccc"]) self.assertEqual(b, c) def test_string_sliced(self): - a = pyarrow.array(["a", None, "ccc"]).slice(1, 1) + a = pyarrow.array(["a", None, "ccc"]).slice(1, 2) b = arrow_pyarrow_integration_testing.round_trip_array(a) b.validate(full=True) @@ -93,10 +112,17 @@ def test_list_array(self): assert a.to_pylist() == b.to_pylist() assert a.type == b.type - def test_struct_array(self): - """ - Python -> Rust -> Python - """ + def test_list_sliced(self): + a = pyarrow.array( + [[], None, [1, 2], [4, 5, 6]], pyarrow.list_(pyarrow.int64()) + ).slice(1, 2) + b = arrow_pyarrow_integration_testing.round_trip_array(a) + + b.validate(full=True) + assert a.to_pylist() == b.to_pylist() + assert a.type == b.type + + def test_struct(self): fields = [ ("f1", pyarrow.int32()), ("f2", pyarrow.string()), @@ -117,6 +143,27 @@ def test_struct_array(self): assert a.to_pylist() == b.to_pylist() assert a.type == b.type + # see https://issues.apache.org/jira/browse/ARROW-14383 + def _test_struct_sliced(self): + fields = [ + ("f1", pyarrow.int32()), + ("f2", pyarrow.string()), + ] + a = pyarrow.array( + [ + {"f1": 1, "f2": "a"}, + None, + {"f1": 3, "f2": None}, + {"f1": None, "f2": "d"}, + {"f1": None, "f2": None}, + ], + pyarrow.struct(fields), + ).slice(1, 2) + b = arrow_pyarrow_integration_testing.round_trip_array(a) + b.validate(full=True) + assert a.to_pylist() == b.to_pylist() + assert a.type == b.type + def test_list_list_array(self): """ Python -> Rust -> Python diff --git a/src/array/README.md b/src/array/README.md index a814839527c..9282cf611c6 100644 --- a/src/array/README.md +++ b/src/array/README.md @@ -41,17 +41,6 @@ This document describes the overall design of this module. * `from_trusted_len_values_iter` from an iterator of trusted len of values * `try_from_trusted_len_iter` from an fallible iterator of trusted len of optional values -### Slot offsets - -* An array MUST have a `offset: usize` measuring the number of slots that the array is currently offsetted by if the specification requires. - -* An array MUST implement `fn slice(&self, offset: usize, length: usize) -> Self` that returns an offseted and/or truncated clone of the array. This function MUST increase the array's offset if it exists. - -* Conversely, `offset` MUST only be changed by `slice`. - -The rational of the above is that it enable us to be fully interoperable with the offset logic supported by the C data interface, while at the same time easily perform array slices -within Rust's type safety mechanism. - ### Mutable Arrays * An array MAY have a mutable counterpart. E.g. `MutablePrimitiveArray` is the mutable counterpart of `PrimitiveArray`. diff --git a/src/array/binary/ffi.rs b/src/array/binary/ffi.rs index d3856a4cfb3..c4c0f698543 100644 --- a/src/array/binary/ffi.rs +++ b/src/array/binary/ffi.rs @@ -10,17 +10,21 @@ use super::BinaryArray; unsafe impl ToFfi for BinaryArray { fn buffers(&self) -> Vec>> { + let offset = self + .validity + .as_ref() + .map(|x| x.offset()) + .unwrap_or_default(); + + assert!(self.offsets.offset() >= offset); vec![ self.validity.as_ref().map(|x| x.as_ptr()), - std::ptr::NonNull::new(self.offsets.as_ptr() as *mut u8), + std::ptr::NonNull::new(unsafe { + self.offsets.as_ptr().offset(-(offset as isize)) as *mut u8 + }), std::ptr::NonNull::new(self.values.as_ptr() as *mut u8), ] } - - #[inline] - fn offset(&self) -> usize { - self.offset - } } impl FromFfi for BinaryArray { diff --git a/src/array/binary/mod.rs b/src/array/binary/mod.rs index 036e3f74122..ed446561dde 100644 --- a/src/array/binary/mod.rs +++ b/src/array/binary/mod.rs @@ -23,7 +23,6 @@ pub struct BinaryArray { offsets: Buffer, values: Buffer, validity: Option, - offset: usize, } // constructors @@ -71,7 +70,6 @@ impl BinaryArray { offsets, values, validity, - offset: 0, } } @@ -113,7 +111,6 @@ impl BinaryArray { offsets, values, validity, - offset: 0, } } @@ -146,7 +143,6 @@ impl BinaryArray { offsets, values: self.values.clone(), validity, - offset: self.offset + offset, } } diff --git a/src/array/boolean/ffi.rs b/src/array/boolean/ffi.rs index 8d968820162..a69c81a461b 100644 --- a/src/array/boolean/ffi.rs +++ b/src/array/boolean/ffi.rs @@ -10,15 +10,18 @@ use super::BooleanArray; unsafe impl ToFfi for BooleanArray { fn buffers(&self) -> Vec>> { + let offset = self + .validity + .as_ref() + .map(|x| x.offset()) + .unwrap_or_default(); + + assert!(self.values.offset() >= offset); vec![ self.validity.as_ref().map(|x| x.as_ptr()), Some(self.values.as_ptr()), ] } - - fn offset(&self) -> usize { - self.offset - } } impl FromFfi for BooleanArray { diff --git a/src/array/boolean/mod.rs b/src/array/boolean/mod.rs index 065eb199775..06935af24ee 100644 --- a/src/array/boolean/mod.rs +++ b/src/array/boolean/mod.rs @@ -20,7 +20,6 @@ pub struct BooleanArray { data_type: DataType, values: Bitmap, validity: Option, - offset: usize, } impl BooleanArray { @@ -50,7 +49,6 @@ impl BooleanArray { data_type, values, validity, - offset: 0, } } @@ -83,7 +81,6 @@ impl BooleanArray { data_type: self.data_type.clone(), values: self.values.clone().slice_unchecked(offset, length), validity, - offset: self.offset + offset, } } diff --git a/src/array/dictionary/ffi.rs b/src/array/dictionary/ffi.rs index 6d1001779a1..5a288afb6eb 100644 --- a/src/array/dictionary/ffi.rs +++ b/src/array/dictionary/ffi.rs @@ -10,11 +10,6 @@ unsafe impl ToFfi for DictionaryArray { fn buffers(&self) -> Vec>> { self.keys.buffers() } - - #[inline] - fn offset(&self) -> usize { - self.offset - } } impl FromFfi for DictionaryArray { diff --git a/src/array/dictionary/mod.rs b/src/array/dictionary/mod.rs index 3b33cb3e17b..012d937d6d9 100644 --- a/src/array/dictionary/mod.rs +++ b/src/array/dictionary/mod.rs @@ -37,7 +37,6 @@ pub struct DictionaryArray { data_type: DataType, keys: PrimitiveArray, values: Arc, - offset: usize, } impl DictionaryArray { @@ -69,7 +68,6 @@ impl DictionaryArray { data_type, keys, values, - offset: 0, } } @@ -81,7 +79,6 @@ impl DictionaryArray { data_type: self.data_type.clone(), keys: self.keys.clone().slice(offset, length), values: self.values.clone(), - offset: self.offset + offset, } } @@ -93,7 +90,6 @@ impl DictionaryArray { data_type: self.data_type.clone(), keys: self.keys.clone().slice_unchecked(offset, length), values: self.values.clone(), - offset: self.offset + offset, } } diff --git a/src/array/ffi.rs b/src/array/ffi.rs index 176b0966ff5..9ed0bfca080 100644 --- a/src/array/ffi.rs +++ b/src/array/ffi.rs @@ -13,9 +13,6 @@ pub unsafe trait ToFfi { /// The pointers to the buffers. fn buffers(&self) -> Vec>>; - /// The offset - fn offset(&self) -> usize; - /// The children fn children(&self) -> Vec> { vec![] diff --git a/src/array/fixed_size_binary/mod.rs b/src/array/fixed_size_binary/mod.rs index 3091aae3b62..a6a02488136 100644 --- a/src/array/fixed_size_binary/mod.rs +++ b/src/array/fixed_size_binary/mod.rs @@ -14,7 +14,6 @@ pub struct FixedSizeBinaryArray { data_type: DataType, values: Buffer, validity: Option, - offset: usize, } impl FixedSizeBinaryArray { @@ -47,7 +46,6 @@ impl FixedSizeBinaryArray { data_type, values, validity, - offset: 0, } } @@ -83,7 +81,6 @@ impl FixedSizeBinaryArray { size: self.size, values, validity, - offset: self.offset + offset, } } @@ -184,15 +181,22 @@ impl std::fmt::Display for FixedSizeBinaryArray { unsafe impl ToFfi for FixedSizeBinaryArray { fn buffers(&self) -> Vec>> { + let offset = self + .validity + .as_ref() + .map(|x| x.offset()) + .unwrap_or_default(); + let offset = offset * self.size(); + + assert!(self.values.offset() >= offset); + vec![ self.validity.as_ref().map(|x| x.as_ptr()), - std::ptr::NonNull::new(self.values.as_ptr() as *mut u8), + std::ptr::NonNull::new( + unsafe { self.values.as_ptr().offset(-(offset as isize)) } as *mut u8 + ), ] } - - fn offset(&self) -> usize { - self.offset - } } impl FixedSizeBinaryArray { diff --git a/src/array/fixed_size_list/mod.rs b/src/array/fixed_size_list/mod.rs index 5b78569eed4..86ecbec3758 100644 --- a/src/array/fixed_size_list/mod.rs +++ b/src/array/fixed_size_list/mod.rs @@ -20,7 +20,6 @@ pub struct FixedSizeListArray { data_type: DataType, values: Arc, validity: Option, - offset: usize, } impl FixedSizeListArray { @@ -60,7 +59,6 @@ impl FixedSizeListArray { data_type, values, validity, - offset: 0, } } @@ -97,7 +95,6 @@ impl FixedSizeListArray { size: self.size, values, validity, - offset: self.offset + offset, } } @@ -200,10 +197,6 @@ unsafe impl ToFfi for FixedSizeListArray { vec![self.validity.as_ref().map(|x| x.as_ptr())] } - fn offset(&self) -> usize { - self.offset - } - fn children(&self) -> Vec> { vec![self.values().clone()] } diff --git a/src/array/list/ffi.rs b/src/array/list/ffi.rs index e2bf65514ba..c17a5dbd43c 100644 --- a/src/array/list/ffi.rs +++ b/src/array/list/ffi.rs @@ -7,16 +7,22 @@ use super::ListArray; unsafe impl ToFfi for ListArray { fn buffers(&self) -> Vec>> { + let offset = self + .validity + .as_ref() + .map(|x| x.offset()) + .unwrap_or_default(); + + assert!(self.offsets.offset() >= offset); + vec![ self.validity.as_ref().map(|x| x.as_ptr()), - std::ptr::NonNull::new(self.offsets.as_ptr() as *mut u8), + std::ptr::NonNull::new( + unsafe { self.offsets.as_ptr().offset(-(offset as isize)) } as *mut u8 + ), ] } - fn offset(&self) -> usize { - self.offset - } - fn children(&self) -> Vec> { vec![self.values.clone()] } diff --git a/src/array/list/mod.rs b/src/array/list/mod.rs index 4a92c352fa3..89566571dcc 100644 --- a/src/array/list/mod.rs +++ b/src/array/list/mod.rs @@ -25,7 +25,6 @@ pub struct ListArray { offsets: Buffer, values: Arc, validity: Option, - offset: usize, } impl ListArray { @@ -78,7 +77,6 @@ impl ListArray { offsets, values, validity, - offset: 0, } } @@ -107,7 +105,6 @@ impl ListArray { offsets, values: self.values.clone(), validity, - offset: self.offset + offset, } } diff --git a/src/array/map/ffi.rs b/src/array/map/ffi.rs index 822acfaf209..953d5dd0d8e 100644 --- a/src/array/map/ffi.rs +++ b/src/array/map/ffi.rs @@ -13,10 +13,6 @@ unsafe impl ToFfi for MapArray { ] } - fn offset(&self) -> usize { - self.offset - } - fn children(&self) -> Vec> { vec![self.field.clone()] } diff --git a/src/array/map/mod.rs b/src/array/map/mod.rs index 0efc83e7a4e..ee908be847f 100644 --- a/src/array/map/mod.rs +++ b/src/array/map/mod.rs @@ -22,7 +22,6 @@ pub struct MapArray { field: Arc, // invariant: offsets.len() - 1 == Bitmap::len() validity: Option, - offset: usize, } impl MapArray { @@ -81,7 +80,6 @@ impl MapArray { data_type, field, offsets, - offset: 0, validity, } } @@ -111,7 +109,6 @@ impl MapArray { offsets, field: self.field.clone(), validity, - offset: self.offset + offset, } } } diff --git a/src/array/null.rs b/src/array/null.rs index 1906cc140f4..e20f754cc75 100644 --- a/src/array/null.rs +++ b/src/array/null.rs @@ -7,7 +7,6 @@ use super::{ffi::ToFfi, Array}; pub struct NullArray { data_type: DataType, length: usize, - offset: usize, } impl NullArray { @@ -23,19 +22,14 @@ impl NullArray { /// Returns a new [`NullArray`]. pub fn from_data(data_type: DataType, length: usize) -> Self { - Self { - data_type, - length, - offset: 0, - } + Self { data_type, length } } /// Returns a slice of the [`NullArray`]. - pub fn slice(&self, offset: usize, length: usize) -> Self { + pub fn slice(&self, _offset: usize, length: usize) -> Self { Self { data_type: self.data_type.clone(), length, - offset: self.offset + offset, } } } @@ -81,9 +75,4 @@ unsafe impl ToFfi for NullArray { fn buffers(&self) -> Vec>> { vec![] } - - #[inline] - fn offset(&self) -> usize { - self.offset - } } diff --git a/src/array/primitive/ffi.rs b/src/array/primitive/ffi.rs index a7bb6add8af..f009a2fb832 100644 --- a/src/array/primitive/ffi.rs +++ b/src/array/primitive/ffi.rs @@ -28,8 +28,6 @@ impl FromFfi for PrimitiveArray { let validity = unsafe { array.validity() }?; let values = unsafe { array.buffer::(0) }?; - println!("{:?}", validity); - Ok(Self::from_data(data_type, values, validity)) } } diff --git a/src/array/struct_.rs b/src/array/struct_.rs index 03c1fb5487c..d7e504436b1 100644 --- a/src/array/struct_.rs +++ b/src/array/struct_.rs @@ -223,11 +223,6 @@ unsafe impl ToFfi for StructArray { vec![self.validity.as_ref().map(|x| x.as_ptr())] } - fn offset(&self) -> usize { - // we do not support offsets in structs. Instead, if an FFI we slice the incoming arrays - 0 - } - fn children(&self) -> Vec> { self.values.clone() } diff --git a/src/array/union/ffi.rs b/src/array/union/ffi.rs index b9360478b1e..7f9dfbb7273 100644 --- a/src/array/union/ffi.rs +++ b/src/array/union/ffi.rs @@ -18,10 +18,6 @@ unsafe impl ToFfi for UnionArray { } } - fn offset(&self) -> usize { - self.offset - } - fn children(&self) -> Vec> { self.fields.clone() } diff --git a/src/array/union/mod.rs b/src/array/union/mod.rs index b5a6a1d2605..026db7eb684 100644 --- a/src/array/union/mod.rs +++ b/src/array/union/mod.rs @@ -31,7 +31,6 @@ pub struct UnionArray { fields: Vec>, offsets: Option>, data_type: DataType, - offset: usize, } impl UnionArray { @@ -78,7 +77,6 @@ impl UnionArray { fields, offsets, types: Buffer::new(), - offset: 0, } } else { panic!("Union struct must be created with the corresponding Union DataType") @@ -125,7 +123,6 @@ impl UnionArray { fields, offsets, types, - offset: 0, } } @@ -193,7 +190,6 @@ impl UnionArray { fields_hash: self.fields_hash.clone(), types: self.types.clone().slice(offset, length), offsets: self.offsets.clone(), - offset: self.offset + offset, } } @@ -210,7 +206,6 @@ impl UnionArray { fields_hash: self.fields_hash.clone(), types: self.types.clone().slice_unchecked(offset, length), offsets: self.offsets.clone(), - offset: self.offset + offset, } } } diff --git a/src/array/utf8/ffi.rs b/src/array/utf8/ffi.rs index 9205ac84e1f..118417fc29b 100644 --- a/src/array/utf8/ffi.rs +++ b/src/array/utf8/ffi.rs @@ -8,16 +8,21 @@ use super::Utf8Array; unsafe impl ToFfi for Utf8Array { fn buffers(&self) -> Vec>> { + let offset = self + .validity + .as_ref() + .map(|x| x.offset()) + .unwrap_or_default(); + + assert!(self.offsets.offset() >= offset); vec![ self.validity.as_ref().map(|x| x.as_ptr()), - std::ptr::NonNull::new(self.offsets.as_ptr() as *mut u8), + std::ptr::NonNull::new(unsafe { + self.offsets.as_ptr().offset(-(offset as isize)) as *mut u8 + }), std::ptr::NonNull::new(self.values.as_ptr() as *mut u8), ] } - - fn offset(&self) -> usize { - self.offset - } } impl FromFfi for Utf8Array { diff --git a/src/array/utf8/mod.rs b/src/array/utf8/mod.rs index 1d6b75faec3..ef895092dc9 100644 --- a/src/array/utf8/mod.rs +++ b/src/array/utf8/mod.rs @@ -36,7 +36,6 @@ pub struct Utf8Array { offsets: Buffer, values: Buffer, validity: Option, - offset: usize, } impl Utf8Array { @@ -86,7 +85,6 @@ impl Utf8Array { offsets, values, validity, - offset: 0, } } @@ -128,7 +126,6 @@ impl Utf8Array { offsets, values, validity, - offset: 0, } } @@ -161,7 +158,6 @@ impl Utf8Array { offsets, values: self.values.clone(), validity, - offset: self.offset + offset, } } diff --git a/src/buffer/immutable.rs b/src/buffer/immutable.rs index eab542e67e3..5367b03443a 100644 --- a/src/buffer/immutable.rs +++ b/src/buffer/immutable.rs @@ -63,11 +63,11 @@ impl Buffer { } /// Auxiliary method to create a new Buffer - pub(crate) fn from_bytes(bytes: Bytes) -> Self { - let length = bytes.len(); + pub(crate) fn from_bytes(bytes: Bytes, offset: usize) -> Self { + let length = bytes.len() - offset; Buffer { data: Arc::new(bytes), - offset: 0, + offset, length, } } @@ -120,6 +120,12 @@ impl Buffer { pub fn as_ptr(&self) -> *const T { unsafe { self.data.ptr().as_ptr().add(self.offset) } } + + /// Returns the offset + #[inline] + pub(crate) fn offset(&self) -> usize { + self.offset + } } impl Buffer { diff --git a/src/buffer/mutable.rs b/src/buffer/mutable.rs index 70c4e4415d0..b7790e8ec79 100644 --- a/src/buffer/mutable.rs +++ b/src/buffer/mutable.rs @@ -431,7 +431,7 @@ impl> From

for MutableBuffer { impl From> for Buffer { #[inline] fn from(buffer: MutableBuffer) -> Self { - Self::from_bytes(buffer.into()) + Self::from_bytes(buffer.into(), 0) } } diff --git a/src/ffi/ffi.rs b/src/ffi/ffi.rs index 9bab21c9035..f4960520594 100644 --- a/src/ffi/ffi.rs +++ b/src/ffi/ffi.rs @@ -190,12 +190,12 @@ unsafe fn create_buffer( assert!(index < array.n_buffers as usize); let ptr = *buffers.add(index); - let ptr = NonNull::new(ptr as *mut T).and_then(|x| NonNull::new(x.as_ptr().add(offset))); + let ptr = NonNull::new(ptr as *mut T); let bytes = ptr - .map(|ptr| Bytes::new(ptr, len, deallocation)) + .map(|ptr| Bytes::new(ptr, offset + len, deallocation)) .ok_or_else(|| ArrowError::Ffi(format!("The buffer at position {} is null", index))); - bytes.map(Buffer::from_bytes) + bytes.map(|x| Buffer::from_bytes(x, offset)) } /// returns a new buffer corresponding to the index `i` of the FFI array. It may not exist (null pointer). @@ -266,22 +266,24 @@ fn buffer_len(array: &Ffi_ArrowArray, data_type: &DataType, i: usize) -> Result< (PhysicalType::Utf8, 2) | (PhysicalType::Binary, 2) => { // the len of the data buffer (buffer 2) equals the last value of the offset buffer (buffer 1) let len = buffer_len(array, data_type, 1)?; + let offset = buffer_offset(array, data_type, 1); // first buffer is the null buffer => add(1) let offset_buffer = unsafe { *(array.buffers as *mut *const u8).add(1) }; // interpret as i32 let offset_buffer = offset_buffer as *const i32; // get last offset - (unsafe { *offset_buffer.add(len - 1) }) as usize + (unsafe { *offset_buffer.add(offset).add(len - 1) }) as usize } (PhysicalType::LargeUtf8, 2) | (PhysicalType::LargeBinary, 2) => { // the len of the data buffer (buffer 2) equals the last value of the offset buffer (buffer 1) let len = buffer_len(array, data_type, 1)?; + let offset = buffer_offset(array, data_type, 1); // first buffer is the null buffer => add(1) let offset_buffer = unsafe { *(array.buffers as *mut *const u8).add(1) }; // interpret as i64 let offset_buffer = offset_buffer as *const i64; // get last offset - (unsafe { *offset_buffer.add(len - 1) }) as usize + (unsafe { *offset_buffer.add(offset).add(len - 1) }) as usize } // buffer len of primitive types _ => array.length as usize,