Decouple buffer deallocation from ffi and allow creating buffers from rust vec

arrow/src/alloc/mod.rs

@@ -19,8 +19,11 @@
 //! regions, cache and allocation alignments.
 
 use std::alloc::{handle_alloc_error, Layout};
+use std::fmt::{Debug, Formatter};
 use std::mem::size_of;
+use std::panic::RefUnwindSafe;
 use std::ptr::NonNull;
+use std::sync::Arc;
 
 mod alignment;
 mod types;
@@ -121,3 +124,30 @@ pub unsafe fn reallocate<T: NativeType>(
         handle_alloc_error(Layout::from_size_align_unchecked(new_size, ALIGNMENT))
     })
 }
+
+/// The owner of an allocation, that is not natively allocated.
+/// The trait implementation is responsible for dropping the allocations once no more references exist.
+pub trait Allocation: RefUnwindSafe {}
+
+impl<T: RefUnwindSafe> Allocation for T {}
+
+/// Mode of deallocating memory regions
+pub enum Deallocation {
+    /// Native deallocation, using Rust deallocator with Arrow-specific memory alignment
+    Native(usize),
+    /// Foreign interface, via a callback
+    Foreign(Arc<dyn Allocation>),
+}
+
+impl Debug for Deallocation {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+        match self {
+            Deallocation::Native(capacity) => {
+                write!(f, "Deallocation::Native {{ capacity: {} }}", capacity)
+            }
+            Deallocation::Foreign(_) => {
+                write!(f, "Deallocation::Foreign {{ capacity: unknown }}")
+            }
+        }
+    }
+}

arrow/src/array/data.rs

@@ -1459,10 +1459,11 @@ impl ArrayDataBuilder {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::ptr::NonNull;
 
     use crate::array::{
-        Array, BooleanBuilder, Int32Array, Int32Builder, Int64Array, StringArray,
-        StructBuilder, UInt64Array,
+        make_array, Array, BooleanBuilder, Int32Array, Int32Builder, Int64Array,
+        StringArray, StructBuilder, UInt64Array,
     };
     use crate::buffer::Buffer;
     use crate::datatypes::Field;
@@ -2594,4 +2595,52 @@ mod tests {
 
         assert_eq!(&struct_array_slice, &cloned);
     }
+
+    #[test]
+    fn test_string_data_from_foreign() {
+        let mut strings = "foobarfoobar".to_owned();
+        let mut offsets = vec![0_i32, 0, 3, 6, 12];
+        let mut bitmap = vec![0b1110_u8];
+
+        let strings_buffer = unsafe {
+            Buffer::from_foreign(
+                NonNull::new_unchecked(strings.as_mut_ptr()),
+                strings.len(),
+                Arc::new(strings),
+            )
+        };
+        let offsets_buffer = unsafe {
+            Buffer::from_foreign(
+                NonNull::new_unchecked(offsets.as_mut_ptr() as *mut u8),
+                offsets.len() * std::mem::size_of::<i32>(),
+                Arc::new(offsets),
+            )
+        };
+        let null_buffer = unsafe {
+            Buffer::from_foreign(
+                NonNull::new_unchecked(bitmap.as_mut_ptr()),
+                bitmap.len(),
+                Arc::new(bitmap),
+            )
+        };
+
+        let data = ArrayData::try_new(
+            DataType::Utf8,
+            4,
+            None,
+            Some(null_buffer),
+            0,
+            vec![offsets_buffer, strings_buffer],
+            vec![],
+        )
+        .unwrap();
+
+        let array = make_array(data);
+        let array = array.as_any().downcast_ref::<StringArray>().unwrap();
+
+        let expected =
+            StringArray::from(vec![None, Some("foo"), Some("bar"), Some("foobar")]);
+
+        assert_eq!(array, &expected);
+    }
 }

arrow/src/buffer/immutable.rs

@@ -21,12 +21,9 @@ use std::ptr::NonNull;
 use std::sync::Arc;
 use std::{convert::AsRef, usize};
 
+use crate::alloc::{Allocation, Deallocation};
 use crate::util::bit_chunk_iterator::{BitChunks, UnalignedBitChunk};
-use crate::{
-    bytes::{Bytes, Deallocation},
-    datatypes::ArrowNativeType,
-    ffi,
-};
+use crate::{bytes::Bytes, datatypes::ArrowNativeType};
 
 use super::ops::bitwise_unary_op_helper;
 use super::MutableBuffer;
@@ -86,18 +83,18 @@ impl Buffer {
     ///
     /// * `ptr` - Pointer to raw parts
     /// * `len` - Length of raw parts in **bytes**
-    /// * `data` - An [ffi::FFI_ArrowArray] with the data
+    /// * `owner` - A [crate::alloc::Allocation] which is responsible for freeing that data
     ///
     /// # Safety
     ///
     /// This function is unsafe as there is no guarantee that the given pointer is valid for `len`
     /// bytes and that the foreign deallocator frees the region.
-    pub unsafe fn from_unowned(
+    pub unsafe fn from_foreign(
         ptr: NonNull<u8>,
         len: usize,
-        data: Arc<ffi::FFI_ArrowArray>,
+        owner: Arc<dyn Allocation>,
     ) -> Self {
-        Buffer::build_with_arguments(ptr, len, Deallocation::Foreign(data))
+        Buffer::build_with_arguments(ptr, len, Deallocation::Foreign(owner))
     }
 
     /// Auxiliary method to create a new Buffer
@@ -533,4 +530,24 @@ mod tests {
             Buffer::from(&[0b01101101, 0b10101010]).count_set_bits_offset(7, 9)
         );
     }
+
+    #[test]
+    fn test_from_foreign_vec() {
+        let mut vector = vec![1_i32, 2, 3, 4, 5];
+        let buffer = unsafe {
+            Buffer::from_foreign(
+                NonNull::new_unchecked(vector.as_mut_ptr() as *mut u8),
+                vector.len() * std::mem::size_of::<i32>(),
+                Arc::new(vector),
+            )
+        };
+
+        let slice = unsafe { buffer.typed_data::<i32>() };
+        assert_eq!(slice, &[1, 2, 3, 4, 5]);
+
+        let buffer = buffer.slice(std::mem::size_of::<i32>());
+
+        let slice = unsafe { buffer.typed_data::<i32>() };
+        assert_eq!(slice, &[2, 3, 4, 5]);
+    }
 }

arrow/src/buffer/mutable.rs

@@ -16,9 +16,10 @@
 // under the License.
 
 use super::Buffer;
+use crate::alloc::Deallocation;
 use crate::{
     alloc,
-    bytes::{Bytes, Deallocation},
+    bytes::Bytes,
     datatypes::{ArrowNativeType, ToByteSlice},
     util::bit_util,
 };

arrow/src/bytes.rs

@@ -21,31 +21,10 @@
 
 use core::slice;
 use std::ptr::NonNull;
-use std::sync::Arc;
 use std::{fmt::Debug, fmt::Formatter};
 
-use crate::{alloc, ffi};
-
-/// Mode of deallocating memory regions
-pub enum Deallocation {
-    /// Native deallocation, using Rust deallocator with Arrow-specific memory alignment
-    Native(usize),
-    /// Foreign interface, via a callback
-    Foreign(Arc<ffi::FFI_ArrowArray>),
-}
-
-impl Debug for Deallocation {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        match self {
-            Deallocation::Native(capacity) => {
-                write!(f, "Deallocation::Native {{ capacity: {} }}", capacity)
-            }
-            Deallocation::Foreign(_) => {
-                write!(f, "Deallocation::Foreign {{ capacity: unknown }}")
-            }
-        }
-    }
-}
+use crate::alloc;
+use crate::alloc::Deallocation;
 
 /// A continuous, fixed-size, immutable memory region that knows how to de-allocate itself.
 /// This structs' API is inspired by the `bytes::Bytes`, but it is not limited to using rust's

arrow/src/ffi.rs

@@ -538,7 +538,7 @@ unsafe fn create_buffer(
     assert!(index < array.n_buffers as usize);
     let ptr = *buffers.add(index);
 
-    NonNull::new(ptr as *mut u8).map(|ptr| Buffer::from_unowned(ptr, len, owner))
+    NonNull::new(ptr as *mut u8).map(|ptr| Buffer::from_foreign(ptr, len, owner))
 }
 
 fn create_child(