This is an automated email from the ASF dual-hosted git repository.
sunchao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new d02425d Remove Clone and copy source structs internally (#1449)
d02425d is described below
commit d02425dc0114c4e192e950aeea13e3c1e750a2b0
Author: Liang-Chi Hsieh <[email protected]>
AuthorDate: Fri Mar 18 23:26:50 2022 -0700
Remove Clone and copy source structs internally (#1449)
* Remove Clone and copy source structs internally
* Remove drop_in_place and add more comment
* Add export_into_raw
* Fix format
* Fix clippy
* Move to export_array_into_raw
* Fix clippy
* Fix doc
* Use write_unaligned
---
arrow/src/array/array.rs | 24 +++++++++++++
arrow/src/array/mod.rs | 2 +-
arrow/src/ffi.rs | 94 +++++++++++++++++++++++++++++++++++++++++-------
3 files changed, 107 insertions(+), 13 deletions(-)
diff --git a/arrow/src/array/array.rs b/arrow/src/array/array.rs
index 795439e..1ad01f4 100644
--- a/arrow/src/array/array.rs
+++ b/arrow/src/array/array.rs
@@ -632,6 +632,30 @@ pub unsafe fn make_array_from_raw(
let data = ArrayData::try_from(array)?;
Ok(make_array(data))
}
+
+/// Exports an array to raw pointers of the C Data Interface provided by the
consumer.
+/// # Safety
+/// Assumes that these pointers represent valid C Data Interfaces, both in
memory
+/// representation and lifetime via the `release` mechanism.
+///
+/// This function copies the content of two FFI structs [ffi::FFI_ArrowArray]
and
+/// [ffi::FFI_ArrowSchema] in the array to the location pointed by the raw
pointers.
+/// Usually the raw pointers are provided by the array data consumer.
+pub unsafe fn export_array_into_raw(
+ src: ArrayRef,
+ out_array: *mut ffi::FFI_ArrowArray,
+ out_schema: *mut ffi::FFI_ArrowSchema,
+) -> Result<()> {
+ let data = src.data();
+ let array = ffi::FFI_ArrowArray::new(data);
+ let schema = ffi::FFI_ArrowSchema::try_from(data.data_type())?;
+
+ std::ptr::write_unaligned(out_array, array);
+ std::ptr::write_unaligned(out_schema, schema);
+
+ Ok(())
+}
+
// Helper function for printing potentially long arrays.
pub(super) fn print_long_array<A, F>(
array: &A,
diff --git a/arrow/src/array/mod.rs b/arrow/src/array/mod.rs
index 864bae1..fc6441b 100644
--- a/arrow/src/array/mod.rs
+++ b/arrow/src/array/mod.rs
@@ -521,7 +521,7 @@ pub use self::cast::{
// ------------------------------ C Data Interface ---------------------------
-pub use self::array::make_array_from_raw;
+pub use self::array::{export_array_into_raw, make_array_from_raw};
#[cfg(test)]
mod tests {
diff --git a/arrow/src/ffi.rs b/arrow/src/ffi.rs
index 461995b..5fb1cce 100644
--- a/arrow/src/ffi.rs
+++ b/arrow/src/ffi.rs
@@ -26,9 +26,10 @@
//!
//! ```rust
//! # use std::sync::Arc;
-//! # use arrow::array::{Int32Array, Array, ArrayData, make_array_from_raw};
+//! # use arrow::array::{Int32Array, Array, ArrayData, export_array_into_raw,
make_array, make_array_from_raw};
//! # use arrow::error::{Result, ArrowError};
//! # use arrow::compute::kernels::arithmetic;
+//! # use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema};
//! # use std::convert::TryFrom;
//! # fn main() -> Result<()> {
//! // create an array natively
@@ -51,7 +52,35 @@
//! // verify
//! assert_eq!(array, Int32Array::from(vec![Some(2), None, Some(6)]));
//!
+//! // Simulate if raw pointers are provided by consumer
+//! let array = make_array(Int32Array::from(vec![Some(1), None,
Some(3)]).data().clone());
+//!
+//! let out_array = Box::new(FFI_ArrowArray::empty());
+//! let out_schema = Box::new(FFI_ArrowSchema::empty());
+//! let out_array_ptr = Box::into_raw(out_array);
+//! let out_schema_ptr = Box::into_raw(out_schema);
+//!
+//! // export array into raw pointers from consumer
+//! unsafe { export_array_into_raw(array, out_array_ptr, out_schema_ptr)?; };
+//!
+//! // import it
+//! let array = unsafe { make_array_from_raw(out_array_ptr, out_schema_ptr)? };
+//!
+//! // perform some operation
+//! let array = array.as_any().downcast_ref::<Int32Array>().ok_or(
+//! ArrowError::ParseError("Expects an int32".to_string()),
+//! )?;
+//! let array = arithmetic::add(&array, &array)?;
+//!
+//! // verify
+//! assert_eq!(array, Int32Array::from(vec![Some(2), None, Some(6)]));
+//!
//! // (drop/release)
+//! unsafe {
+//! Box::from_raw(out_array_ptr);
+//! Box::from_raw(out_schema_ptr);
+//! }
+//!
//! Ok(())
//! }
//! ```
@@ -107,7 +136,7 @@ bitflags! {
/// See
<https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions>
/// This was created by bindgen
#[repr(C)]
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub struct FFI_ArrowSchema {
format: *const c_char,
name: *const c_char,
@@ -336,7 +365,7 @@ fn bit_width(data_type: &DataType, i: usize) ->
Result<usize> {
/// See
<https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions>
/// This was created by bindgen
#[repr(C)]
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub struct FFI_ArrowArray {
pub(crate) length: i64,
pub(crate) null_count: i64,
@@ -396,7 +425,7 @@ impl FFI_ArrowArray {
/// # Safety
/// This method releases `buffers`. Consumers of this struct *must* call
`release` before
/// releasing this struct, or contents in `buffers` leak.
- fn new(data: &ArrayData) -> Self {
+ pub(crate) fn new(data: &ArrayData) -> Self {
// * insert the null buffer at the start
// * make all others `Option<Buffer>`.
let buffers = iter::once(data.null_buffer().cloned())
@@ -769,6 +798,9 @@ impl ArrowArray {
/// creates a new [ArrowArray] from two pointers. Used to import from the
C Data Interface.
/// # Safety
/// See safety of [ArrowArray]
+ /// Note that this function will copy the content pointed by the raw
pointers. Considering
+ /// the raw pointers can be from `Arc::into_raw` or other raw pointers,
users must be responsible
+ /// on managing the allocation of the structs by themselves.
/// # Error
/// Errors if any of the pointers is null
pub unsafe fn try_from_raw(
@@ -781,11 +813,16 @@ impl ArrowArray {
.to_string(),
));
};
- let ffi_array = (*array).clone();
- let ffi_schema = (*schema).clone();
+
+ let array_mut = array as *mut FFI_ArrowArray;
+ let schema_mut = schema as *mut FFI_ArrowSchema;
+
+ let array_data = std::ptr::replace(array_mut, FFI_ArrowArray::empty());
+ let schema_data = std::ptr::replace(schema_mut,
FFI_ArrowSchema::empty());
+
Ok(Self {
- array: Arc::new(ffi_array),
- schema: Arc::new(ffi_schema),
+ array: Arc::new(array_data),
+ schema: Arc::new(schema_data),
})
}
@@ -822,10 +859,10 @@ impl<'a> ArrowArrayChild<'a> {
mod tests {
use super::*;
use crate::array::{
- make_array, Array, ArrayData, BinaryOffsetSizeTrait, BooleanArray,
DecimalArray,
- DictionaryArray, GenericBinaryArray, GenericListArray,
GenericStringArray,
- Int32Array, OffsetSizeTrait, StringOffsetSizeTrait,
Time32MillisecondArray,
- TimestampMillisecondArray,
+ export_array_into_raw, make_array, Array, ArrayData,
BinaryOffsetSizeTrait,
+ BooleanArray, DecimalArray, DictionaryArray, GenericBinaryArray,
+ GenericListArray, GenericStringArray, Int32Array, OffsetSizeTrait,
+ StringOffsetSizeTrait, Time32MillisecondArray,
TimestampMillisecondArray,
};
use crate::compute::kernels;
use crate::datatypes::{Field, Int8Type};
@@ -1164,4 +1201,37 @@ mod tests {
// (drop/release)
Ok(())
}
+
+ #[test]
+ fn test_export_array_into_raw() -> Result<()> {
+ let array = make_array(Int32Array::from(vec![1, 2, 3]).data().clone());
+
+ // Assume two raw pointers provided by the consumer
+ let out_array = Box::new(FFI_ArrowArray::empty());
+ let out_schema = Box::new(FFI_ArrowSchema::empty());
+ let out_array_ptr = Box::into_raw(out_array);
+ let out_schema_ptr = Box::into_raw(out_schema);
+
+ unsafe {
+ export_array_into_raw(array, out_array_ptr, out_schema_ptr)?;
+ }
+
+ // (simulate consumer) import it
+ unsafe {
+ let array = ArrowArray::try_from_raw(out_array_ptr,
out_schema_ptr).unwrap();
+ let data = ArrayData::try_from(array)?;
+ let array = make_array(data);
+
+ // perform some operation
+ let array = array.as_any().downcast_ref::<Int32Array>().unwrap();
+ let array = kernels::arithmetic::add(array, array).unwrap();
+
+ // verify
+ assert_eq!(array, Int32Array::from(vec![2, 4, 6]));
+
+ Box::from_raw(out_array_ptr);
+ Box::from_raw(out_schema_ptr);
+ }
+ Ok(())
+ }
}
