This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 6e699d7 ARROW-2252: [Python] Create buffer from address, size and base
6e699d7 is described below
commit 6e699d77693f22aeeb5ac47f96b6180dcc7f64c0
Author: Korn, Uwe <[email protected]>
AuthorDate: Sun Mar 4 23:26:38 2018 -0500
ARROW-2252: [Python] Create buffer from address, size and base
Usage with Arrow Java vectors:
```
import jpype
import numpy as np
import pyarrow as pa
import sys
# Start JVM with Arrow and all of its dependencies.
jpype.startJVM(getDefaultJVMPath(),
"-Djava.class.path=arrow-tools-0.9.0-SNAPSHOT-jar-with-dependencies.jar")
# Create vector
ra = jpype.JPackage("org").apache.arrow.memory.RootAllocator(sys.maxsize)
uint1 = jpype.JPackage("org").apache.arrow.vector.UInt1Vector("int", ra)
uint1.allocateNew(128)
for i in range(128):
uint1.setSafe(i, i)
uint1.setValueCount(128)
# Access it in Python
addr = uint1.getDataBuffer().unwrap().memoryAddress()
size = uint1.getDataBuffer().unwrap().capacity()
fb = pa.ForeignBuffer(addr, size, n)
np.asarray(fb)
```
Output (as expected):
```
array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38,
39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64,
65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77,
78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90,
91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103,
104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116,
117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127], dtype=int8)
```
Author: Korn, Uwe <[email protected]>
Closes #1693 from xhochy/ARROW-2252 and squashes the following commits:
86a9a6df <Korn, Uwe> ARROW-2252: Create buffer from address, size and base
---
python/pyarrow/__init__.py | 4 ++--
python/pyarrow/io.pxi | 12 ++++++++++++
python/pyarrow/lib.pxd | 5 +++++
python/pyarrow/tests/test_io.py | 9 +++++++++
4 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 15a37ca..8cb4b3b 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -72,8 +72,8 @@ from pyarrow.lib import (null, bool_,
from pyarrow.lib import TimestampType
# Buffers, allocation
-from pyarrow.lib import (Buffer, ResizableBuffer, compress, decompress,
- allocate_buffer, frombuffer)
+from pyarrow.lib import (Buffer, ForeignBuffer, ResizableBuffer, compress,
+ decompress, allocate_buffer, frombuffer)
from pyarrow.lib import (MemoryPool, total_allocated_bytes,
set_memory_pool, default_memory_pool,
diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
index 325c582..5c8411b 100644
--- a/python/pyarrow/io.pxi
+++ b/python/pyarrow/io.pxi
@@ -720,6 +720,18 @@ cdef class Buffer:
return self.size
+cdef class ForeignBuffer(Buffer):
+
+ def __init__(self, addr, size, base):
+ cdef:
+ intptr_t c_addr = addr
+ int64_t c_size = size
+ self.base = base
+ cdef shared_ptr[CBuffer] buffer = make_shared[CBuffer](
+ <uint8_t*>c_addr, c_size)
+ self.init(<shared_ptr[CBuffer]> buffer)
+
+
cdef class ResizableBuffer(Buffer):
cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer):
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index e4d574f..c37bc2b 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -324,6 +324,11 @@ cdef class Buffer:
cdef int _check_nullptr(self) except -1
+cdef class ForeignBuffer(Buffer):
+ cdef:
+ object base
+
+
cdef class ResizableBuffer(Buffer):
cdef void init_rz(self, const shared_ptr[CResizableBuffer]& buffer)
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index d269ad0..17aca43 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -24,6 +24,7 @@ import sys
import weakref
import numpy as np
+import numpy.testing as npt
import pandas as pd
@@ -253,6 +254,14 @@ def test_buffer_equals():
assert buf2.equals(buf5)
+def test_foreign_buffer():
+ n = np.array([1, 2])
+ addr = n.__array_interface__["data"][0]
+ size = n.nbytes
+ fb = pa.ForeignBuffer(addr, size, n)
+ npt.assert_array_equal(np.asarray(fb), n.view(dtype=np.int8))
+
+
def test_allocate_buffer():
buf = pa.allocate_buffer(100)
assert buf.size == 100
--
To stop receiving notification emails like this one, please contact
[email protected].
