This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new d6368d06 refactor(python): Split ArrowArray and ArrowArrayStream
modules (#559)
d6368d06 is described below
commit d6368d0672470304f0a01cc365133c8d19d1ea4c
Author: Dewey Dunnington <[email protected]>
AuthorDate: Fri Jul 19 22:39:47 2024 -0300
refactor(python): Split ArrowArray and ArrowArrayStream modules (#559)
This is the final splitting of modules from `_lib.pyx`, which no longer
exists! It separates the array stream and array components to live in
separate .pyx files and adds basic docstrings to internal functions. It
also removes a few references to `_lib` and `c_lib` that had made it
through to this PR.
---
python/setup.py | 5 +-
python/src/nanoarrow/__init__.py | 1 -
python/src/nanoarrow/_array.pxd | 55 ++
python/src/nanoarrow/{_lib.pyx => _array.pyx} | 859 +++++++++-----------------
python/src/nanoarrow/_array_stream.pyx | 398 ++++++++++++
python/src/nanoarrow/_buffer.pyx | 6 +-
python/src/nanoarrow/_ipc_lib.pyi | 49 --
python/src/nanoarrow/_lib.pyi | 595 ------------------
python/src/nanoarrow/_repr_utils.py | 2 +-
python/src/nanoarrow/array.py | 13 +-
python/src/nanoarrow/array_stream.py | 2 +-
python/src/nanoarrow/c_array.py | 2 +-
python/src/nanoarrow/c_array_stream.py | 2 +-
python/src/nanoarrow/c_buffer.py | 4 +-
python/src/nanoarrow/device.py | 2 +-
python/src/nanoarrow/ipc.py | 2 +-
python/src/nanoarrow/iterator.py | 2 +-
python/src/nanoarrow/visitor.py | 14 +-
python/tests/test_c_array.py | 2 +-
python/tests/test_c_buffer.py | 8 +-
20 files changed, 797 insertions(+), 1226 deletions(-)
diff --git a/python/setup.py b/python/setup.py
index 00c299bb..45c37879 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -136,8 +136,11 @@ setup(
nanoarrow_extension(
"nanoarrow._device", nanoarrow_c=True, nanoarrow_device=True
),
+ nanoarrow_extension(
+ "nanoarrow._array", nanoarrow_c=True, nanoarrow_device=True
+ ),
+ nanoarrow_extension("nanoarrow._array_stream", nanoarrow_c=True),
nanoarrow_extension("nanoarrow._buffer", nanoarrow_c=True),
- nanoarrow_extension("nanoarrow._lib", nanoarrow_c=True,
nanoarrow_device=True),
nanoarrow_extension("nanoarrow._ipc_lib", nanoarrow_c=True,
nanoarrow_ipc=True),
nanoarrow_extension("nanoarrow._schema", nanoarrow_c=True),
],
diff --git a/python/src/nanoarrow/__init__.py b/python/src/nanoarrow/__init__.py
index 9f9b48e3..2a97e9f4 100644
--- a/python/src/nanoarrow/__init__.py
+++ b/python/src/nanoarrow/__init__.py
@@ -87,7 +87,6 @@ __all__ = [
"c_array_from_buffers",
"c_array_stream",
"c_buffer",
- "c_lib",
"c_schema",
"c_version",
"date32",
diff --git a/python/src/nanoarrow/_array.pxd b/python/src/nanoarrow/_array.pxd
new file mode 100644
index 00000000..01712efc
--- /dev/null
+++ b/python/src/nanoarrow/_array.pxd
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: language_level = 3
+
+from libc.stdint cimport int64_t
+
+from nanoarrow_c cimport (
+ ArrowArray,
+ ArrowArrayView,
+)
+
+from nanoarrow_device_c cimport (
+ ArrowDeviceArray,
+ ArrowDeviceType
+)
+
+from nanoarrow._device cimport Device
+from nanoarrow._schema cimport CSchema
+
+
+cdef class CArray:
+ cdef object _base
+ cdef ArrowArray* _ptr
+ cdef CSchema _schema
+ cdef ArrowDeviceType _device_type
+ cdef int _device_id
+
+ cdef _set_device(self, ArrowDeviceType device_type, int64_t device_id)
+
+
+cdef class CArrayView:
+ cdef object _base
+ cdef object _array_base
+ cdef ArrowArrayView* _ptr
+ cdef Device _device
+
+cdef class CDeviceArray:
+ cdef object _base
+ cdef ArrowDeviceArray* _ptr
+ cdef CSchema _schema
diff --git a/python/src/nanoarrow/_lib.pyx b/python/src/nanoarrow/_array.pyx
similarity index 65%
rename from python/src/nanoarrow/_lib.pyx
rename to python/src/nanoarrow/_array.pyx
index 7cef1d6a..e79aaa4c 100644
--- a/python/src/nanoarrow/_lib.pyx
+++ b/python/src/nanoarrow/_array.pyx
@@ -16,20 +16,6 @@
# under the License.
# cython: language_level = 3
-# cython: linetrace=True
-
-"""Low-level nanoarrow Python bindings
-
-This Cython extension provides low-level Python wrappers around the
-Arrow C Data and Arrow C Stream interface structs. In general, there
-is one wrapper per C struct and pointer validity is managed by keeping
-strong references to Python objects. These wrappers are intended to
-be literal and stay close to the structure definitions: higher level
-interfaces can and should be built in Python where it is faster to
-iterate and where it is easier to create a better user experience
-by default (i.e., classes, methods, and functions implemented in Python
-generally have better autocomplete + documentation available to IDEs).
-"""
from libc.stdint cimport uintptr_t, uint8_t, int64_t
from cpython.pycapsule cimport PyCapsule_GetPointer
@@ -41,7 +27,47 @@ from cpython cimport (
PyBUF_ANY_CONTIGUOUS,
PyBUF_FORMAT,
)
-from nanoarrow_c cimport *
+
+from nanoarrow_c cimport (
+ ArrowArray,
+ ArrowArrayAppendBytes,
+ ArrowArrayAppendNull,
+ ArrowArrayAppendString,
+ ArrowArrayBuffer,
+ ArrowArrayFinishBuilding,
+ ArrowArrayInitFromSchema,
+ ArrowArrayInitFromType,
+ ArrowArrayMove,
+ ArrowArrayRelease,
+ ArrowArrayStartAppending,
+ ArrowArrayView,
+ ArrowArrayViewComputeNullCount,
+ ArrowArrayViewInitFromSchema,
+ ArrowArrayViewSetArray,
+ ArrowArrayViewSetArrayMinimal,
+ ArrowBitCountSet,
+ ArrowBuffer,
+ ArrowBufferMove,
+ ArrowBufferType,
+ ArrowBufferView,
+ ArrowSchemaInitFromType,
+ ArrowStringView,
+ ArrowType,
+ ArrowTypeString,
+ ArrowValidationLevel,
+ NANOARROW_BUFFER_TYPE_DATA,
+ NANOARROW_BUFFER_TYPE_DATA_OFFSET,
+ NANOARROW_BUFFER_TYPE_TYPE_ID,
+ NANOARROW_BUFFER_TYPE_UNION_OFFSET,
+ NANOARROW_BUFFER_TYPE_VALIDITY,
+ NANOARROW_VALIDATION_LEVEL_DEFAULT,
+ NANOARROW_VALIDATION_LEVEL_FULL,
+ NANOARROW_VALIDATION_LEVEL_MINIMAL,
+ NANOARROW_VALIDATION_LEVEL_NONE,
+ NANOARROW_OK,
+)
+
+
from nanoarrow_device_c cimport (
ARROW_DEVICE_CPU,
ArrowDeviceType,
@@ -51,12 +77,10 @@ from nanoarrow_device_c cimport (
from nanoarrow._device cimport Device
-from nanoarrow cimport _types
from nanoarrow._buffer cimport CBuffer, CBufferView
-from nanoarrow._schema cimport CSchema, CLayout, assert_type_equal
+from nanoarrow._schema cimport CSchema, CLayout
from nanoarrow._utils cimport (
alloc_c_array,
- alloc_c_array_stream,
alloc_c_device_array,
alloc_c_array_view,
c_array_shallow_copy,
@@ -64,10 +88,184 @@ from nanoarrow._utils cimport (
Error
)
+from typing import Iterable, Tuple, Union
+
from nanoarrow import _repr_utils
from nanoarrow._device import DEVICE_CPU, DeviceType
+cdef class CArrayView:
+ """Low-level ArrowArrayView wrapper
+
+ This object is a literal wrapper around an ArrowArrayView. It provides
field accessors
+ that return Python objects and handles the structure lifecycle (i.e.,
initialized
+ ArrowArrayView structures are always released).
+
+ See `nanoarrow.c_array_view()` for construction and usage examples.
+ """
+
+ def __cinit__(self, object base, uintptr_t addr):
+ self._base = base
+ self._ptr = <ArrowArrayView*>addr
+ self._device = DEVICE_CPU
+
+ def _set_array(self, CArray array, Device device=DEVICE_CPU):
+ cdef Error error = Error()
+ cdef int code
+
+ if device is DEVICE_CPU:
+ code = ArrowArrayViewSetArray(self._ptr, array._ptr,
&error.c_error)
+ else:
+ code = ArrowArrayViewSetArrayMinimal(self._ptr, array._ptr,
&error.c_error)
+
+ error.raise_message_not_ok("ArrowArrayViewSetArray()", code)
+ self._array_base = array._base
+ self._device = device
+ return self
+
+ @property
+ def storage_type_id(self):
+ return self._ptr.storage_type
+
+ @property
+ def storage_type(self):
+ cdef const char* type_str = ArrowTypeString(self._ptr.storage_type)
+ if type_str != NULL:
+ return type_str.decode('UTF-8')
+
+ @property
+ def layout(self):
+ return CLayout(self, <uintptr_t>&self._ptr.layout)
+
+ def __len__(self):
+ return self._ptr.length
+
+ @property
+ def length(self):
+ return len(self)
+
+ @property
+ def offset(self):
+ return self._ptr.offset
+
+ @property
+ def null_count(self):
+ if self._ptr.null_count != -1:
+ return self._ptr.null_count
+
+ cdef ArrowBufferType buffer_type = self._ptr.layout.buffer_type[0]
+ cdef const uint8_t* validity_bits =
self._ptr.buffer_views[0].data.as_uint8
+
+ if buffer_type != NANOARROW_BUFFER_TYPE_VALIDITY:
+ self._ptr.null_count = 0
+ elif validity_bits == NULL:
+ self._ptr.null_count = 0
+ elif self._device is DEVICE_CPU:
+ self._ptr.null_count = ArrowArrayViewComputeNullCount(self._ptr)
+
+ return self._ptr.null_count
+
+ @property
+ def n_children(self):
+ return self._ptr.n_children
+
+ def child(self, int64_t i):
+ if i < 0 or i >= self._ptr.n_children:
+ raise IndexError(f"{i} out of range [0, {self._ptr.n_children})")
+
+ cdef CArrayView child = CArrayView(
+ self._base,
+ <uintptr_t>self._ptr.children[i]
+ )
+
+ child._device = self._device
+ return child
+
+ @property
+ def children(self):
+ for i in range(self.n_children):
+ yield self.child(i)
+
+ @property
+ def n_buffers(self):
+ return self.layout.n_buffers
+
+ def buffer_type(self, int64_t i):
+ if i < 0 or i >= self.n_buffers:
+ raise IndexError(f"{i} out of range [0, {self.n_buffers}]")
+
+ buffer_type = self._ptr.layout.buffer_type[i]
+ if buffer_type == NANOARROW_BUFFER_TYPE_VALIDITY:
+ return "validity"
+ elif buffer_type == NANOARROW_BUFFER_TYPE_TYPE_ID:
+ return "type_id"
+ elif buffer_type == NANOARROW_BUFFER_TYPE_UNION_OFFSET:
+ return "union_offset"
+ elif buffer_type == NANOARROW_BUFFER_TYPE_DATA_OFFSET:
+ return "data_offset"
+ elif buffer_type == NANOARROW_BUFFER_TYPE_DATA:
+ return "data"
+ else:
+ return "none"
+
+ def buffer(self, int64_t i):
+ if i < 0 or i >= self.n_buffers:
+ raise IndexError(f"{i} out of range [0, {self.n_buffers}]")
+
+ cdef ArrowBufferView* buffer_view = &(self._ptr.buffer_views[i])
+
+ # Check the buffer size here because the error later is cryptic.
+ # Buffer sizes are set to -1 when they are "unknown", so because of
errors
+ # in nanoarrow/C or because the array is on a non-CPU device, that -1
value
+ # could leak its way here.
+ if buffer_view.size_bytes < 0:
+ raise RuntimeError(f"ArrowArrayView buffer {i} has size_bytes < 0")
+
+ return CBufferView(
+ self._array_base,
+ <uintptr_t>buffer_view.data.data,
+ buffer_view.size_bytes,
+ self._ptr.layout.buffer_data_type[i],
+ self._ptr.layout.element_size_bits[i],
+ self._device
+ )
+
+ @property
+ def buffers(self):
+ for i in range(self.n_buffers):
+ yield self.buffer(i)
+
+ @property
+ def dictionary(self):
+ if self._ptr.dictionary == NULL:
+ return None
+ else:
+ return CArrayView(
+ self,
+ <uintptr_t>self._ptr.dictionary
+ )
+
+ def __repr__(self):
+ return _repr_utils.array_view_repr(self)
+
+ @staticmethod
+ def from_schema(CSchema schema):
+ cdef ArrowArrayView* c_array_view
+ base = alloc_c_array_view(&c_array_view)
+
+ cdef Error error = Error()
+ cdef int code = ArrowArrayViewInitFromSchema(c_array_view,
+ schema._ptr,
&error.c_error)
+ error.raise_message_not_ok("ArrowArrayViewInitFromSchema()", code)
+
+ return CArrayView(base, <uintptr_t>c_array_view)
+
+ @staticmethod
+ def from_array(CArray array, Device device=DEVICE_CPU):
+ out = CArrayView.from_schema(array._schema)
+ return out._set_array(array, device)
+
+
cdef class CArray:
"""Low-level ArrowArray wrapper
@@ -77,14 +275,10 @@ cdef class CArray:
See `nanoarrow.c_array()` for construction and usage examples.
"""
- cdef object _base
- cdef ArrowArray* _ptr
- cdef CSchema _schema
- cdef ArrowDeviceType _device_type
- cdef int _device_id
@staticmethod
- def allocate(CSchema schema):
+ def allocate(CSchema schema) -> CArray:
+ """Allocate a released ArrowArray"""
cdef ArrowArray* c_array_out
base = alloc_c_array(&c_array_out)
return CArray(base, <uintptr_t>c_array_out, schema)
@@ -101,9 +295,8 @@ cdef class CArray:
self._device_id = device_id
@staticmethod
- def _import_from_c_capsule(schema_capsule, array_capsule):
- """
- Import from a ArrowSchema and ArrowArray PyCapsule tuple.
+ def _import_from_c_capsule(schema_capsule, array_capsule) -> CArray:
+ """Import from a ArrowSchema and ArrowArray PyCapsule tuple.
Parameters
----------
@@ -127,7 +320,7 @@ cdef class CArray:
return out
- def __getitem__(self, k):
+ def __getitem__(self, k) -> CArray:
self._assert_valid()
if not isinstance(k, slice):
@@ -197,10 +390,11 @@ cdef class CArray:
return self._schema.__arrow_c_schema__(), array_capsule
- def _addr(self):
+ def _addr(self) -> int:
return <uintptr_t>self._ptr
- def is_valid(self):
+ def is_valid(self) -> bool:
+ """Check for a non-null and non-released underlying ArrowArray"""
return self._ptr != NULL and self._ptr.release != NULL
def _assert_valid(self):
@@ -209,56 +403,57 @@ cdef class CArray:
if self._ptr.release == NULL:
raise RuntimeError("CArray is released")
- def view(self):
+ def view(self) -> CArrayView:
+ """Allocate a :class:`CArrayView` to access the buffers of this
array"""
device = Device.resolve(self._device_type, self._device_id)
return CArrayView.from_array(self, device)
@property
- def schema(self):
+ def schema(self) -> CSchema:
return self._schema
@property
- def device_type(self):
+ def device_type(self) -> DeviceType:
return DeviceType(self._device_type)
@property
- def device_type_id(self):
+ def device_type_id(self) -> int:
return self._device_type
@property
- def device_id(self):
+ def device_id(self) -> int:
return self._device_id
- def __len__(self):
+ def __len__(self) -> int:
self._assert_valid()
return self._ptr.length
@property
- def length(self):
+ def length(self) -> int:
return len(self)
@property
- def offset(self):
+ def offset(self) -> int:
self._assert_valid()
return self._ptr.offset
@property
- def null_count(self):
+ def null_count(self) -> int:
self._assert_valid()
return self._ptr.null_count
@property
- def n_buffers(self):
+ def n_buffers(self) -> int:
self._assert_valid()
return self._ptr.n_buffers
@property
- def buffers(self):
+ def buffers(self) -> Tuple[int, ...]:
self._assert_valid()
return tuple(<uintptr_t>self._ptr.buffers[i] for i in
range(self._ptr.n_buffers))
@property
- def n_children(self):
+ def n_children(self) -> int:
self._assert_valid()
return self._ptr.n_children
@@ -275,12 +470,12 @@ cdef class CArray:
return out
@property
- def children(self):
+ def children(self) -> Iterable[CArray]:
for i in range(self.n_children):
yield self.child(i)
@property
- def dictionary(self):
+ def dictionary(self) -> Union[CArray, None]:
self._assert_valid()
cdef CArray out
if self._ptr.dictionary != NULL:
@@ -290,187 +485,16 @@ cdef class CArray:
else:
return None
- def __repr__(self):
+ def __repr__(self) -> str:
return _repr_utils.array_repr(self)
-cdef class CArrayView:
- """Low-level ArrowArrayView wrapper
-
- This object is a literal wrapper around an ArrowArrayView. It provides
field accessors
- that return Python objects and handles the structure lifecycle (i.e.,
initialized
- ArrowArrayView structures are always released).
+cdef class CArrayBuilder:
+ """Helper for constructing an ArrowArray
- See `nanoarrow.c_array_view()` for construction and usage examples.
+ The primary function of this class is to wrap the nanoarrow C library calls
+ that build up the components of an ArrowArray.
"""
- cdef object _base
- cdef object _array_base
- cdef ArrowArrayView* _ptr
- cdef Device _device
-
- def __cinit__(self, object base, uintptr_t addr):
- self._base = base
- self._ptr = <ArrowArrayView*>addr
- self._device = DEVICE_CPU
-
- def _set_array(self, CArray array, Device device=DEVICE_CPU):
- cdef Error error = Error()
- cdef int code
-
- if device is DEVICE_CPU:
- code = ArrowArrayViewSetArray(self._ptr, array._ptr,
&error.c_error)
- else:
- code = ArrowArrayViewSetArrayMinimal(self._ptr, array._ptr,
&error.c_error)
-
- error.raise_message_not_ok("ArrowArrayViewSetArray()", code)
- self._array_base = array._base
- self._device = device
- return self
-
- @property
- def storage_type_id(self):
- return self._ptr.storage_type
-
- @property
- def storage_type(self):
- cdef const char* type_str = ArrowTypeString(self._ptr.storage_type)
- if type_str != NULL:
- return type_str.decode('UTF-8')
-
- @property
- def layout(self):
- return CLayout(self, <uintptr_t>&self._ptr.layout)
-
- def __len__(self):
- return self._ptr.length
-
- @property
- def length(self):
- return len(self)
-
- @property
- def offset(self):
- return self._ptr.offset
-
- @property
- def null_count(self):
- if self._ptr.null_count != -1:
- return self._ptr.null_count
-
- cdef ArrowBufferType buffer_type = self._ptr.layout.buffer_type[0]
- cdef const uint8_t* validity_bits =
self._ptr.buffer_views[0].data.as_uint8
-
- if buffer_type != NANOARROW_BUFFER_TYPE_VALIDITY:
- self._ptr.null_count = 0
- elif validity_bits == NULL:
- self._ptr.null_count = 0
- elif self._device is DEVICE_CPU:
- self._ptr.null_count = ArrowArrayViewComputeNullCount(self._ptr)
-
- return self._ptr.null_count
-
- @property
- def n_children(self):
- return self._ptr.n_children
-
- def child(self, int64_t i):
- if i < 0 or i >= self._ptr.n_children:
- raise IndexError(f"{i} out of range [0, {self._ptr.n_children})")
-
- cdef CArrayView child = CArrayView(
- self._base,
- <uintptr_t>self._ptr.children[i]
- )
-
- child._device = self._device
- return child
-
- @property
- def children(self):
- for i in range(self.n_children):
- yield self.child(i)
-
- @property
- def n_buffers(self):
- return self.layout.n_buffers
-
- def buffer_type(self, int64_t i):
- if i < 0 or i >= self.n_buffers:
- raise IndexError(f"{i} out of range [0, {self.n_buffers}]")
-
- buffer_type = self._ptr.layout.buffer_type[i]
- if buffer_type == NANOARROW_BUFFER_TYPE_VALIDITY:
- return "validity"
- elif buffer_type == NANOARROW_BUFFER_TYPE_TYPE_ID:
- return "type_id"
- elif buffer_type == NANOARROW_BUFFER_TYPE_UNION_OFFSET:
- return "union_offset"
- elif buffer_type == NANOARROW_BUFFER_TYPE_DATA_OFFSET:
- return "data_offset"
- elif buffer_type == NANOARROW_BUFFER_TYPE_DATA:
- return "data"
- else:
- return "none"
-
- def buffer(self, int64_t i):
- if i < 0 or i >= self.n_buffers:
- raise IndexError(f"{i} out of range [0, {self.n_buffers}]")
-
- cdef ArrowBufferView* buffer_view = &(self._ptr.buffer_views[i])
-
- # Check the buffer size here because the error later is cryptic.
- # Buffer sizes are set to -1 when they are "unknown", so because of
errors
- # in nanoarrow/C or because the array is on a non-CPU device, that -1
value
- # could leak its way here.
- if buffer_view.size_bytes < 0:
- raise RuntimeError(f"ArrowArrayView buffer {i} has size_bytes < 0")
-
- return CBufferView(
- self._array_base,
- <uintptr_t>buffer_view.data.data,
- buffer_view.size_bytes,
- self._ptr.layout.buffer_data_type[i],
- self._ptr.layout.element_size_bits[i],
- self._device
- )
-
- @property
- def buffers(self):
- for i in range(self.n_buffers):
- yield self.buffer(i)
-
- @property
- def dictionary(self):
- if self._ptr.dictionary == NULL:
- return None
- else:
- return CArrayView(
- self,
- <uintptr_t>self._ptr.dictionary
- )
-
- def __repr__(self):
- return _repr_utils.array_view_repr(self)
-
- @staticmethod
- def from_schema(CSchema schema):
- cdef ArrowArrayView* c_array_view
- base = alloc_c_array_view(&c_array_view)
-
- cdef Error error = Error()
- cdef int code = ArrowArrayViewInitFromSchema(c_array_view,
- schema._ptr,
&error.c_error)
- error.raise_message_not_ok("ArrowArrayViewInitFromSchema()", code)
-
- return CArrayView(base, <uintptr_t>c_array_view)
-
- @staticmethod
- def from_array(CArray array, Device device=DEVICE_CPU):
- out = CArrayView.from_schema(array._schema)
- return out._set_array(array, device)
-
-
-cdef class CArrayBuilder:
cdef CArray c_array
cdef ArrowArray* _ptr
cdef bint _can_validate
@@ -482,15 +506,22 @@ cdef class CArrayBuilder:
@staticmethod
def allocate():
+ """Create a CArrayBuilder
+
+ Allocates memory for an ArrowArray and populates it with nanoarrow's
+ ArrowArray private_data/release callback implementation. This should
+ usually be followed by :meth:`init_from_type` or
:meth:`init_from_schema`.
+ """
return CArrayBuilder(CArray.allocate(CSchema.allocate()))
- def is_empty(self):
+ def is_empty(self) -> bool:
+ """Check if any items have been appended to this builder"""
if self._ptr.release == NULL:
raise RuntimeError("CArrayBuilder is not initialized")
return self._ptr.length == 0
- def init_from_type(self, int type_id):
+ def init_from_type(self, int type_id) -> CArrayBuilder:
if self._ptr.release != NULL:
raise RuntimeError("CArrayBuilder is already initialized")
@@ -502,7 +533,7 @@ cdef class CArrayBuilder:
return self
- def init_from_schema(self, CSchema schema):
+ def init_from_schema(self, CSchema schema) -> CArrayBuilder:
if self._ptr.release != NULL:
raise RuntimeError("CArrayBuilder is already initialized")
@@ -513,12 +544,17 @@ cdef class CArrayBuilder:
self.c_array._schema = schema
return self
- def start_appending(self):
+ def start_appending(self) -> CArrayBuilder:
+ """Use append mode for building this ArrowArray
+
+ Calling this method is required to produce a valid array prior to
calling
+ :meth:`append_strings` or `append_bytes`.
+ """
cdef int code = ArrowArrayStartAppending(self._ptr)
Error.raise_error_not_ok("ArrowArrayStartAppending()", code)
return self
- def append_strings(self, obj):
+ def append_strings(self, obj: Iterable[Union[str, None]]) -> CArrayBuilder:
cdef int code
cdef Py_ssize_t item_utf8_size
cdef ArrowStringView item
@@ -540,7 +576,7 @@ cdef class CArrayBuilder:
return self
- def append_bytes(self, obj):
+ def append_bytes(self, obj: Iterable[Union[str, None]]) -> CArrayBuilder:
cdef Py_buffer buffer
cdef ArrowBufferView item
@@ -565,22 +601,23 @@ cdef class CArrayBuilder:
if code != NANOARROW_OK:
Error.raise_error(f"append bytes item {py_item}", code)
- def set_offset(self, int64_t offset):
+ def set_offset(self, int64_t offset) -> CArrayBuilder:
self.c_array._assert_valid()
self._ptr.offset = offset
return self
- def set_length(self, int64_t length):
+ def set_length(self, int64_t length) -> CArrayBuilder:
self.c_array._assert_valid()
self._ptr.length = length
return self
- def set_null_count(self, int64_t null_count):
+ def set_null_count(self, int64_t null_count) -> CArrayBuilder:
self.c_array._assert_valid()
self._ptr.null_count = null_count
return self
- def resolve_null_count(self):
+ def resolve_null_count(self) -> CArrayBuilder:
+ """Ensure the output null count is synchronized with existing
buffers"""
self.c_array._assert_valid()
# This doesn't apply to unions. We currently don't have a schema view
@@ -618,14 +655,17 @@ cdef class CArrayBuilder:
self._ptr.null_count = self._ptr.length - count
return self
- def set_buffer(self, int64_t i, CBuffer buffer, move=False):
- """Sets a buffer of this ArrowArray such the pointer at
array->buffers[i] is
+ def set_buffer(self, int64_t i, CBuffer buffer, move=False) ->
CArrayBuilder:
+ """Set an ArrowArray buffer
+
+ Sets a buffer of this ArrowArray such the pointer at array->buffers[i]
is
equal to buffer->data and such that the buffer's lifcycle is managed by
the array. If move is True, the input Python object that previously
wrapped
the ArrowBuffer will be invalidated, which is usually the desired
behaviour
if you built or imported a buffer specifically to build this array. If
move
is False (the default), this function will a make a shallow copy via
another
- layer of Python object wrapping."""
+ layer of Python object wrapping.
+ """
if i < 0 or i > 3:
raise IndexError("i must be >= 0 and <= 3")
@@ -642,7 +682,14 @@ cdef class CArrayBuilder:
return self
- def set_child(self, int64_t i, CArray c_array, move=False):
+ def set_child(self, int64_t i, CArray c_array, move=False) ->
CArrayBuilder:
+ """Set an ArrowArray child
+
+ Set a child of this array by performing a show copy or optionally
+ transferring ownership to this object. The initialized child array
+ must have been initialized before this call by initializing this
+ builder with a schema containing the correct number of children.
+ """
cdef CArray child = self.c_array.child(i)
if child._ptr.release != NULL:
ArrowArrayRelease(child._ptr)
@@ -659,7 +706,20 @@ cdef class CArrayBuilder:
return self
- def finish(self, validation_level=None):
+ def finish(self, validation_level=None) -> CArray:
+ """Finish building this array
+
+ Performs any steps required to return a valid ArrowArray and optionally
+ validates the output to ensure that the result is valid (given the
information
+ the array has available to it).
+
+ Parameters
+ ----------
+ validation_level : None, "full", "default", "minimal", or "none",
optional
+ Explicitly define a validation level or use None to perform default
+ validation if possible. Validation may not be possible if children
+ were set that were not created by nanoarrow.
+ """
self.c_array._assert_valid()
cdef ArrowValidationLevel c_validation_level
cdef Error error = Error()
@@ -688,316 +748,15 @@ cdef class CArrayBuilder:
return out
-cdef class CArrayStream:
- """Low-level ArrowArrayStream wrapper
+cdef class CDeviceArray:
+ """Low-level ArrowDeviceArray wrapper
- This object is a literal wrapper around an ArrowArrayStream. It provides
methods that
- that wrap the underlying C callbacks and handles the C Data interface
lifecycle
- (i.e., initialized ArrowArrayStream structures are always released).
+ This object is a literal wrapper around an ArrowDeviceArray. It provides
field accessors
+ that return Python objects and handles the structure lifecycle (i.e.,
initialized
+ ArrowDeviceArray structures are always released).
- See `nanoarrow.c_array_stream()` for construction and usage examples.
+ See `nanoarrow.device.c_device_array()` for construction and usage
examples.
"""
- cdef object _base
- cdef ArrowArrayStream* _ptr
- cdef object _cached_schema
-
- def __cinit__(self, object base, uintptr_t addr):
- self._base = base
- self._ptr = <ArrowArrayStream*>addr
- self._cached_schema = None
-
- @staticmethod
- def allocate():
- cdef ArrowArrayStream* c_array_stream_out
- base = alloc_c_array_stream(&c_array_stream_out)
- return CArrayStream(base, <uintptr_t>c_array_stream_out)
-
- @staticmethod
- def from_c_arrays(arrays, CSchema schema, move=False, validate=True):
- cdef ArrowArrayStream* c_array_stream_out
- base = alloc_c_array_stream(&c_array_stream_out)
-
- # Don't create more copies than we have to (but make sure
- # one exists for validation if requested)
- cdef CSchema out_schema = schema
- if validate and not move:
- validate_schema = schema
- out_schema = schema.__deepcopy__()
- elif validate:
- validate_schema = schema.__deepcopy__()
- out_schema = schema
- elif not move:
- out_schema = schema.__deepcopy__()
-
- cdef int code = ArrowBasicArrayStreamInit(c_array_stream_out,
out_schema._ptr, len(arrays))
- Error.raise_error_not_ok("ArrowBasicArrayStreamInit()", code)
-
- cdef ArrowArray tmp
- cdef CArray array
- for i in range(len(arrays)):
- array = arrays[i]
-
- if validate:
- assert_type_equal(array.schema, validate_schema, False)
-
- if not move:
- c_array_shallow_copy(array._base, array._ptr, &tmp)
- ArrowBasicArrayStreamSetArray(c_array_stream_out, i, &tmp)
- else:
- ArrowBasicArrayStreamSetArray(c_array_stream_out, i,
array._ptr)
-
- cdef Error error = Error()
- if validate:
- code = ArrowBasicArrayStreamValidate(c_array_stream_out,
&error.c_error)
- error.raise_message_not_ok("ArrowBasicArrayStreamValidate()", code)
-
- return CArrayStream(base, <uintptr_t>c_array_stream_out)
-
- def release(self):
- if self.is_valid():
- self._ptr.release(self._ptr)
-
- @staticmethod
- def _import_from_c_capsule(stream_capsule):
- """
- Import from a ArrowArrayStream PyCapsule.
-
- Parameters
- ----------
- stream_capsule : PyCapsule
- A valid PyCapsule with name 'arrow_array_stream' containing an
- ArrowArrayStream pointer.
- """
- return CArrayStream(
- stream_capsule,
- <uintptr_t>PyCapsule_GetPointer(stream_capsule,
'arrow_array_stream')
- )
-
- def __arrow_c_stream__(self, requested_schema=None):
- """
- Export the stream as an Arrow C stream PyCapsule.
-
- Parameters
- ----------
- requested_schema : PyCapsule | None
- A PyCapsule containing a C ArrowSchema representation of a
requested
- schema. Not supported.
-
- Returns
- -------
- PyCapsule
- """
- self._assert_valid()
-
- if requested_schema is not None:
- raise NotImplementedError("requested_schema")
-
- cdef:
- ArrowArrayStream* c_array_stream_out
-
- array_stream_capsule = alloc_c_array_stream(&c_array_stream_out)
- ArrowArrayStreamMove(self._ptr, c_array_stream_out)
- return array_stream_capsule
-
- def _addr(self):
- return <uintptr_t>self._ptr
-
- def is_valid(self):
- return self._ptr != NULL and self._ptr.release != NULL
-
- def _assert_valid(self):
- if self._ptr == NULL:
- raise RuntimeError("array stream pointer is NULL")
- if self._ptr.release == NULL:
- raise RuntimeError("array stream is released")
-
- def _get_schema(self, CSchema schema):
- self._assert_valid()
- cdef Error error = Error()
- cdef int code = ArrowArrayStreamGetSchema(self._ptr, schema._ptr,
&error.c_error)
- error.raise_message_not_ok("ArrowArrayStream::get_schema()", code)
-
- def _get_cached_schema(self):
- if self._cached_schema is None:
- self._cached_schema = CSchema.allocate()
- self._get_schema(self._cached_schema)
-
- return self._cached_schema
-
- def get_schema(self):
- """Get the schema associated with this stream
- """
- out = CSchema.allocate()
- self._get_schema(out)
- return out
-
- def get_next(self):
- """Get the next Array from this stream
-
- Raises StopIteration when there are no more arrays in this stream.
- """
- self._assert_valid()
-
- # We return a reference to the same Python object for each
- # Array that is returned. This is independent of get_schema(),
- # which is guaranteed to call the C object's callback and
- # faithfully pass on the returned value.
-
- cdef Error error = Error()
- cdef CArray array = CArray.allocate(self._get_cached_schema())
- cdef int code = ArrowArrayStreamGetNext(self._ptr, array._ptr,
&error.c_error)
- error.raise_message_not_ok("ArrowArrayStream::get_next()", code)
-
- if not array.is_valid():
- raise StopIteration()
- else:
- return array
-
- def __iter__(self):
- return self
-
- def __next__(self):
- return self.get_next()
-
- def __enter__(self):
- return self
-
- def __exit__(self, *args, **kwargs):
- self.release()
-
- def __repr__(self):
- return _repr_utils.array_stream_repr(self)
-
-
-cdef class CMaterializedArrayStream:
- cdef CSchema _schema
- cdef CBuffer _array_ends
- cdef list _arrays
- cdef int64_t _total_length
-
- def __cinit__(self):
- self._arrays = []
- self._total_length = 0
- self._schema = CSchema.allocate()
- self._array_ends = CBuffer.empty()
- cdef int code = ArrowBufferAppendInt64(self._array_ends._ptr, 0)
- Error.raise_error_not_ok("ArrowBufferAppendInt64()", code)
-
- cdef _finalize(self):
- self._array_ends._set_data_type(<ArrowType>_types.INT64)
-
- @property
- def schema(self):
- return self._schema
-
- def __getitem__(self, k):
- cdef int64_t kint
- cdef int array_i
- cdef const int64_t* sorted_offsets =
<int64_t*>self._array_ends._ptr.data
-
- if isinstance(k, slice):
- raise NotImplementedError("index with slice")
-
- kint = k
- if kint < 0:
- kint += self._total_length
- if kint < 0 or kint >= self._total_length:
- raise IndexError(f"Index {kint} is out of range")
-
- array_i = ArrowResolveChunk64(kint, sorted_offsets, 0,
len(self._arrays))
- kint -= sorted_offsets[array_i]
- return self._arrays[array_i], kint
-
- def __len__(self):
- return self._array_ends[len(self._arrays)]
-
- def __iter__(self):
- for c_array in self._arrays:
- for item_i in range(len(c_array)):
- yield c_array, item_i
-
- def array(self, int64_t i):
- return self._arrays[i]
-
- @property
- def n_arrays(self):
- return len(self._arrays)
-
- @property
- def arrays(self):
- return iter(self._arrays)
-
- def __arrow_c_stream__(self, requested_schema=None):
- # When an array stream from iterable is supported, that could be used
here
- # to avoid unnessary shallow copies.
- stream = CArrayStream.from_c_arrays(
- self._arrays,
- self._schema,
- move=False,
- validate=False
- )
-
- return stream.__arrow_c_stream__(requested_schema=requested_schema)
-
- def child(self, int64_t i):
- cdef CMaterializedArrayStream out = CMaterializedArrayStream()
- cdef int code
-
- out._schema = self._schema.child(i)
- out._arrays = [chunk.child(i) for chunk in self._arrays]
- for child_chunk in out._arrays:
- out._total_length += len(child_chunk)
- code = ArrowBufferAppendInt64(out._array_ends._ptr,
out._total_length)
- Error.raise_error_not_ok("ArrowBufferAppendInt64()", code)
-
- out._finalize()
- return out
-
- @staticmethod
- def from_c_arrays(arrays, CSchema schema, bint validate=True):
- cdef CMaterializedArrayStream out = CMaterializedArrayStream()
-
- for array in arrays:
- if not isinstance(array, CArray):
- raise TypeError(f"Expected CArray but got
{type(array).__name__}")
-
- if len(array) == 0:
- continue
-
- if validate:
- assert_type_equal(array.schema, schema, False)
-
- out._total_length += len(array)
- code = ArrowBufferAppendInt64(out._array_ends._ptr,
out._total_length)
- Error.raise_error_not_ok("ArrowBufferAppendInt64()", code)
- out._arrays.append(array)
-
- out._schema = schema
- out._finalize()
- return out
-
- @staticmethod
- def from_c_array(CArray array):
- return CMaterializedArrayStream.from_c_arrays(
- [array],
- array.schema,
- validate=False
- )
-
- @staticmethod
- def from_c_array_stream(CArrayStream stream):
- with stream:
- return CMaterializedArrayStream.from_c_arrays(
- stream,
- stream._get_cached_schema(),
- validate=False
- )
-
-
-cdef class CDeviceArray:
- cdef object _base
- cdef ArrowDeviceArray* _ptr
- cdef CSchema _schema
def __cinit__(self, object base, uintptr_t addr, CSchema schema):
self._base = base
@@ -1016,30 +775,30 @@ cdef class CDeviceArray:
return CDeviceArray(holder, <uintptr_t>device_array_ptr, schema)
@property
- def schema(self):
+ def schema(self) -> CSchema:
return self._schema
@property
- def device_type(self):
+ def device_type(self) -> DeviceType:
return DeviceType(self._ptr.device_type)
@property
- def device_type_id(self):
+ def device_type_id(self) -> int:
return self._ptr.device_type
@property
- def device_id(self):
+ def device_id(self) -> int:
return self._ptr.device_id
@property
- def array(self):
+ def array(self) -> CArray:
# TODO: We lose access to the sync_event here, so we probably need to
# synchronize (or propagate it, or somehow prevent data access
downstream)
cdef CArray array = CArray(self, <uintptr_t>&self._ptr.array,
self._schema)
array._set_device(self._ptr.device_type, self._ptr.device_id)
return array
- def view(self):
+ def view(self) -> CArrayView:
return self.array.view()
def __arrow_c_array__(self, requested_schema=None):
@@ -1058,7 +817,7 @@ cdef class CDeviceArray:
return self._schema.__arrow_c_schema__(), device_array_capsule
@staticmethod
- def _import_from_c_capsule(schema_capsule, device_array_capsule):
+ def _import_from_c_capsule(schema_capsule, device_array_capsule) ->
CDeviceArray:
"""
Import from an ArrowSchema and ArrowArray PyCapsule tuple.
@@ -1084,5 +843,5 @@ cdef class CDeviceArray:
return out
- def __repr__(self):
+ def __repr__(self) -> str:
return _repr_utils.device_array_repr(self)
diff --git a/python/src/nanoarrow/_array_stream.pyx
b/python/src/nanoarrow/_array_stream.pyx
new file mode 100644
index 00000000..b7692711
--- /dev/null
+++ b/python/src/nanoarrow/_array_stream.pyx
@@ -0,0 +1,398 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: language_level = 3
+
+
+from libc.stdint cimport uintptr_t, int64_t
+from cpython.pycapsule cimport PyCapsule_GetPointer
+
+from nanoarrow_c cimport (
+ ArrowArray,
+ ArrowArrayStream,
+ ArrowArrayStreamGetNext,
+ ArrowArrayStreamGetSchema,
+ ArrowArrayStreamMove,
+ ArrowBasicArrayStreamInit,
+ ArrowBasicArrayStreamSetArray,
+ ArrowBasicArrayStreamValidate,
+ ArrowBufferAppendInt64,
+ ArrowResolveChunk64,
+ ArrowType,
+)
+
+from nanoarrow cimport _types
+from nanoarrow._array cimport CArray
+from nanoarrow._buffer cimport CBuffer
+from nanoarrow._schema cimport CSchema, assert_type_equal
+from nanoarrow._utils cimport (
+ alloc_c_array_stream,
+ c_array_shallow_copy,
+ Error
+)
+
+from typing import Iterable, List, Tuple
+
+from nanoarrow import _repr_utils
+
+
+cdef class CArrayStream:
+ """Low-level ArrowArrayStream wrapper
+
+ This object is a literal wrapper around an ArrowArrayStream. It provides
methods that
+ that wrap the underlying C callbacks and handles the C Data interface
lifecycle
+ (i.e., initialized ArrowArrayStream structures are always released).
+
+ See `nanoarrow.c_array_stream()` for construction and usage examples.
+ """
+ cdef object _base
+ cdef ArrowArrayStream* _ptr
+ cdef object _cached_schema
+
+ def __cinit__(self, object base, uintptr_t addr):
+ self._base = base
+ self._ptr = <ArrowArrayStream*>addr
+ self._cached_schema = None
+
+ @staticmethod
+ def allocate() -> CArrayStream:
+ """Allocate a released ArrowArrayStream"""
+ cdef ArrowArrayStream* c_array_stream_out
+ base = alloc_c_array_stream(&c_array_stream_out)
+ return CArrayStream(base, <uintptr_t>c_array_stream_out)
+
+ @staticmethod
+ def from_c_arrays(arrays: List[CArray], CSchema schema, move=False,
validate=True) -> CArrayStream:
+ """Create an ArrowArrayStream from an existing set of arrays
+
+ Given a previously resolved list of arrays, create an ArrowArrayStream
+ representation of the sequence of chunks.
+
+ Parameters
+ ----------
+ arrays : List[CArray]
+ A list of arrays to use as batches.
+ schema : CSchema
+ The schema that will be returned. Must be type equal with the
schema
+ of each array (this is checked if validate is ``True``)
+ move : bool, optional
+ If True, transfer ownership from each array instead of creating a
+ shallow copy. This is only safe if the caller knows the origin of
the
+ arrays and knows that they will not be accessed after this stream
has been
+ created.
+ validate : bool, optional
+ If True, enforce type equality between the provided schema and the
schema
+ of each array.
+ """
+ cdef ArrowArrayStream* c_array_stream_out
+ base = alloc_c_array_stream(&c_array_stream_out)
+
+ # Don't create more copies than we have to (but make sure
+ # one exists for validation if requested)
+ cdef CSchema out_schema = schema
+ if validate and not move:
+ validate_schema = schema
+ out_schema = schema.__deepcopy__()
+ elif validate:
+ validate_schema = schema.__deepcopy__()
+ out_schema = schema
+ elif not move:
+ out_schema = schema.__deepcopy__()
+
+ cdef int code = ArrowBasicArrayStreamInit(c_array_stream_out,
out_schema._ptr, len(arrays))
+ Error.raise_error_not_ok("ArrowBasicArrayStreamInit()", code)
+
+ cdef ArrowArray tmp
+ cdef CArray array
+ for i in range(len(arrays)):
+ array = arrays[i]
+
+ if validate:
+ assert_type_equal(array.schema, validate_schema, False)
+
+ if not move:
+ c_array_shallow_copy(array._base, array._ptr, &tmp)
+ ArrowBasicArrayStreamSetArray(c_array_stream_out, i, &tmp)
+ else:
+ ArrowBasicArrayStreamSetArray(c_array_stream_out, i,
array._ptr)
+
+ cdef Error error = Error()
+ if validate:
+ code = ArrowBasicArrayStreamValidate(c_array_stream_out,
&error.c_error)
+ error.raise_message_not_ok("ArrowBasicArrayStreamValidate()", code)
+
+ return CArrayStream(base, <uintptr_t>c_array_stream_out)
+
+ def release(self):
+ """Explicitly call the release callback of this stream"""
+ if self.is_valid():
+ self._ptr.release(self._ptr)
+
+ @staticmethod
+ def _import_from_c_capsule(stream_capsule) -> CArrayStream:
+ """Import from a ArrowArrayStream PyCapsule.
+
+ Parameters
+ ----------
+ stream_capsule : PyCapsule
+ A valid PyCapsule with name 'arrow_array_stream' containing an
+ ArrowArrayStream pointer.
+ """
+ return CArrayStream(
+ stream_capsule,
+ <uintptr_t>PyCapsule_GetPointer(stream_capsule,
'arrow_array_stream')
+ )
+
+ def __arrow_c_stream__(self, requested_schema=None):
+ """
+ Export the stream as an Arrow C stream PyCapsule.
+
+ Parameters
+ ----------
+ requested_schema : PyCapsule | None
+ A PyCapsule containing a C ArrowSchema representation of a
requested
+ schema. Not supported.
+
+ Returns
+ -------
+ PyCapsule
+ """
+ self._assert_valid()
+
+ if requested_schema is not None:
+ raise NotImplementedError("requested_schema")
+
+ cdef:
+ ArrowArrayStream* c_array_stream_out
+
+ array_stream_capsule = alloc_c_array_stream(&c_array_stream_out)
+ ArrowArrayStreamMove(self._ptr, c_array_stream_out)
+ return array_stream_capsule
+
+ def _addr(self) -> int:
+ return <uintptr_t>self._ptr
+
+ def is_valid(self) -> bool:
+ """Check for a non-null and non-released underlying ArrowArrayStream"""
+ return self._ptr != NULL and self._ptr.release != NULL
+
+ def _assert_valid(self):
+ if self._ptr == NULL:
+ raise RuntimeError("array stream pointer is NULL")
+ if self._ptr.release == NULL:
+ raise RuntimeError("array stream is released")
+
+ def _get_schema(self, CSchema schema):
+ self._assert_valid()
+ cdef Error error = Error()
+ cdef int code = ArrowArrayStreamGetSchema(self._ptr, schema._ptr,
&error.c_error)
+ error.raise_message_not_ok("ArrowArrayStream::get_schema()", code)
+
+ def _get_cached_schema(self):
+ if self._cached_schema is None:
+ self._cached_schema = CSchema.allocate()
+ self._get_schema(self._cached_schema)
+
+ return self._cached_schema
+
+ def get_schema(self) -> CSchema:
+ """Get the schema associated with this stream
+
+ Calling this method will always issue a call to the underlying stream's
+ get_schema callback.
+ """
+ out = CSchema.allocate()
+ self._get_schema(out)
+ return out
+
+ def get_next(self) -> CArray:
+ """Get the next Array from this stream
+
+ Raises StopIteration when there are no more arrays in this stream.
+ """
+ self._assert_valid()
+
+ # We return a reference to the same Python object for each
+ # Array that is returned. This is independent of get_schema(),
+ # which is guaranteed to call the C object's callback and
+ # faithfully pass on the returned value.
+
+ cdef Error error = Error()
+ cdef CArray array = CArray.allocate(self._get_cached_schema())
+ cdef int code = ArrowArrayStreamGetNext(self._ptr, array._ptr,
&error.c_error)
+ error.raise_message_not_ok("ArrowArrayStream::get_next()", code)
+
+ if not array.is_valid():
+ raise StopIteration()
+ else:
+ return array
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return self.get_next()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args, **kwargs):
+ self.release()
+
+ def __repr__(self):
+ return _repr_utils.array_stream_repr(self)
+
+
+cdef class CMaterializedArrayStream:
+ """Optimized representation of a fully consumed ArrowArrayStream
+
+ This class provides a data structure similar to pyarrow's ChunkedArray
+ where each consumed array is a referenced-counted shared array. This
+ class wraps the utilities provided by the nanoarrow C library to iterate
+ over and facilitate log(n) random access to items in this container.
+ """
+ cdef CSchema _schema
+ cdef CBuffer _array_ends
+ cdef list _arrays
+ cdef int64_t _total_length
+
+ def __cinit__(self):
+ self._arrays = []
+ self._total_length = 0
+ self._schema = CSchema.allocate()
+ self._array_ends = CBuffer.empty()
+ cdef int code = ArrowBufferAppendInt64(self._array_ends._ptr, 0)
+ Error.raise_error_not_ok("ArrowBufferAppendInt64()", code)
+
+ cdef _finalize(self):
+ self._array_ends._set_data_type(<ArrowType>_types.INT64)
+
+ @property
+ def schema(self) -> CSchema:
+ return self._schema
+
+ def __getitem__(self, k) -> Tuple[CArray, int]:
+ cdef int64_t kint
+ cdef int array_i
+ cdef const int64_t* sorted_offsets =
<int64_t*>self._array_ends._ptr.data
+
+ if isinstance(k, slice):
+ raise NotImplementedError("index with slice")
+
+ kint = k
+ if kint < 0:
+ kint += self._total_length
+ if kint < 0 or kint >= self._total_length:
+ raise IndexError(f"Index {kint} is out of range")
+
+ array_i = ArrowResolveChunk64(kint, sorted_offsets, 0,
len(self._arrays))
+ kint -= sorted_offsets[array_i]
+ return self._arrays[array_i], kint
+
+ def __len__(self) -> int:
+ return self._array_ends[len(self._arrays)]
+
+ def __iter__(self) -> Iterable[Tuple[CArray, int]]:
+ for c_array in self._arrays:
+ for item_i in range(len(c_array)):
+ yield c_array, item_i
+
+ def array(self, int64_t i) -> CArray:
+ return self._arrays[i]
+
+ @property
+ def n_arrays(self) -> int:
+ return len(self._arrays)
+
+ @property
+ def arrays(self) -> Iterable[CArray]:
+ return iter(self._arrays)
+
+ def __arrow_c_stream__(self, requested_schema=None):
+ # When an array stream from iterable is supported, that could be used
here
+ # to avoid unnessary shallow copies.
+ stream = CArrayStream.from_c_arrays(
+ self._arrays,
+ self._schema,
+ move=False,
+ validate=False
+ )
+
+ return stream.__arrow_c_stream__(requested_schema=requested_schema)
+
+ def child(self, int64_t i) -> CMaterializedArrayStream:
+ cdef CMaterializedArrayStream out = CMaterializedArrayStream()
+ cdef int code
+
+ out._schema = self._schema.child(i)
+ out._arrays = [chunk.child(i) for chunk in self._arrays]
+ for child_chunk in out._arrays:
+ out._total_length += len(child_chunk)
+ code = ArrowBufferAppendInt64(out._array_ends._ptr,
out._total_length)
+ Error.raise_error_not_ok("ArrowBufferAppendInt64()", code)
+
+ out._finalize()
+ return out
+
+ @staticmethod
+ def from_c_arrays(arrays: Iterable[CArray], CSchema schema, bint
validate=True):
+ """"Create a materialized array stream from an existing iterable of
arrays
+
+ This is slightly more efficient than creating a stream and then
consuming it
+ because the implementation can avoid a shallow copy of each array.
+ """
+ cdef CMaterializedArrayStream out = CMaterializedArrayStream()
+
+ for array in arrays:
+ if not isinstance(array, CArray):
+ raise TypeError(f"Expected CArray but got
{type(array).__name__}")
+
+ if len(array) == 0:
+ continue
+
+ if validate:
+ assert_type_equal(array.schema, schema, False)
+
+ out._total_length += len(array)
+ code = ArrowBufferAppendInt64(out._array_ends._ptr,
out._total_length)
+ Error.raise_error_not_ok("ArrowBufferAppendInt64()", code)
+ out._arrays.append(array)
+
+ out._schema = schema
+ out._finalize()
+ return out
+
+ @staticmethod
+ def from_c_array(CArray array) -> CMaterializedArrayStream:
+ """"Create a materialized array stream from a single array
+ """
+ return CMaterializedArrayStream.from_c_arrays(
+ [array],
+ array.schema,
+ validate=False
+ )
+
+ @staticmethod
+ def from_c_array_stream(CArrayStream stream) -> CMaterializedArrayStream:
+ """"Create a materialized array stream from an unmaterialized
ArrowArrayStream
+ """
+ with stream:
+ return CMaterializedArrayStream.from_c_arrays(
+ stream,
+ stream._get_cached_schema(),
+ validate=False
+ )
diff --git a/python/src/nanoarrow/_buffer.pyx b/python/src/nanoarrow/_buffer.pyx
index 575fa1eb..8dad50cd 100644
--- a/python/src/nanoarrow/_buffer.pyx
+++ b/python/src/nanoarrow/_buffer.pyx
@@ -510,7 +510,7 @@ cdef class CBufferView:
pass
def __repr__(self):
- class_label = _repr_utils.make_class_label(self,
module="nanoarrow.c_lib")
+ class_label = _repr_utils.make_class_label(self,
module="nanoarrow.c_buffer")
return f"{class_label}({_repr_utils.buffer_view_repr(self)})"
@@ -666,7 +666,7 @@ cdef class CBuffer:
self._get_buffer_count -= 1
def __repr__(self):
- class_label = _repr_utils.make_class_label(self,
module="nanoarrow.c_lib")
+ class_label = _repr_utils.make_class_label(self,
module="nanoarrow.c_buffer")
if self._ptr == NULL:
return f"{class_label}(<invalid>)"
@@ -905,7 +905,7 @@ cdef class CBufferBuilder:
return out
def __repr__(self):
- class_label = _repr_utils.make_class_label(self,
module="nanoarrow.c_lib")
+ class_label = _repr_utils.make_class_label(self,
module="nanoarrow.c_buffer")
return f"{class_label}({self.size_bytes}/{self.capacity_bytes})"
diff --git a/python/src/nanoarrow/_ipc_lib.pyi
b/python/src/nanoarrow/_ipc_lib.pyi
deleted file mode 100644
index dcd2abdf..00000000
--- a/python/src/nanoarrow/_ipc_lib.pyi
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import _cython_3_0_10
-from _typeshed import Incomplete
-
-__reduce_cython__: _cython_3_0_10.cython_function_or_method
-__setstate_cython__: _cython_3_0_10.cython_function_or_method
-__test__: dict
-init_array_stream: _cython_3_0_10.cython_function_or_method
-
-class CIpcInputStream:
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- @staticmethod
- def from_readable(*args, **kwargs): ...
- def is_valid(self, *args, **kwargs): ...
- def release(self, *args, **kwargs): ...
- def __reduce__(self): ...
-
-class PyInputStreamPrivate:
- close_obj: Incomplete
- obj: Incomplete
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- def set_buffer(self, *args, **kwargs): ...
- def __buffer__(self, *args, **kwargs):
- """Return a buffer object that exposes the underlying memory of the
object."""
- def __len__(self) -> int:
- """Return len(self)."""
- def __reduce__(self): ...
- def __release_buffer__(self, *args, **kwargs):
- """Release the buffer object that exposes the underlying memory of the
object."""
diff --git a/python/src/nanoarrow/_lib.pyi b/python/src/nanoarrow/_lib.pyi
deleted file mode 100644
index d7fb0b0f..00000000
--- a/python/src/nanoarrow/_lib.pyi
+++ /dev/null
@@ -1,595 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import _cython_3_0_10
-import enum
-import types
-from _typeshed import Incomplete
-from typing import Callable, ClassVar
-
-DEVICE_CPU: Device
-__reduce_cython__: _cython_3_0_10.cython_function_or_method
-__setstate_cython__: _cython_3_0_10.cython_function_or_method
-__test__: dict
-assert_type_equal: _cython_3_0_10.cython_function_or_method
-c_version: _cython_3_0_10.cython_function_or_method
-get_pyobject_buffer_count: _cython_3_0_10.cython_function_or_method
-sys_byteorder: str
-
-class CArray:
- __pyx_vtable__: ClassVar[PyCapsule] = ...
- buffers: Incomplete
- children: Incomplete
- device_id: Incomplete
- device_type: Incomplete
- device_type_id: Incomplete
- dictionary: Incomplete
- length: Incomplete
- n_buffers: Incomplete
- n_children: Incomplete
- null_count: Incomplete
- offset: Incomplete
- schema: Incomplete
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- @staticmethod
- def allocate(*args, **kwargs): ...
- def child(self, *args, **kwargs): ...
- def is_valid(self, *args, **kwargs): ...
- def view(self, *args, **kwargs): ...
- def __arrow_c_array__(self, *args, **kwargs):
- """
- Get a pair of PyCapsules containing a C ArrowArray representation of
the object.
-
- Parameters
- ----------
- requested_schema : PyCapsule | None
- A PyCapsule containing a C ArrowSchema representation of a
requested
- schema. Not supported.
-
- Returns
- -------
- Tuple[PyCapsule, PyCapsule]
- A pair of PyCapsules containing a C ArrowSchema and ArrowArray,
- respectively.
- """
- def __getitem__(self, index):
- """Return self[key]."""
- def __len__(self) -> int:
- """Return len(self)."""
- def __reduce__(self): ...
-
-class CArrayBuilder:
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- @staticmethod
- def allocate(*args, **kwargs): ...
- def append_bytes(self, *args, **kwargs): ...
- def append_strings(self, *args, **kwargs): ...
- def finish(self, *args, **kwargs): ...
- def init_from_schema(self, *args, **kwargs): ...
- def init_from_type(self, *args, **kwargs): ...
- def is_empty(self, *args, **kwargs): ...
- def resolve_null_count(self, *args, **kwargs): ...
- def set_buffer(self, *args, **kwargs):
- """Sets a buffer of this ArrowArray such the pointer at
array->buffers[i] is
- equal to buffer->data and such that the buffer's lifcycle is managed by
- the array. If move is True, the input Python object that previously
wrapped
- the ArrowBuffer will be invalidated, which is usually the desired
behaviour
- if you built or imported a buffer specifically to build this array. If
move
- is False (the default), this function will a make a shallow copy via
another
- layer of Python object wrapping."""
- def set_child(self, *args, **kwargs): ...
- def set_length(self, *args, **kwargs): ...
- def set_null_count(self, *args, **kwargs): ...
- def set_offset(self, *args, **kwargs): ...
- def start_appending(self, *args, **kwargs): ...
- def __reduce__(self): ...
-
-class CArrayStream:
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- @staticmethod
- def allocate(*args, **kwargs): ...
- @staticmethod
- def from_c_arrays(*args, **kwargs): ...
- def get_next(self, *args, **kwargs):
- """Get the next Array from this stream
-
- Raises StopIteration when there are no more arrays in this stream.
- """
- def get_schema(self, *args, **kwargs):
- """Get the schema associated with this stream"""
- def is_valid(self, *args, **kwargs): ...
- def release(self, *args, **kwargs): ...
- def __arrow_c_stream__(self, *args, **kwargs):
- """
- Export the stream as an Arrow C stream PyCapsule.
-
- Parameters
- ----------
- requested_schema : PyCapsule | None
- A PyCapsule containing a C ArrowSchema representation of a
requested
- schema. Not supported.
-
- Returns
- -------
- PyCapsule
- """
- def __enter__(self): ...
- def __exit__(
- self,
- type: type[BaseException] | None,
- value: BaseException | None,
- traceback: types.TracebackType | None,
- ): ...
- def __iter__(self):
- """Implement iter(self)."""
- def __next__(self): ...
- def __reduce__(self): ...
-
-class CArrayView:
- buffers: Incomplete
- children: Incomplete
- dictionary: Incomplete
- layout: Incomplete
- length: Incomplete
- n_buffers: Incomplete
- n_children: Incomplete
- null_count: Incomplete
- offset: Incomplete
- storage_type: Incomplete
- storage_type_id: Incomplete
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- def buffer(self, *args, **kwargs): ...
- def buffer_type(self, *args, **kwargs): ...
- def child(self, *args, **kwargs): ...
- @staticmethod
- def from_array(*args, **kwargs): ...
- @staticmethod
- def from_schema(*args, **kwargs): ...
- def __len__(self) -> int:
- """Return len(self)."""
- def __reduce__(self): ...
-
-class CArrowTimeUnit:
- MICRO: ClassVar[int] = ...
- MILLI: ClassVar[int] = ...
- NANO: ClassVar[int] = ...
- SECOND: ClassVar[int] = ...
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- def __reduce__(self): ...
- def __reduce_cython__(self, *args, **kwargs): ...
- def __setstate_cython__(self, *args, **kwargs): ...
-
-class CArrowType:
- BINARY: ClassVar[int] = ...
- BOOL: ClassVar[int] = ...
- DATE32: ClassVar[int] = ...
- DATE64: ClassVar[int] = ...
- DECIMAL128: ClassVar[int] = ...
- DECIMAL256: ClassVar[int] = ...
- DENSE_UNION: ClassVar[int] = ...
- DICTIONARY: ClassVar[int] = ...
- DOUBLE: ClassVar[int] = ...
- DURATION: ClassVar[int] = ...
- EXTENSION: ClassVar[int] = ...
- FIXED_SIZE_BINARY: ClassVar[int] = ...
- FIXED_SIZE_LIST: ClassVar[int] = ...
- FLOAT: ClassVar[int] = ...
- HALF_FLOAT: ClassVar[int] = ...
- INT16: ClassVar[int] = ...
- INT32: ClassVar[int] = ...
- INT64: ClassVar[int] = ...
- INT8: ClassVar[int] = ...
- INTERVAL_DAY_TIME: ClassVar[int] = ...
- INTERVAL_MONTHS: ClassVar[int] = ...
- INTERVAL_MONTH_DAY_NANO: ClassVar[int] = ...
- LARGE_BINARY: ClassVar[int] = ...
- LARGE_LIST: ClassVar[int] = ...
- LARGE_STRING: ClassVar[int] = ...
- LIST: ClassVar[int] = ...
- MAP: ClassVar[int] = ...
- NA: ClassVar[int] = ...
- SPARSE_UNION: ClassVar[int] = ...
- STRING: ClassVar[int] = ...
- STRUCT: ClassVar[int] = ...
- TIME32: ClassVar[int] = ...
- TIME64: ClassVar[int] = ...
- TIMESTAMP: ClassVar[int] = ...
- UINT16: ClassVar[int] = ...
- UINT32: ClassVar[int] = ...
- UINT64: ClassVar[int] = ...
- UINT8: ClassVar[int] = ...
- UNINITIALIZED: ClassVar[int] = ...
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- def __reduce__(self): ...
- def __reduce_cython__(self, *args, **kwargs): ...
- def __setstate_cython__(self, *args, **kwargs): ...
-
-class CBuffer:
- __pyx_vtable__: ClassVar[PyCapsule] = ...
- data_type: Incomplete
- data_type_id: Incomplete
- element_size_bits: Incomplete
- format: Incomplete
- itemsize: Incomplete
- n_elements: Incomplete
- size_bytes: Incomplete
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- def element(self, *args, **kwargs): ...
- def elements(self, *args, **kwargs): ...
- @staticmethod
- def empty(*args, **kwargs): ...
- @staticmethod
- def from_pybuffer(*args, **kwargs): ...
- def __buffer__(self, *args, **kwargs):
- """Return a buffer object that exposes the underlying memory of the
object."""
- def __getitem__(self, index):
- """Return self[key]."""
- def __iter__(self):
- """Implement iter(self)."""
- def __len__(self) -> int:
- """Return len(self)."""
- def __reduce__(self): ...
- def __release_buffer__(self, *args, **kwargs):
- """Release the buffer object that exposes the underlying memory of the
object."""
-
-class CBufferBuilder:
- __pyx_vtable__: ClassVar[PyCapsule] = ...
- capacity_bytes: Incomplete
- format: Incomplete
- itemsize: Incomplete
- size_bytes: Incomplete
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- def advance(self, *args, **kwargs):
- """Manually increase :attr:`size_bytes` by ``additional_bytes``
-
- This can be used after writing to the buffer using the buffer protocol
- to ensure that :attr:`size_bytes` accurately reflects the number of
- bytes written to the buffer.
- """
- def finish(self, *args, **kwargs):
- """Finish building this buffer
-
- Performs any steps required to finish building this buffer and
- returns the result. Any behaviour resulting from calling methods
- on this object after it has been finished is not currently
- defined (but should not crash).
- """
- def reserve_bytes(self, *args, **kwargs):
- """Ensure that the underlying buffer has space for ``additional_bytes``
- more bytes to be written"""
- def set_data_type(self, *args, **kwargs):
- """Set the data type used to interpret elements in
:meth:`write_elements`."""
- def set_format(self, *args, **kwargs):
- """Set the Python buffer format used to interpret elements in
- :meth:`write_elements`.
- """
- def write(self, *args, **kwargs):
- """Write bytes to this buffer
-
- Writes the bytes of ``content`` without considering the element type of
- ``content`` or the element type of this buffer.
-
- This method returns the number of bytes that were written.
- """
- def write_elements(self, *args, **kwargs):
- """ "Write an iterable of elements to this buffer
-
- Writes the elements of iterable ``obj`` according to the binary
- representation specified by :attr:`format`. This is currently
- powered by ``struct.pack_into()`` except when building bitmaps
- where an internal implementation is used.
-
- This method returns the number of elements that were written.
- """
- def write_fill(self, *args, **kwargs):
- """Write fill bytes to this buffer
-
- Appends the byte ``value`` to this buffer ``size_bytes`` times.
- """
- def __buffer__(self, *args, **kwargs):
- """Return a buffer object that exposes the underlying memory of the
object."""
- def __len__(self) -> int:
- """Return len(self)."""
- def __reduce__(self): ...
- def __release_buffer__(self, *args, **kwargs):
- """Release the buffer object that exposes the underlying memory of the
object."""
-
-class CBufferView:
- __pyx_vtable__: ClassVar[PyCapsule] = ...
- data_type: Incomplete
- data_type_id: Incomplete
- device: Incomplete
- element_size_bits: Incomplete
- format: Incomplete
- itemsize: Incomplete
- n_elements: Incomplete
- size_bytes: Incomplete
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- def copy(self, *args, **kwargs): ...
- def copy_into(self, *args, **kwargs): ...
- def element(self, *args, **kwargs): ...
- def elements(self, *args, **kwargs): ...
- def unpack_bits(self, *args, **kwargs): ...
- def unpack_bits_into(self, *args, **kwargs): ...
- def __buffer__(self, *args, **kwargs):
- """Return a buffer object that exposes the underlying memory of the
object."""
- def __dlpack__(self, *args, **kwargs):
- """
- Export CBufferView as a DLPack capsule.
-
- Parameters
- ----------
- stream : int, optional
- A Python integer representing a pointer to a stream. Currently not
supported.
- Stream is provided by the consumer to the producer to instruct the
producer
- to ensure that operations can safely be performed on the array.
-
- Returns
- -------
- capsule : PyCapsule
- A DLPack capsule for the array, pointing to a DLManagedTensor.
- """
- def __dlpack_device__(self, *args, **kwargs):
- """
- Return the DLPack device tuple this CBufferView resides on.
-
- Returns
- -------
- tuple : Tuple[int, int]
- Tuple with index specifying the type of the device (where
- CPU = 1, see python/src/nanoarrow/dpack_abi.h) and index of the
- device which is 0 by default for CPU.
- """
- def __getitem__(self, index):
- """Return self[key]."""
- def __iter__(self):
- """Implement iter(self)."""
- def __len__(self) -> int:
- """Return len(self)."""
- def __reduce__(self): ...
- def __release_buffer__(self, *args, **kwargs):
- """Release the buffer object that exposes the underlying memory of the
object."""
-
-class CDeviceArray:
- array: Incomplete
- device_id: Incomplete
- device_type: Incomplete
- device_type_id: Incomplete
- schema: Incomplete
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- def view(self, *args, **kwargs): ...
- def __arrow_c_array__(self, *args, **kwargs): ...
- def __arrow_c_device_array__(self, *args, **kwargs): ...
- def __reduce__(self): ...
-
-class CLayout:
- buffer_data_type_id: Incomplete
- child_size_elements: Incomplete
- element_size_bits: Incomplete
- n_buffers: Incomplete
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- def __reduce__(self): ...
-
-class CMaterializedArrayStream:
- __pyx_vtable__: ClassVar[PyCapsule] = ...
- arrays: Incomplete
- n_arrays: Incomplete
- schema: Incomplete
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- def array(self, *args, **kwargs): ...
- def child(self, *args, **kwargs): ...
- @staticmethod
- def from_c_array(*args, **kwargs): ...
- @staticmethod
- def from_c_array_stream(*args, **kwargs): ...
- @staticmethod
- def from_c_arrays(*args, **kwargs): ...
- def __arrow_c_stream__(self, *args, **kwargs): ...
- def __getitem__(self, index):
- """Return self[key]."""
- def __iter__(self):
- """Implement iter(self)."""
- def __len__(self) -> int:
- """Return len(self)."""
- def __reduce__(self): ...
-
-class CSchema:
- children: Incomplete
- dictionary: Incomplete
- flags: Incomplete
- format: Incomplete
- metadata: Incomplete
- n_children: Incomplete
- name: Incomplete
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- @staticmethod
- def allocate(*args, **kwargs): ...
- def child(self, *args, **kwargs): ...
- def is_valid(self, *args, **kwargs): ...
- def modify(self, *args, **kwargs): ...
- def type_equals(self, *args, **kwargs): ...
- def __arrow_c_schema__(self, *args, **kwargs):
- """
- Export to a ArrowSchema PyCapsule
- """
- def __deepcopy__(self): ...
- def __reduce__(self): ...
-
-class CSchemaBuilder:
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- @staticmethod
- def allocate(*args, **kwargs): ...
- def allocate_children(self, *args, **kwargs): ...
- def append_metadata(self, *args, **kwargs): ...
- def child(self, *args, **kwargs): ...
- def finish(self, *args, **kwargs): ...
- def set_child(self, *args, **kwargs): ...
- def set_dictionary(self, *args, **kwargs): ...
- def set_dictionary_ordered(self, *args, **kwargs): ...
- def set_flags(self, *args, **kwargs): ...
- def set_format(self, *args, **kwargs): ...
- def set_name(self, *args, **kwargs): ...
- def set_nullable(self, *args, **kwargs): ...
- def set_type(self, *args, **kwargs): ...
- def set_type_date_time(self, *args, **kwargs): ...
- def set_type_decimal(self, *args, **kwargs): ...
- def set_type_fixed_size(self, *args, **kwargs): ...
- def validate(self, *args, **kwargs): ...
- def __reduce__(self): ...
-
-class CSchemaView:
- _decimal_types: ClassVar[tuple] = ...
- _fixed_size_types: ClassVar[tuple] = ...
- _time_unit_types: ClassVar[tuple] = ...
- _union_types: ClassVar[tuple] = ...
- buffer_format: Incomplete
- decimal_bitwidth: Incomplete
- decimal_precision: Incomplete
- decimal_scale: Incomplete
- dictionary_ordered: Incomplete
- extension_metadata: Incomplete
- extension_name: Incomplete
- fixed_size: Incomplete
- layout: Incomplete
- map_keys_sorted: Incomplete
- nullable: Incomplete
- storage_type: Incomplete
- storage_type_id: Incomplete
- time_unit: Incomplete
- time_unit_id: Incomplete
- timezone: Incomplete
- type: Incomplete
- type_id: Incomplete
- union_type_ids: Incomplete
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- def __reduce__(self): ...
-
-class Device:
- device_id: Incomplete
- device_type: Incomplete
- device_type_id: Incomplete
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- @staticmethod
- def resolve(*args, **kwargs): ...
- def __reduce__(self): ...
-
-class DeviceType(enum.Enum):
- __new__: ClassVar[Callable] = ...
- CPU: ClassVar[DeviceType] = ...
- CUDA: ClassVar[DeviceType] = ...
- CUDA_HOST: ClassVar[DeviceType] = ...
- CUDA_MANAGED: ClassVar[DeviceType] = ...
- EXT_DEV: ClassVar[DeviceType] = ...
- HEXAGON: ClassVar[DeviceType] = ...
- METAL: ClassVar[DeviceType] = ...
- ONEAPI: ClassVar[DeviceType] = ...
- OPENCL: ClassVar[DeviceType] = ...
- ROCM: ClassVar[DeviceType] = ...
- ROCM_HOST: ClassVar[DeviceType] = ...
- VPI: ClassVar[DeviceType] = ...
- VULKAN: ClassVar[DeviceType] = ...
- WEBGPU: ClassVar[DeviceType] = ...
- _generate_next_value_: ClassVar[Callable] = ...
- _member_map_: ClassVar[dict] = ...
- _member_names_: ClassVar[list] = ...
- _member_type_: ClassVar[type[object]] = ...
- _unhashable_values_: ClassVar[list] = ...
- _use_args_: ClassVar[bool] = ...
- _value2member_map_: ClassVar[dict] = ...
- _value_repr_: ClassVar[None] = ...
-
-class Error:
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- @staticmethod
- def raise_error(*args, **kwargs):
- """Raise a NanoarrowException without a message"""
- @staticmethod
- def raise_error_not_ok(*args, **kwargs): ...
- def raise_message(self, *args, **kwargs):
- """Raise a NanoarrowException from this message"""
- def raise_message_not_ok(self, *args, **kwargs): ...
- def __reduce__(self): ...
-
-class NanoarrowException(RuntimeError):
- def __init__(self, *args, **kwargs) -> None: ...
-
-class NoneAwareWrapperIterator:
- __pyx_vtable__: ClassVar[PyCapsule] = ...
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- def finish(self, *args, **kwargs):
- """Obtain the total count, null count, and validity bitmap after
- consuming this iterable."""
- def reserve(self, *args, **kwargs): ...
- def __iter__(self):
- """Implement iter(self)."""
- def __reduce__(self): ...
-
-class SchemaMetadata:
- __pyx_vtable__: ClassVar[PyCapsule] = ...
- @classmethod
- def __init__(cls, *args, **kwargs) -> None:
- """Create and return a new object. See help(type) for accurate
signature."""
- @staticmethod
- def empty(*args, **kwargs): ...
- def items(self, *args, **kwargs): ...
- def keys(self, *args, **kwargs): ...
- def values(self, *args, **kwargs): ...
- def __contains__(self, other) -> bool:
- """Return bool(key in self)."""
- def __getitem__(self, index):
- """Return self[key]."""
- def __iter__(self):
- """Implement iter(self)."""
- def __len__(self) -> int:
- """Return len(self)."""
- def __reduce__(self): ...
diff --git a/python/src/nanoarrow/_repr_utils.py
b/python/src/nanoarrow/_repr_utils.py
index 7658ae79..df53a206 100644
--- a/python/src/nanoarrow/_repr_utils.py
+++ b/python/src/nanoarrow/_repr_utils.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-# The functions here are imported in _lib.pyx. They're defined here
+# The functions here are imported from Cython. They're defined here
# instead of there to make it easier to iterate (no need to rebuild
# after editing when working with an editable installation)
diff --git a/python/src/nanoarrow/array.py b/python/src/nanoarrow/array.py
index 15a24eea..4a7210fb 100644
--- a/python/src/nanoarrow/array.py
+++ b/python/src/nanoarrow/array.py
@@ -19,9 +19,10 @@ import itertools
from functools import cached_property
from typing import Iterable, Tuple
+from nanoarrow._array import CArray, CArrayView
+from nanoarrow._array_stream import CMaterializedArrayStream
from nanoarrow._buffer import CBufferView
from nanoarrow._device import DEVICE_CPU, Device
-from nanoarrow._lib import CArray, CArrayView, CMaterializedArrayStream
from nanoarrow.c_array import c_array, c_array_view
from nanoarrow.c_array_stream import c_array_stream
from nanoarrow.c_schema import c_schema
@@ -299,7 +300,7 @@ class Array(ArrayViewVisitable):
>>> import nanoarrow as na
>>> array = na.Array([1, 2, 3], na.int32())
>>> array.buffer(1)
- nanoarrow.c_lib.CBufferView(int32[12 b] 1 2 3)
+ nanoarrow.c_buffer.CBufferView(int32[12 b] 1 2 3)
"""
return self.buffers[i]
@@ -314,8 +315,8 @@ class Array(ArrayViewVisitable):
>>> array = na.Array([1, 2, 3], na.int32())
>>> for buffer in array.buffers:
... print(buffer)
- nanoarrow.c_lib.CBufferView(bool[0 b] )
- nanoarrow.c_lib.CBufferView(int32[12 b] 1 2 3)
+ nanoarrow.c_buffer.CBufferView(bool[0 b] )
+ nanoarrow.c_buffer.CBufferView(int32[12 b] 1 2 3)
"""
view = c_array_view(self)
return tuple(view.buffers)
@@ -335,8 +336,8 @@ class Array(ArrayViewVisitable):
... print(validity)
... print(data)
0 3
- nanoarrow.c_lib.CBufferView(bool[0 b] )
- nanoarrow.c_lib.CBufferView(int32[12 b] 1 2 3)
+ nanoarrow.c_buffer.CBufferView(bool[0 b] )
+ nanoarrow.c_buffer.CBufferView(int32[12 b] 1 2 3)
"""
return iter_array_views(self)
diff --git a/python/src/nanoarrow/array_stream.py
b/python/src/nanoarrow/array_stream.py
index 88f70211..e7282721 100644
--- a/python/src/nanoarrow/array_stream.py
+++ b/python/src/nanoarrow/array_stream.py
@@ -18,7 +18,7 @@
from functools import cached_property
from typing import Iterable, Tuple
-from nanoarrow._lib import CMaterializedArrayStream
+from nanoarrow._array_stream import CMaterializedArrayStream
from nanoarrow._repr_utils import make_class_label
from nanoarrow.array import Array
from nanoarrow.c_array_stream import c_array_stream
diff --git a/python/src/nanoarrow/c_array.py b/python/src/nanoarrow/c_array.py
index 579e64a1..062b8033 100644
--- a/python/src/nanoarrow/c_array.py
+++ b/python/src/nanoarrow/c_array.py
@@ -17,8 +17,8 @@
from typing import Any, Iterable, Literal, Tuple
+from nanoarrow._array import CArray, CArrayBuilder, CArrayView
from nanoarrow._buffer import CBuffer, CBufferBuilder, NoneAwareWrapperIterator
-from nanoarrow._lib import CArray, CArrayBuilder, CArrayView
from nanoarrow._schema import CSchema, CSchemaBuilder
from nanoarrow._utils import obj_is_buffer, obj_is_capsule
from nanoarrow.c_buffer import c_buffer
diff --git a/python/src/nanoarrow/c_array_stream.py
b/python/src/nanoarrow/c_array_stream.py
index ad1fd981..4628a20d 100644
--- a/python/src/nanoarrow/c_array_stream.py
+++ b/python/src/nanoarrow/c_array_stream.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from nanoarrow._lib import CArrayStream
+from nanoarrow._array_stream import CArrayStream
from nanoarrow._utils import obj_is_capsule
from nanoarrow.c_array import c_array
from nanoarrow.c_schema import c_schema
diff --git a/python/src/nanoarrow/c_buffer.py b/python/src/nanoarrow/c_buffer.py
index a25c7c40..f1c1fd4b 100644
--- a/python/src/nanoarrow/c_buffer.py
+++ b/python/src/nanoarrow/c_buffer.py
@@ -57,9 +57,9 @@ def c_buffer(obj, schema=None) -> CBuffer:
>>> import nanoarrow as na
>>> na.c_buffer(b"1234")
- nanoarrow.c_lib.CBuffer(uint8[4 b] 49 50 51 52)
+ nanoarrow.c_buffer.CBuffer(uint8[4 b] 49 50 51 52)
>>> na.c_buffer([1, 2, 3], na.int32())
- nanoarrow.c_lib.CBuffer(int32[12 b] 1 2 3)
+ nanoarrow.c_buffer.CBuffer(int32[12 b] 1 2 3)
"""
if isinstance(obj, CBuffer) and schema is None:
return obj
diff --git a/python/src/nanoarrow/device.py b/python/src/nanoarrow/device.py
index ff8f9826..ee62cf5c 100644
--- a/python/src/nanoarrow/device.py
+++ b/python/src/nanoarrow/device.py
@@ -15,8 +15,8 @@
# specific language governing permissions and limitations
# under the License.
+from nanoarrow._array import CDeviceArray
from nanoarrow._device import DEVICE_CPU, Device, DeviceType # noqa: F401
-from nanoarrow._lib import CDeviceArray
from nanoarrow.c_array import c_array
from nanoarrow.c_schema import c_schema
diff --git a/python/src/nanoarrow/ipc.py b/python/src/nanoarrow/ipc.py
index 86d141f3..91916904 100644
--- a/python/src/nanoarrow/ipc.py
+++ b/python/src/nanoarrow/ipc.py
@@ -17,8 +17,8 @@
import io
+from nanoarrow._array_stream import CArrayStream
from nanoarrow._ipc_lib import CIpcInputStream, init_array_stream
-from nanoarrow._lib import CArrayStream
from nanoarrow._utils import obj_is_buffer
from nanoarrow import _repr_utils
diff --git a/python/src/nanoarrow/iterator.py b/python/src/nanoarrow/iterator.py
index d448adf2..874f285f 100644
--- a/python/src/nanoarrow/iterator.py
+++ b/python/src/nanoarrow/iterator.py
@@ -20,7 +20,7 @@ from functools import cached_property
from itertools import islice, repeat
from typing import Iterable, Tuple
-from nanoarrow._lib import CArrayView
+from nanoarrow._array import CArrayView
from nanoarrow.c_array_stream import c_array_stream
from nanoarrow.c_schema import c_schema, c_schema_view
from nanoarrow.schema import Schema
diff --git a/python/src/nanoarrow/visitor.py b/python/src/nanoarrow/visitor.py
index 8a81b733..d1f95e47 100644
--- a/python/src/nanoarrow/visitor.py
+++ b/python/src/nanoarrow/visitor.py
@@ -17,8 +17,8 @@
from typing import Any, Callable, List, Sequence, Tuple, Union
+from nanoarrow._array import CArrayView
from nanoarrow._buffer import CBuffer, CBufferBuilder
-from nanoarrow._lib import CArrayView
from nanoarrow.c_array_stream import c_array_stream
from nanoarrow.c_schema import c_schema_view
from nanoarrow.iterator import ArrayViewBaseIterator, PyIterator
@@ -76,7 +76,7 @@ class ArrayViewVisitable:
>>> names
['col1', 'col2']
>>> columns
- [nanoarrow.c_lib.CBuffer(int64[24 b] 1 2 3), ['a', 'b', 'c']]
+ [nanoarrow.c_buffer.CBuffer(int64[24 b] 1 2 3), ['a', 'b', 'c']]
"""
return ToColumnsPysequenceConverter.visit(self,
handle_nulls=handle_nulls)
@@ -105,7 +105,7 @@ class ArrayViewVisitable:
--------
>>> import nanoarrow as na
>>> na.Array([1, 2, 3], na.int32()).to_pysequence()
- nanoarrow.c_lib.CBuffer(int32[12 b] 1 2 3)
+ nanoarrow.c_buffer.CBuffer(int32[12 b] 1 2 3)
"""
return ToPySequenceConverter.visit(self, handle_nulls=handle_nulls)
@@ -120,7 +120,7 @@ def nulls_forbid() -> Callable[[CBuffer, Sequence],
Sequence]:
>>> import nanoarrow as na
>>> na.Array([1, 2, 3],
na.int32()).to_pysequence(handle_nulls=na.nulls_forbid())
- nanoarrow.c_lib.CBuffer(int32[12 b] 1 2 3)
+ nanoarrow.c_buffer.CBuffer(int32[12 b] 1 2 3)
>>> na.Array([1, None, 3],
na.int32()).to_pysequence(handle_nulls=na.nulls_forbid())
Traceback (most recent call last):
...
@@ -193,13 +193,13 @@ def nulls_separate() -> Callable[[CBuffer, Sequence],
Tuple[CBuffer, Sequence]]:
>>> import nanoarrow as na
>>> na_array = na.Array([1, 2, 3], na.int32())
>>> na_array.to_pysequence(handle_nulls=na.nulls_separate())
- (None, nanoarrow.c_lib.CBuffer(int32[12 b] 1 2 3))
+ (None, nanoarrow.c_buffer.CBuffer(int32[12 b] 1 2 3))
>>> na_array = na.Array([1, None, 3], na.int32())
>>> result = na_array.to_pysequence(handle_nulls=na.nulls_separate())
>>> result[0]
- nanoarrow.c_lib.CBuffer(uint8[3 b] True False True)
+ nanoarrow.c_buffer.CBuffer(uint8[3 b] True False True)
>>> result[1]
- nanoarrow.c_lib.CBuffer(int32[12 b] 1 0 3)
+ nanoarrow.c_buffer.CBuffer(int32[12 b] 1 0 3)
"""
def handle(is_valid, data):
diff --git a/python/tests/test_c_array.py b/python/tests/test_c_array.py
index 0a74c756..a28bacd5 100644
--- a/python/tests/test_c_array.py
+++ b/python/tests/test_c_array.py
@@ -19,7 +19,7 @@ import array
from datetime import date, datetime, timezone
import pytest
-from nanoarrow._lib import CArrayBuilder
+from nanoarrow._array import CArrayBuilder
from nanoarrow._utils import NanoarrowException
from nanoarrow.c_schema import c_schema_view
diff --git a/python/tests/test_c_buffer.py b/python/tests/test_c_buffer.py
index 28962204..41db69b8 100644
--- a/python/tests/test_c_buffer.py
+++ b/python/tests/test_c_buffer.py
@@ -35,7 +35,7 @@ def test_buffer_invalid():
with pytest.raises(RuntimeError, match="CBuffer is not valid"):
memoryview(invalid)
- assert repr(invalid) == "nanoarrow.c_lib.CBuffer(<invalid>)"
+ assert repr(invalid) == "nanoarrow.c_buffer.CBuffer(<invalid>)"
def test_c_buffer_constructor():
@@ -68,7 +68,7 @@ def test_c_buffer_empty():
assert empty.size_bytes == 0
assert bytes(empty) == b""
- assert repr(empty) == "nanoarrow.c_lib.CBuffer(binary[0 b] b'')"
+ assert repr(empty) == "nanoarrow.c_buffer.CBuffer(binary[0 b] b'')"
# Export it via the Python buffer protocol wrapped in a new CBuffer
empty_roundtrip = na.c_buffer(empty)
@@ -85,7 +85,7 @@ def test_c_buffer_pybuffer():
assert buffer.size_bytes == len(data)
assert bytes(buffer) == b"abcdefghijklmnopqrstuvwxyz"
- assert repr(buffer).startswith("nanoarrow.c_lib.CBuffer(uint8[26 b] 97 98")
+ assert repr(buffer).startswith("nanoarrow.c_buffer.CBuffer(uint8[26 b] 97
98")
def test_c_buffer_unsupported_type():
@@ -201,7 +201,7 @@ def test_c_buffer_builder():
builder = CBufferBuilder()
assert builder.size_bytes == 0
assert builder.capacity_bytes == 0
- assert repr(builder) == "nanoarrow.c_lib.CBufferBuilder(0/0)"
+ assert repr(builder) == "nanoarrow.c_buffer.CBufferBuilder(0/0)"
builder.reserve_bytes(123)
assert builder.size_bytes == 0
