paleolimbot commented on code in PR #444: URL: https://github.com/apache/arrow-nanoarrow/pull/444#discussion_r1584764466
########## python/src/nanoarrow/c_array.py: ########## @@ -0,0 +1,573 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Any, Iterable, Literal, Tuple + +from nanoarrow._lib import ( + CArray, + CArrayBuilder, + CArrayView, + CArrowType, + CBuffer, + CBufferBuilder, + CSchema, + CSchemaBuilder, + NoneAwareWrapperIterator, + _obj_is_buffer, + _obj_is_capsule, +) +from nanoarrow.c_buffer import c_buffer +from nanoarrow.c_schema import c_schema, c_schema_view + + +def c_array(obj, schema=None) -> CArray: + """ArrowArray wrapper + + This class provides a user-facing interface to access the fields of an ArrowArray + as defined in the Arrow C Data interface, holding an optional reference to a + :class:`CSchema` that can be used to safely deserialize the content. + + These objects are created using :func:`c_array`, which accepts any array-like + object according to the Arrow PyCapsule interface, Python buffer protocol, + or iterable of Python objects. + + This Python wrapper allows access to array fields but does not automatically + deserialize their content: use :func:`c_array_view` to validate and deserialize + the content into a more easily inspectable object. + + Note that the :class:`CArray` objects returned by ``.child()`` hold strong + references to the original ``ArrowArray`` to avoid copies while inspecting an + imported structure. + + Parameters + ---------- + obj : array-like + An object supporting the Arrow PyCapsule interface, the Python buffer + protocol, or an iterable of Python objects. + schema : schema-like or None + A schema-like object as sanitized by :func:`c_schema` or None. This value + will be used to request a data type from ``obj``; however, the conversion + is best-effort (i.e., the data type of the returned ``CArray`` may be + different than ``schema``). + + Examples + -------- + + >>> import nanoarrow as na + >>> # Create from iterable + >>> array = na.c_array([1, 2, 3], na.int32()) + >>> # Create from Python buffer (e.g., numpy array) + >>> import numpy as np + >>> array = na.c_array(np.array([1, 2, 3])) + >>> # Create from Arrow PyCapsule (e.g., pyarrow array) + >>> import pyarrow as pa + >>> array = na.c_array(pa.array([1, 2, 3])) + >>> # Access array fields + >>> array.length + 3 + >>> array.null_count + 0 + """ + + if schema is not None: + schema = c_schema(schema) + + if isinstance(obj, CArray) and schema is None: + return obj + + # Try Arrow PyCapsule protocol + if hasattr(obj, "__arrow_c_array__"): + schema_capsule = None if schema is None else schema.__arrow_c_schema__() + return CArray._import_from_c_capsule( + *obj.__arrow_c_array__(requested_schema=schema_capsule) + ) + + # Try import of bare capsule + if _obj_is_capsule(obj, "arrow_array"): + if schema is None: + schema_capsule = CSchema.allocate()._capsule + else: + schema_capsule = schema.__arrow_c_schema__() + + return CArray._import_from_c_capsule(schema_capsule, obj) + + # Try _export_to_c for Array/RecordBatch objects if pyarrow < 14.0 + if _obj_is_pyarrow_array(obj): + out = CArray.allocate(CSchema.allocate()) + obj._export_to_c(out._addr(), out.schema._addr()) + return out + + # Use the ArrayBuilder classes to handle various strategies for other + # types of objects (e.g., iterable, pybuffer, empty). + builder_cls = _resolve_builder(obj) + + if schema is None: + obj, schema = builder_cls.infer_schema(obj) + + builder = builder_cls(schema) + return builder.build_c_array(obj) + + +def _resolve_builder(obj): + if _obj_is_empty(obj): + return EmptyArrayBuilder + + if _obj_is_buffer(obj): + return ArrayFromPyBufferBuilder + + if _obj_is_iterable(obj): + return ArrayFromIterableBuilder + + raise TypeError( + f"Can't resolve ArrayBuilder for object of type {type(obj).__name__}" + ) + + +def allocate_c_array(schema=None) -> CArray: + """Allocate an uninitialized ArrowArray + + Examples + -------- + + >>> import pyarrow as pa + >>> import nanoarrow as na + >>> schema = na.allocate_c_schema() + >>> pa.int32()._export_to_c(schema._addr()) + """ + if schema is not None: + schema = c_schema(schema) + + return CArray.allocate(CSchema.allocate() if schema is None else schema) + + +def c_array_view(obj, schema=None) -> CArrayView: + """ArrowArrayView wrapper + + The ``ArrowArrayView`` is a nanoarrow C library structure that provides + structured access to buffers addresses, buffer sizes, and buffer + data types. The buffer data is usually propagated from an ArrowArray + but can also be propagated from other types of objects (e.g., serialized + IPC). The offset and length of this view are independent of its parent + (i.e., this object can also represent a slice of its parent). + + Examples + -------- + + >>> import pyarrow as pa + >>> import numpy as np + >>> import nanoarrow as na + >>> array = na.c_array(pa.array(["one", "two", "three", None])) + >>> array_view = na.c_array_view(array) + >>> np.array(array_view.buffer(1)) + array([ 0, 3, 6, 11, 11], dtype=int32) + >>> np.array(array_view.buffer(2)) + array([b'o', b'n', b'e', b't', b'w', b'o', b't', b'h', b'r', b'e', b'e'], + dtype='|S1') + """ + + if isinstance(obj, CArrayView) and schema is None: + return obj + + return c_array(obj, schema).view() + + +def c_array_from_buffers( + schema, + length: int, + buffers: Iterable[Any], + null_count: int = -1, + offset: int = 0, + children: Iterable[Any] = (), + validation_level: Literal[None, "full", "default", "minimal", "none"] = None, + move: bool = False, +) -> CArray: + """Create an ArrowArray wrapper from components + + Given a schema, build an ArrowArray buffer-wise. This allows almost any array + to be assembled; however, requires some knowledge of the Arrow Columnar + specification. This function will do its best to validate the sizes and + content of buffers according to ``validation_level``; however, not all + types of arrays can currently be validated when constructed in this way. + + Parameters + ---------- + schema : schema-like + The data type of the desired array as sanitized by :func:`c_schema`. + length : int + The length of the output array. + buffers : Iterable of buffer-like or None + An iterable of buffers as sanitized by :func:`c_buffer`. Any object + supporting the Python Buffer protocol is accepted. Buffer data types + are not checked. A buffer value of ``None`` will skip setting a buffer + (i.e., that buffer will be of length zero and its pointer will + be ``NULL``). + null_count : int, optional + The number of null values, if known in advance. If -1 (the default), + the null count will be calculated based on the validity bitmap. If + the validity bitmap was set to ``None``, the calculated null count + will be zero. + offset : int, optional + The logical offset from the start of the array. + children : Iterable of array-like + An iterable of arrays used to set child fields of the array. Can contain + any object accepted by :func:`c_array`. Must contain the exact number of + required children as specifed by ``schema``. + validation_level: None or str, optional + One of "none" (no check), "minimal" (check buffer sizes that do not require + dereferencing buffer content), "default" (check all buffer sizes), or "full" + (check all buffer sizes and all buffer content). The default, ``None``, + will validate at the "default" level where possible. + move : bool, optional + Use ``True`` to move ownership of any input buffers or children to the + output array. + + Examples + -------- + + >>> import nanoarrow as na + >>> c_array = na.c_array_from_buffers(na.uint8(), 5, [None, b"12345"]) + >>> na.c_array_view(c_array) + <nanoarrow.c_lib.CArrayView> + - storage_type: 'uint8' + - length: 5 + - offset: 0 + - null_count: 0 + - buffers[2]: + - validity <bool[0 b] > + - data <uint8[5 b] 49 50 51 52 53> + - dictionary: NULL + - children[0]: + """ + schema = c_schema(schema) + builder = CArrayBuilder.allocate() + + # Ensures that the output array->n_buffers is set and that the correct number + # of children have been initialized. + builder.init_from_schema(schema) + + # Set buffers, optionally moving ownership of the buffers as well (i.e., + # the objects in the input buffers would be replaced with an empty ArrowBuffer) + for i, buffer in enumerate(buffers): + if buffer is None: + continue + + # If we're setting a CBuffer from something else, we can avoid an extra + # level of Python wrapping by using move=True + move = move or not isinstance(buffer, CBuffer) + builder.set_buffer(i, c_buffer(buffer), move=move) + + # Set children, optionally moving ownership of the children as well (i.e., + # the objects in the input children would be marked released). + n_children = 0 + for child_src in children: + # If we're setting a CArray from something else, we can avoid an extra + # level of Python wrapping by using move=True + move = move or not isinstance(child_src, CArray) + builder.set_child(n_children, c_array(child_src), move=move) + n_children += 1 + + if n_children != schema.n_children: + raise ValueError(f"Expected {schema.n_children} children but got {n_children}") + + # Set array fields + builder.set_length(length) + builder.set_offset(offset) + builder.set_null_count(null_count) + + # Calculates the null count if -1 (and if applicable) + builder.resolve_null_count() + + # Validate + finish + return builder.finish(validation_level=validation_level) + + +class ArrayBuilder: + """Internal utility to build CArrays from various types of input + + This class and its subclasses are designed to help separate the code + that actually builds a CArray from the code that chooses the strategy + used to do the building. + """ + + @classmethod + def infer_schema(cls, obj) -> Tuple[CSchema, Any]: + """Infer the Arrow data type from a target object + + Returns the type as a :class:`CSchema` and an object that can be + consumed in the same way by append() in the event it had to be + modified to infer its type (e.g., for an iterable, it would be + necessary to consume the first element from the original iterator). + """ + raise NotImplementedError() + + def __init__(self, schema): + self._schema = c_schema(schema) + self._schema_view = c_schema_view(self._schema) + + def build_c_array(self, obj): + builder = CArrayBuilder.allocate() + builder.init_from_schema(self._schema) + self.start_building(builder) + self.append(builder, obj) + return self.finish_building(builder) + + def start_building(self, c_builder: CArrayBuilder) -> None: + pass + + def append(self, c_builder: CArrayBuilder, obj: Any) -> None: + raise NotImplementedError() + + def finish_building(self, c_builder: CArrayBuilder) -> CArray: + return c_builder.finish() + + +class EmptyArrayBuilder(ArrayBuilder): + """Build an empty CArray of any type + + This builder accepts any empty input and produces a valid length zero + array as output. + """ + + @classmethod + def infer_schema(cls, obj) -> Tuple[Any, CSchema]: + return obj, CSchemaBuilder.allocate().set_type(CArrowType.NA) + + def start_building(self, c_builder: CArrayBuilder) -> None: + c_builder.start_appending() + + def append(self, c_builder: CArrayBuilder, obj: Any) -> None: + if len(obj) != 0: + raise ValueError( + f"Can't build empty array from {type(obj).__name__} " + f"with length {len(obj)}" + ) + + +class ArrayFromPyBufferBuilder(ArrayBuilder): + """Build a CArray from a Python Buffer + + This builder converts a Python buffer (e.g., numpy array, bytes, array.array) + to a CArray (without copying the contents of the buffer). + """ + + @classmethod + def infer_schema(cls, obj) -> Tuple[CBuffer, CSchema]: + if not isinstance(obj, CBuffer): + obj = CBuffer.from_pybuffer(obj) + + type_id = obj.data_type_id + element_size_bits = obj.element_size_bits + + # Fixed-size binary needs a schema + if type_id == CArrowType.BINARY and element_size_bits != 0: + schema = ( + CSchemaBuilder.allocate() + .set_type_fixed_size( + CArrowType.FIXED_SIZE_BINARY, element_size_bits // 8 + ) + .finish() + ) + elif type_id == CArrowType.STRING: + schema = CSchemaBuilder.allocate().set_type(CArrowType.INT8).finish() + elif type_id == CArrowType.BINARY: + schema = CSchemaBuilder.allocate().set_type(CArrowType.UINT8).finish() + else: + schema = CSchemaBuilder.allocate().set_type(type_id).finish() + + return obj, schema + + def __init__(self, schema): + super().__init__(schema) + + if self._schema_view.buffer_format is None: + raise ValueError( + f"Can't build array of type {self._schema_view.type} from PyBuffer" + ) + + def append(self, c_builder: CArrayBuilder, obj: Any) -> None: + if not c_builder.empty(): + raise ValueError("Can't append to non-empty ArrayFromPyBufferBuilder") + + if not isinstance(obj, CBuffer): + obj = CBuffer.from_pybuffer(obj) + + if ( + self._schema_view.buffer_format in ("b", "c") + and obj.format not in ("b", "c") + ) and self._schema_view.buffer_format != obj.format: + raise ValueError( + f"Expected buffer with format '{self._schema_view.buffer_format}' " + f"but got buffer with format '{obj.format}'" + ) + + c_builder.set_buffer(1, obj) + c_builder.set_length(len(obj)) + c_builder.set_null_count(0) + c_builder.set_offset(0) + + +class ArrayFromIterableBuilder(ArrayBuilder): + """Build a CArray from an iterable of scalar objects + + This builder converts an iterable to a CArray using some heuristics to pick + the fastest available method for converting to a particular type of array. + Briefly, the methods are (1) ArrowArrayAppendXXX() functions from the C + library (string, binary), (2) array.array() (integer/float except float16), + (3) CBufferBuilder.write_elements() (everything else). + """ + + @classmethod + def infer_schema(cls, obj) -> Tuple[CBuffer, CSchema]: + raise ValueError("schema is required to build array from iterable") + + def __init__(self, schema): + super().__init__(schema) + + # Resolve the method name we are going to use to do the building from + # the provided schema. + type_id = self._schema_view.type_id + if type_id not in _ARRAY_BUILDER_FROM_ITERABLE_METHOD: + raise ValueError( + f"Can't build array of type {self._schema_view.type} from iterable" + ) + + method_name = _ARRAY_BUILDER_FROM_ITERABLE_METHOD[type_id] + + # If there might be nulls, we may need to pick a different strategy + if ( + self._schema_view.nullable + and method_name in _ARRAY_BUILDER_FROM_NULLABLE_ITERABLE_METHOD + ): + method_name = _ARRAY_BUILDER_FROM_NULLABLE_ITERABLE_METHOD[method_name] + + self._append_impl = getattr(self, method_name) + + def start_building(self, c_builder: CArrayBuilder) -> None: + c_builder.start_appending() + + def append(self, c_builder: CArrayBuilder, obj: Any) -> None: + self._append_impl(c_builder, obj) + + def _append_strings(self, c_builder: CArrayBuilder, obj: Iterable) -> None: + c_builder.append_strings(obj) + + def _append_bytes(self, c_builder: CArrayBuilder, obj: Iterable) -> None: + c_builder.append_bytes(obj) + + def _build_nullable_array_using_array( + self, c_builder: CArrayBuilder, obj: Iterable + ) -> None: + wrapper = NoneAwareWrapperIterator( + obj, self._schema_view.storage_type_id, self._schema_view.fixed_size + ) + self._append_using_array(c_builder, wrapper) + + _, null_count, validity = wrapper.finish() + if validity is not None: + c_builder.set_buffer(0, validity, move=True) + + c_builder.set_null_count(null_count) + + def _build_nullable_array_using_buffer_builder( + self, c_builder: CArrayBuilder, obj: Iterable + ) -> None: + wrapper = NoneAwareWrapperIterator( + obj, self._schema_view.storage_type_id, self._schema_view.fixed_size + ) + self._append_using_buffer_builder(c_builder, wrapper) + + _, null_count, validity = wrapper.finish() + if validity is not None: + c_builder.set_buffer(0, validity, move=True) + + c_builder.set_null_count(null_count) + + def _append_using_array(self, c_builder: CArrayBuilder, obj: Iterable) -> None: Review Comment: The `array` module only implements a subset of buffer protocol strings (basically just integers and doubles and floats). Float16, interval, and fixed-size binary all have valid buffer protocol format strings (but no array implementation), and boolean has no buffer protocol format string (but nanoarrow's Cython buffer builder can handle it anyway). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
