paleolimbot commented on code in PR #444: URL: https://github.com/apache/arrow-nanoarrow/pull/444#discussion_r1584787038
########## python/src/nanoarrow/c_array.py: ########## @@ -0,0 +1,573 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Any, Iterable, Literal, Tuple + +from nanoarrow._lib import ( + CArray, + CArrayBuilder, + CArrayView, + CArrowType, + CBuffer, + CBufferBuilder, + CSchema, + CSchemaBuilder, + NoneAwareWrapperIterator, + _obj_is_buffer, + _obj_is_capsule, +) +from nanoarrow.c_buffer import c_buffer +from nanoarrow.c_schema import c_schema, c_schema_view + + +def c_array(obj, schema=None) -> CArray: + """ArrowArray wrapper + + This class provides a user-facing interface to access the fields of an ArrowArray + as defined in the Arrow C Data interface, holding an optional reference to a + :class:`CSchema` that can be used to safely deserialize the content. + + These objects are created using :func:`c_array`, which accepts any array-like + object according to the Arrow PyCapsule interface, Python buffer protocol, + or iterable of Python objects. + + This Python wrapper allows access to array fields but does not automatically + deserialize their content: use :func:`c_array_view` to validate and deserialize + the content into a more easily inspectable object. + + Note that the :class:`CArray` objects returned by ``.child()`` hold strong + references to the original ``ArrowArray`` to avoid copies while inspecting an + imported structure. + + Parameters + ---------- + obj : array-like + An object supporting the Arrow PyCapsule interface, the Python buffer + protocol, or an iterable of Python objects. + schema : schema-like or None + A schema-like object as sanitized by :func:`c_schema` or None. This value + will be used to request a data type from ``obj``; however, the conversion + is best-effort (i.e., the data type of the returned ``CArray`` may be + different than ``schema``). + + Examples + -------- + + >>> import nanoarrow as na + >>> # Create from iterable + >>> array = na.c_array([1, 2, 3], na.int32()) + >>> # Create from Python buffer (e.g., numpy array) + >>> import numpy as np + >>> array = na.c_array(np.array([1, 2, 3])) + >>> # Create from Arrow PyCapsule (e.g., pyarrow array) + >>> import pyarrow as pa + >>> array = na.c_array(pa.array([1, 2, 3])) + >>> # Access array fields + >>> array.length + 3 + >>> array.null_count + 0 + """ + + if schema is not None: + schema = c_schema(schema) + + if isinstance(obj, CArray) and schema is None: + return obj + + # Try Arrow PyCapsule protocol + if hasattr(obj, "__arrow_c_array__"): + schema_capsule = None if schema is None else schema.__arrow_c_schema__() + return CArray._import_from_c_capsule( + *obj.__arrow_c_array__(requested_schema=schema_capsule) + ) + + # Try import of bare capsule + if _obj_is_capsule(obj, "arrow_array"): + if schema is None: + schema_capsule = CSchema.allocate()._capsule + else: + schema_capsule = schema.__arrow_c_schema__() + + return CArray._import_from_c_capsule(schema_capsule, obj) + + # Try _export_to_c for Array/RecordBatch objects if pyarrow < 14.0 + if _obj_is_pyarrow_array(obj): + out = CArray.allocate(CSchema.allocate()) + obj._export_to_c(out._addr(), out.schema._addr()) + return out + + # Use the ArrayBuilder classes to handle various strategies for other + # types of objects (e.g., iterable, pybuffer, empty). + builder_cls = _resolve_builder(obj) + + if schema is None: + obj, schema = builder_cls.infer_schema(obj) + + builder = builder_cls(schema) + return builder.build_c_array(obj) + + +def _resolve_builder(obj): + if _obj_is_empty(obj): + return EmptyArrayBuilder + + if _obj_is_buffer(obj): + return ArrayFromPyBufferBuilder + + if _obj_is_iterable(obj): + return ArrayFromIterableBuilder + + raise TypeError( + f"Can't resolve ArrayBuilder for object of type {type(obj).__name__}" + ) + + +def allocate_c_array(schema=None) -> CArray: + """Allocate an uninitialized ArrowArray + + Examples + -------- + + >>> import pyarrow as pa + >>> import nanoarrow as na + >>> schema = na.allocate_c_schema() + >>> pa.int32()._export_to_c(schema._addr()) + """ + if schema is not None: + schema = c_schema(schema) + + return CArray.allocate(CSchema.allocate() if schema is None else schema) + + +def c_array_view(obj, schema=None) -> CArrayView: + """ArrowArrayView wrapper + + The ``ArrowArrayView`` is a nanoarrow C library structure that provides + structured access to buffers addresses, buffer sizes, and buffer + data types. The buffer data is usually propagated from an ArrowArray + but can also be propagated from other types of objects (e.g., serialized + IPC). The offset and length of this view are independent of its parent + (i.e., this object can also represent a slice of its parent). + + Examples + -------- + + >>> import pyarrow as pa + >>> import numpy as np + >>> import nanoarrow as na + >>> array = na.c_array(pa.array(["one", "two", "three", None])) + >>> array_view = na.c_array_view(array) + >>> np.array(array_view.buffer(1)) + array([ 0, 3, 6, 11, 11], dtype=int32) + >>> np.array(array_view.buffer(2)) + array([b'o', b'n', b'e', b't', b'w', b'o', b't', b'h', b'r', b'e', b'e'], + dtype='|S1') + """ + + if isinstance(obj, CArrayView) and schema is None: + return obj + + return c_array(obj, schema).view() + + +def c_array_from_buffers( + schema, + length: int, + buffers: Iterable[Any], + null_count: int = -1, + offset: int = 0, + children: Iterable[Any] = (), + validation_level: Literal[None, "full", "default", "minimal", "none"] = None, + move: bool = False, +) -> CArray: + """Create an ArrowArray wrapper from components + + Given a schema, build an ArrowArray buffer-wise. This allows almost any array + to be assembled; however, requires some knowledge of the Arrow Columnar + specification. This function will do its best to validate the sizes and + content of buffers according to ``validation_level``; however, not all + types of arrays can currently be validated when constructed in this way. + + Parameters + ---------- + schema : schema-like + The data type of the desired array as sanitized by :func:`c_schema`. + length : int + The length of the output array. + buffers : Iterable of buffer-like or None + An iterable of buffers as sanitized by :func:`c_buffer`. Any object + supporting the Python Buffer protocol is accepted. Buffer data types + are not checked. A buffer value of ``None`` will skip setting a buffer + (i.e., that buffer will be of length zero and its pointer will + be ``NULL``). + null_count : int, optional + The number of null values, if known in advance. If -1 (the default), + the null count will be calculated based on the validity bitmap. If + the validity bitmap was set to ``None``, the calculated null count + will be zero. + offset : int, optional + The logical offset from the start of the array. + children : Iterable of array-like + An iterable of arrays used to set child fields of the array. Can contain + any object accepted by :func:`c_array`. Must contain the exact number of + required children as specifed by ``schema``. + validation_level: None or str, optional + One of "none" (no check), "minimal" (check buffer sizes that do not require + dereferencing buffer content), "default" (check all buffer sizes), or "full" + (check all buffer sizes and all buffer content). The default, ``None``, + will validate at the "default" level where possible. + move : bool, optional + Use ``True`` to move ownership of any input buffers or children to the + output array. + + Examples + -------- + + >>> import nanoarrow as na + >>> c_array = na.c_array_from_buffers(na.uint8(), 5, [None, b"12345"]) + >>> na.c_array_view(c_array) + <nanoarrow.c_lib.CArrayView> + - storage_type: 'uint8' + - length: 5 + - offset: 0 + - null_count: 0 + - buffers[2]: + - validity <bool[0 b] > + - data <uint8[5 b] 49 50 51 52 53> + - dictionary: NULL + - children[0]: + """ + schema = c_schema(schema) + builder = CArrayBuilder.allocate() + + # Ensures that the output array->n_buffers is set and that the correct number + # of children have been initialized. + builder.init_from_schema(schema) + + # Set buffers, optionally moving ownership of the buffers as well (i.e., + # the objects in the input buffers would be replaced with an empty ArrowBuffer) + for i, buffer in enumerate(buffers): + if buffer is None: + continue + + # If we're setting a CBuffer from something else, we can avoid an extra + # level of Python wrapping by using move=True + move = move or not isinstance(buffer, CBuffer) + builder.set_buffer(i, c_buffer(buffer), move=move) + + # Set children, optionally moving ownership of the children as well (i.e., + # the objects in the input children would be marked released). + n_children = 0 + for child_src in children: + # If we're setting a CArray from something else, we can avoid an extra + # level of Python wrapping by using move=True + move = move or not isinstance(child_src, CArray) + builder.set_child(n_children, c_array(child_src), move=move) + n_children += 1 + + if n_children != schema.n_children: + raise ValueError(f"Expected {schema.n_children} children but got {n_children}") + + # Set array fields + builder.set_length(length) + builder.set_offset(offset) + builder.set_null_count(null_count) + + # Calculates the null count if -1 (and if applicable) + builder.resolve_null_count() + + # Validate + finish + return builder.finish(validation_level=validation_level) + + +class ArrayBuilder: + """Internal utility to build CArrays from various types of input + + This class and its subclasses are designed to help separate the code + that actually builds a CArray from the code that chooses the strategy + used to do the building. + """ + + @classmethod + def infer_schema(cls, obj) -> Tuple[CSchema, Any]: + """Infer the Arrow data type from a target object + + Returns the type as a :class:`CSchema` and an object that can be + consumed in the same way by append() in the event it had to be + modified to infer its type (e.g., for an iterable, it would be + necessary to consume the first element from the original iterator). + """ + raise NotImplementedError() + + def __init__(self, schema): + self._schema = c_schema(schema) + self._schema_view = c_schema_view(self._schema) + + def build_c_array(self, obj): + builder = CArrayBuilder.allocate() + builder.init_from_schema(self._schema) + self.start_building(builder) + self.append(builder, obj) + return self.finish_building(builder) Review Comment: Possibly! I thought about it; however, there are perfectly valid ways to implement `build_c_array()` without a `CArrayBuilder` (although I avoided all of them for this PR by keeping them in `c_array()`). Eventually this will have to scale to deal with nested types and iterables that require more than one array of output (e.g., string arrays larger than 2 GB), and I think one or both of those will force some refactoring here (which may include fully committing to `ArrayBuilder` as a Python-level wrapper around the `CArrayBuilder`). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
