danepitkin commented on code in PR #388: URL: https://github.com/apache/arrow-nanoarrow/pull/388#discussion_r1496589109
########## python/src/nanoarrow/ipc.py: ########## @@ -0,0 +1,184 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import io + +from nanoarrow._ipc_lib import CIpcInputStream, init_array_stream +from nanoarrow._lib import CArrayStream + + +class Stream: + """Stream of serialized Arrow data + + Reads file paths or otherwise readable file objects that contain + serialized Arrow data. Arrow documentation typically refers to this format + as "Arrow IPC" because its origin was as a means to transmit tables between + processes; however, this format can also be written to and read from files + or URLs and is essentially a high-performance equivalent of a CSV file that + does a better job maintaining type fidelity. + + Use :staticmethod:`from_readable`, :staticmethod:`from_path`, or + :staticmethod:`from_url` to construct these streams. + """ + + def __init__(self): + self._stream = None + self._desc = None + + def _is_valid(self) -> bool: + return self._stream is not None and self._stream.is_valid() + + def __arrow_c_stream__(self, requested_schema=None): + """Export this stream as an ArrowArrayStream + + Implements the Arrow PyCapsule interface by transferring ownership of this + input stream to an ArrowArrayStream wrapped by a PyCapsule. + """ + if not self._is_valid(): + raise RuntimeError("nanoarrow.ipc.Stream is no longer valid") + + array_stream = CArrayStream.allocate() + init_array_stream(self._stream, array_stream._addr()) + return array_stream.__arrow_c_stream__(requested_schema=requested_schema) + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + if self._stream is not None: + self._stream.release() + + @staticmethod + def from_readable(obj): + """Wrap an open readable object as an Arrow stream + + Wraps a readable object (specificially, an object that implements a + ``readinto()`` method) as a non-owning Stream. Closing ``obj`` remains + the caller's responsibility: neither this stream nor the resulting array + stream will call ``obj.close()``. + + Parameters + ---------- + obj : readable file-like + An object implementing ``readinto()``. + """ + out = Stream() + out._stream = CIpcInputStream.from_readable(obj) + out._desc = repr(obj) + return out + + @staticmethod + def from_path(obj, *args, **kwargs): + """Wrap a local file as an Arrow stream + + Wraps a pathlike object (specificially, one that can be passed to ``open()``) + as an owning Stream. The file will be opened in binary mode and will be closed + when this stream or the resulting array stream is released. + + Parameters + ---------- + obj : path-like + A string or path-like object that can be passed to ``open()`` + """ + out = Stream() + out._stream = CIpcInputStream.from_readable( + open(obj, "rb", *args, **kwargs), close_stream=True + ) + out._desc = repr(obj) + return out + + @staticmethod + def from_url(obj, *args, **kwargs): + """Wrap a URL as an Arrow stream + + Wraps a URL (specificially, one that can be passed to + ``urllib.request.urlopen()``) as an owning Stream. The URL will be + closed when this stream or the resulting array stream is released. + + Parameters + ---------- + obj : str + A URL that can be passed to ``urllib.request.urlopen()`` + """ + import urllib.request + + out = Stream() + out._stream = CIpcInputStream.from_readable( + urllib.request.urlopen(obj, *args, **kwargs), close_stream=True + ) + out._desc = repr(obj) + return out + + @staticmethod + def example(): Review Comment: I personally like to keep all test-related code and data in the `/tests` dir. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
