westonpace commented on a change in pull request #11302:
URL: https://github.com/apache/arrow/pull/11302#discussion_r721820721
##########
File path: python/pyarrow/tests/test_array.py
##########
@@ -2162,6 +2163,101 @@ def test_array_from_numpy_ascii():
assert arrow_arr.equals(expected)
+@pytest.mark.nopandas
+def test_interval_array_from_timedelta():
+ data = [
+ None,
+ datetime.timedelta(days=1, seconds=1, microseconds=1,
+ milliseconds=1, minutes=1, hours=1, weeks=1)]
+
+ # From timedelta (explicit type required)
+ arr = pa.array(data, pa.month_day_nano_interval())
+ assert isinstance(arr, pa.MonthDayNanoIntervalArray)
+ assert arr.type == pa.month_day_nano_interval()
+ expected_list = [
+ None,
+ pa.MonthDayNanoTuple([0, 8,
+ (datetime.timedelta(seconds=1, microseconds=1,
+ milliseconds=1, minutes=1,
+ hours=1) //
+ datetime.timedelta(microseconds=1)) * 1000])]
+ expected = pa.array(expected_list)
+ assert arr.equals(expected)
+ assert arr.to_pylist() == expected_list
+
+# dateutil is dependency of pandas
+
+
+@pytest.mark.pandas
+def test_interval_array_from_relativedelta():
+ from dateutil.relativedelta import relativedelta
+ from pandas.tseries.offsets import DateOffset
+ data = [
+ None,
+ relativedelta(years=1, months=1,
+ days=1, seconds=1, microseconds=1,
+ minutes=1, hours=1, weeks=1, leapdays=1)]
+ # Note leapdays are ignored.
Review comment:
Nit: Can we mention this in at least one place in a docstring.
##########
File path: cpp/src/arrow/python/arrow_to_python.h
##########
@@ -0,0 +1,161 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Functions for converting between pandas's NumPy-based data representation
+// and Arrow data structures
+
+#pragma once
+
+#include "arrow/chunked_array.h"
+#include "arrow/python/common.h"
+#include "arrow/python/platform.h"
+#include "arrow/util/hashing.h"
+
+namespace arrow {
+
+class Array;
+class Scalar;
+
+namespace py {
+
+struct ArrowToPythonObjectOptions {
+ MemoryPool* pool = default_memory_pool();
+ bool deduplicate_objects = false;
+};
+
+class ARROW_PYTHON_EXPORT ArrowToPython {
+ public:
+ // \brief Converts the given Array to a PyList object. Returns NULL if there
+ // is an error converting the Array. The list elements are the same ones
+ // generated via ToLogical()
+ //
+ // N.B. This has limited type support. ARROW-12976 tracks extending the
implementation.
+ Result<PyObject*> ToPyList(const Array& array);
+
+ // Populates out_objects with the result of converting the array values
+ // to python objects. The same logic as ToLogical().
+ //
+ // N.B. Not all types are supported. ARROW-12976 tracks extending the
implementation.
+ Status ToNumpyObjectArray(const ArrowToPythonObjectOptions& options,
+ const ChunkedArray& array, PyObject** out_objects);
+
+ // \brief Converts the given Scalar to a python object that best corresponds
+ // with the Scalar's type.
+ //
+ // For example timestamp[ms] is translated into datetime.datetime.
+ //
+ // N.B. This has limited type support. ARROW-12976 tracks extending the
implementation.
+ Result<PyObject*> ToLogical(const Scalar& scalar);
+
+ // \brief Converts the given Scalar the type that is closed to its arrow
+ // representation.
+ //
+ // For instance timestamp would be translated to a integer representing an
+ // offset from the unix epoch.
+ //
+ // Returns nullptr on error.
+ //
+ // GIL must be health when calling this method.
+ // N.B. This has limited type support. ARROW-12976 tracks full
implementation.
+ Result<PyObject*> ToPrimitive(const Scalar& scalar);
+
+ private:
+ Status Init();
+};
+
+namespace internal {
+// TODO(ARROW-12976): See if we can refactor Pandas ObjectWriter logic
+// to the .cc file and move this there as well if we can.
+
+// Generic Array -> PyObject** converter that handles object deduplication, if
+// requested
+template <typename ArrayType, typename WriteValue, typename Assigner>
+inline Status WriteArrayObjects(const ArrayType& arr, WriteValue&& write_func,
+ Assigner out_values) {
+ // TODO(ARROW-12976): Use visitor here?
+ const bool has_nulls = arr.null_count() > 0;
+ for (int64_t i = 0; i < arr.length(); ++i) {
+ if (has_nulls && arr.IsNull(i)) {
+ Py_INCREF(Py_None);
+ *out_values = Py_None;
+ } else {
+ RETURN_NOT_OK(write_func(arr.GetView(i), out_values));
+ }
+ ++out_values;
+ }
Review comment:
Is this `has_nulls` enabling some kind of compiler optimization? From a
naive read it doesn't look like it is providing any value.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -514,6 +514,32 @@ cdef class DurationScalar(Scalar):
return datetime.timedelta(microseconds=sp.value // 1000)
+cdef class MonthDayNanoIntervalScalar(Scalar):
+ """
+ Concrete class for month, day, nanosecond scalars.
+ """
+
+ @property
+ def value(self):
+ """
+ Returns this value pyarrow.MonthDayNanoTuple.
Review comment:
```suggestion
Returns this value as a pyarrow.MonthDayNanoTuple.
```
##########
File path: python/pyarrow/types.pxi
##########
@@ -2154,6 +2154,14 @@ def duration(unit):
return out
+def month_day_nano_interval():
+ """
+ Create instance of a interval representing the time between two calendar
+ instances represented as a triple of months, days and nanoseconds.
+ """
+ return primitive_type(_Type_INTERVAL_MONTH_DAY_NANO)
Review comment:
Nit: I would avoid "the time between two calendar instances" because a
duration/Joda-interval is traditionally defined as "the time between two
instants". I haven't seen intervals / periods defined in this way before
(possibly because calendars could change).
Also, instead of `a triple of months` (which might make a python user think
3-tuple) could we say "represented by values for months, days, and
nanoseconds"? Or, if we want to be real precise, "represented by a signed 32
bit integer of months, a signed 32 bit integer of days, and a signed 64 bit
integer of nanoseconds".
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -514,6 +514,32 @@ cdef class DurationScalar(Scalar):
return datetime.timedelta(microseconds=sp.value // 1000)
+cdef class MonthDayNanoIntervalScalar(Scalar):
+ """
+ Concrete class for month, day, nanosecond scalars.
+ """
+
+ @property
+ def value(self):
+ """
+ Returns this value pyarrow.MonthDayNanoTuple.
+ """
+ cdef PyObject* val
+ val = GetResultValue(ARROW_TO_PYTHON.ToPrimitive(
+ (deref(self.wrapped.get()))))
+ return PyObject_to_object(val)
+
+ def as_py(self):
+ """
+ Return this value as a Pandas DateOffset instance if Pandas is present
+ otherwise as a named tuple containing months days and nanoseconds.
Review comment:
I'm going to agree with @pitrou . With pyarrow we even have cases where
users aren't even really aware they are using pyarrow. I could see a
circumstance where they install their application on some new machine or
environment and suddenly start getting errors. Maybe someday we can add a
`use_pandas_types` option to `as_py`, `to_pylist`, and `to_pydict`. That way
we can error if the expected library is not available instead of silently
changing the type.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
