[jira] [Commented] (ARROW-2151) [Python] Error when converting from list of uint64 arrays
[
https://issues.apache.org/jira/browse/ARROW-2151?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16366015#comment-16366015
]
ASF GitHub Bot commented on ARROW-2151:
---
xhochy closed pull request #1602: ARROW-2151: [Python] Fix conversion from
np.uint64 scalars
URL: https://github.com/apache/arrow/pull/1602
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/cpp/src/arrow/python/builtin_convert.cc
b/cpp/src/arrow/python/builtin_convert.cc
index 60f58ea5b..a286c6bd5 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -538,8 +538,8 @@ class UInt64Converter : public
TypedConverterVisitor(PyLong_AsUnsignedLongLong(obj));
-RETURN_IF_PYERROR();
+uint64_t val;
+RETURN_NOT_OK(internal::UInt64FromPythonInt(obj, &val));
return typed_builder_->Append(val);
}
};
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index 494f92900..df1db9991 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -176,6 +176,23 @@ bool IsPyInteger(PyObject* obj) {
#endif
}
+Status UInt64FromPythonInt(PyObject* obj, uint64_t* out) {
+ OwnedRef ref;
+ // PyLong_AsUnsignedLongLong() doesn't handle conversion from non-ints
+ // (e.g. np.uint64), so do it ourselves
+ if (!PyLong_Check(obj)) {
+ref.reset(PyNumber_Long(obj));
+RETURN_IF_PYERROR();
+obj = ref.obj();
+ }
+ auto result = static_cast(PyLong_AsUnsignedLongLong(obj));
+ if (result == static_cast(-1)) {
+RETURN_IF_PYERROR();
+ }
+ *out = static_cast(result);
+ return Status::OK();
+}
+
} // namespace internal
} // namespace py
} // namespace arrow
diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
index c82bdabc4..c0171aa2f 100644
--- a/cpp/src/arrow/python/helpers.h
+++ b/cpp/src/arrow/python/helpers.h
@@ -57,6 +57,8 @@ Status DecimalFromPythonDecimal(PyObject* python_decimal,
const DecimalType& arr
Decimal128* out);
bool IsPyInteger(PyObject* obj);
+Status UInt64FromPythonInt(PyObject* obj, uint64_t* out);
+
} // namespace internal
} // namespace py
} // namespace arrow
diff --git a/python/pyarrow/tests/test_convert_builtin.py
b/python/pyarrow/tests/test_convert_builtin.py
index 5cd4a52a2..8423ff00b 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -28,6 +28,20 @@
import six
+int_type_pairs = [
+(np.int8, pa.int8()),
+(np.int16, pa.int64()),
+(np.int32, pa.int32()),
+(np.int64, pa.int64()),
+(np.uint8, pa.uint8()),
+(np.uint16, pa.uint64()),
+(np.uint32, pa.uint32()),
+(np.uint64, pa.uint64())]
+
+
+np_int_types, _ = zip(*int_type_pairs)
+
+
class StrangeIterable:
def __init__(self, lst):
self.lst = lst
@@ -146,7 +160,20 @@ def test_sequence_all_none():
@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
-def test_sequence_integer(seq):
+@pytest.mark.parametrize("np_scalar_pa_type", int_type_pairs)
+def test_sequence_integer(seq, np_scalar_pa_type):
+np_scalar, pa_type = np_scalar_pa_type
+expected = [1, None, 3, None,
+np.iinfo(np_scalar).min, np.iinfo(np_scalar).max]
+arr = pa.array(seq(expected), type=pa_type)
+assert len(arr) == 6
+assert arr.null_count == 2
+assert arr.type == pa_type
+assert arr.to_pylist() == expected
+
+
+@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
+def test_sequence_integer_inferred(seq):
expected = [1, None, 3, None]
arr = pa.array(seq(expected))
assert len(arr) == 4
@@ -156,12 +183,32 @@ def test_sequence_integer(seq):
@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
-@pytest.mark.parametrize("np_scalar", [np.int16, np.int32, np.int64, np.uint16,
- np.uint32, np.uint64])
-def test_sequence_numpy_integer(seq, np_scalar):
+@pytest.mark.parametrize("np_scalar_pa_type", int_type_pairs)
+def test_sequence_numpy_integer(seq, np_scalar_pa_type):
+np_scalar, pa_type = np_scalar_pa_type
+expected = [np_scalar(1), None, np_scalar(3), None,
+np_scalar(np.iinfo(np_scalar).min),
+np_scalar(np.iinfo(np_scalar).max)]
+arr = pa.array(seq(expected), type=pa_type)
+assert len(arr) == 6
+assert arr.null_count == 2
+assert arr.type == pa_type
+assert arr.to_pylist() == expected
+
+
+@pytest.mark.parametrize("seq", [_as_list, _as_tuple, _as_dict_values])
+@pytest.mark.parametrize("np_scalar_pa_type", int_type_pairs)
+def test_sequence_numpy_integer_inferred(seq, np_scalar_pa_typ
[jira] [Commented] (ARROW-2151) [Python] Error when converting from list of uint64 arrays
[
https://issues.apache.org/jira/browse/ARROW-2151?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16362577#comment-16362577
]
ASF GitHub Bot commented on ARROW-2151:
---
wesm commented on issue #1602: ARROW-2151: [Python] Fix conversion from
np.uint64 scalars
URL: https://github.com/apache/arrow/pull/1602#issuecomment-365318379
rebased
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> [Python] Error when converting from list of uint64 arrays
> -
>
> Key: ARROW-2151
> URL: https://issues.apache.org/jira/browse/ARROW-2151
> Project: Apache Arrow
> Issue Type: Bug
> Components: Python
>Affects Versions: 0.8.0
>Reporter: Antoine Pitrou
>Assignee: Antoine Pitrou
>Priority: Minor
> Labels: pull-request-available
>
> {code:python}
> >>> pa.array(np.uint64([0,1,2]), type=pa.uint64())
>
> [
> 0,
> 1,
> 2
> ]
> >>> pa.array([np.uint64([0,1,2])], type=pa.list_(pa.uint64()))
> Traceback (most recent call last):
> File "", line 1, in
> pa.array([np.uint64([0,1,2])], type=pa.list_(pa.uint64()))
> File "array.pxi", line 181, in pyarrow.lib.array
> File "array.pxi", line 36, in pyarrow.lib._sequence_to_array
> File "error.pxi", line 98, in pyarrow.lib.check_status
> ArrowException: Unknown error:
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:979 code:
> AppendPySequence(seq, size, real_type, builder.get())
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:402 code:
> static_cast(this)->AppendSingle(ref.obj())
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:402 code:
> static_cast(this)->AppendSingle(ref.obj())
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:542 code:
> CheckPyError()
> an integer is required
> {code}
> More simply, it also fails with a sequence of Numpy uint64 scalars:
> {code:python}
> >>> pa.array([np.uint64(1)], type=pa.uint64())
> Traceback (most recent call last):
> File "", line 1, in
> pa.array([np.uint64(1)], type=pa.uint64())
> File "array.pxi", line 181, in pyarrow.lib.array
> File "array.pxi", line 36, in pyarrow.lib._sequence_to_array
> File "error.pxi", line 98, in pyarrow.lib.check_status
> ArrowException: Unknown error:
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:979 code:
> AppendPySequence(seq, size, real_type, builder.get())
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:402 code:
> static_cast(this)->AppendSingle(ref.obj())
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:542 code:
> CheckPyError()
> an integer is required
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (ARROW-2151) [Python] Error when converting from list of uint64 arrays
[
https://issues.apache.org/jira/browse/ARROW-2151?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16362416#comment-16362416
]
ASF GitHub Bot commented on ARROW-2151:
---
pitrou opened a new pull request #1602: ARROW-2151: [Python] Fix conversion
from np.uint64 scalars
URL: https://github.com/apache/arrow/pull/1602
Also add tests for ARROW-1345.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> [Python] Error when converting from list of uint64 arrays
> -
>
> Key: ARROW-2151
> URL: https://issues.apache.org/jira/browse/ARROW-2151
> Project: Apache Arrow
> Issue Type: Bug
> Components: Python
>Affects Versions: 0.8.0
>Reporter: Antoine Pitrou
>Assignee: Antoine Pitrou
>Priority: Minor
> Labels: pull-request-available
>
> {code:python}
> >>> pa.array(np.uint64([0,1,2]), type=pa.uint64())
>
> [
> 0,
> 1,
> 2
> ]
> >>> pa.array([np.uint64([0,1,2])], type=pa.list_(pa.uint64()))
> Traceback (most recent call last):
> File "", line 1, in
> pa.array([np.uint64([0,1,2])], type=pa.list_(pa.uint64()))
> File "array.pxi", line 181, in pyarrow.lib.array
> File "array.pxi", line 36, in pyarrow.lib._sequence_to_array
> File "error.pxi", line 98, in pyarrow.lib.check_status
> ArrowException: Unknown error:
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:979 code:
> AppendPySequence(seq, size, real_type, builder.get())
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:402 code:
> static_cast(this)->AppendSingle(ref.obj())
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:402 code:
> static_cast(this)->AppendSingle(ref.obj())
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:542 code:
> CheckPyError()
> an integer is required
> {code}
> More simply, it also fails with a sequence of Numpy uint64 scalars:
> {code:python}
> >>> pa.array([np.uint64(1)], type=pa.uint64())
> Traceback (most recent call last):
> File "", line 1, in
> pa.array([np.uint64(1)], type=pa.uint64())
> File "array.pxi", line 181, in pyarrow.lib.array
> File "array.pxi", line 36, in pyarrow.lib._sequence_to_array
> File "error.pxi", line 98, in pyarrow.lib.check_status
> ArrowException: Unknown error:
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:979 code:
> AppendPySequence(seq, size, real_type, builder.get())
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:402 code:
> static_cast(this)->AppendSingle(ref.obj())
> /home/antoine/arrow/cpp/src/arrow/python/builtin_convert.cc:542 code:
> CheckPyError()
> an integer is required
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
