This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new b926574 ARROW-2722: [Python] Sanitize dtype number to handle edge
cases
b926574 is described below
commit b9265743c422b6df325948c1ae6f7f86a8b5d092
Author: Antoine Pitrou <[email protected]>
AuthorDate: Mon Jul 9 16:06:20 2018 -0400
ARROW-2722: [Python] Sanitize dtype number to handle edge cases
Author: Antoine Pitrou <[email protected]>
Closes #2234 from pitrou/ARROW-2722-sanitize-numpy-type-num and squashes
the following commits:
75faf9ae <Antoine Pitrou> ARROW-2722: Sanitize dtype number to handle edge
cases
---
cpp/src/arrow/python/numpy_convert.cc | 32 ++---------------------------
cpp/src/arrow/python/numpy_interop.h | 13 ++++++++++++
cpp/src/arrow/python/type_traits.h | 12 ++---------
python/pyarrow/tests/test_convert_pandas.py | 6 ++++++
4 files changed, 23 insertions(+), 40 deletions(-)
diff --git a/cpp/src/arrow/python/numpy_convert.cc
b/cpp/src/arrow/python/numpy_convert.cc
index e91aae1..97220a5 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -45,22 +45,6 @@ bool is_contiguous(PyObject* array) {
}
}
-int cast_npy_type_compat(int type_num) {
- // Both LONGLONG and INT64 can be observed in the wild, which is buggy. We
set
- // U/LONGLONG to U/INT64 so things work properly.
-
-#if (NPY_INT64 == NPY_LONGLONG) && (NPY_SIZEOF_LONGLONG == 8)
- if (type_num == NPY_LONGLONG) {
- type_num = NPY_INT64;
- }
- if (type_num == NPY_ULONGLONG) {
- type_num = NPY_UINT64;
- }
-#endif
-
- return type_num;
-}
-
NumPyBuffer::NumPyBuffer(PyObject* ao) : Buffer(nullptr, 0) {
arr_ = ao;
Py_INCREF(ao);
@@ -92,7 +76,7 @@ Status GetTensorType(PyObject* dtype,
std::shared_ptr<DataType>* out) {
return Status::TypeError("Did not pass numpy.dtype object");
}
PyArray_Descr* descr = reinterpret_cast<PyArray_Descr*>(dtype);
- int type_num = cast_npy_type_compat(descr->type_num);
+ int type_num = fix_numpy_type_num(descr->type_num);
switch (type_num) {
TO_ARROW_TYPE_CASE(BOOL, uint8);
@@ -100,16 +84,10 @@ Status GetTensorType(PyObject* dtype,
std::shared_ptr<DataType>* out) {
TO_ARROW_TYPE_CASE(INT16, int16);
TO_ARROW_TYPE_CASE(INT32, int32);
TO_ARROW_TYPE_CASE(INT64, int64);
-#if (NPY_INT64 != NPY_LONGLONG)
- TO_ARROW_TYPE_CASE(LONGLONG, int64);
-#endif
TO_ARROW_TYPE_CASE(UINT8, uint8);
TO_ARROW_TYPE_CASE(UINT16, uint16);
TO_ARROW_TYPE_CASE(UINT32, uint32);
TO_ARROW_TYPE_CASE(UINT64, uint64);
-#if (NPY_UINT64 != NPY_ULONGLONG)
- TO_ARROW_CASE(ULONGLONG);
-#endif
TO_ARROW_TYPE_CASE(FLOAT16, float16);
TO_ARROW_TYPE_CASE(FLOAT32, float32);
TO_ARROW_TYPE_CASE(FLOAT64, float64);
@@ -160,7 +138,7 @@ Status NumPyDtypeToArrow(PyObject* dtype,
std::shared_ptr<DataType>* out) {
}
Status NumPyDtypeToArrow(PyArray_Descr* descr, std::shared_ptr<DataType>* out)
{
- int type_num = cast_npy_type_compat(descr->type_num);
+ int type_num = fix_numpy_type_num(descr->type_num);
switch (type_num) {
TO_ARROW_TYPE_CASE(BOOL, boolean);
@@ -168,16 +146,10 @@ Status NumPyDtypeToArrow(PyArray_Descr* descr,
std::shared_ptr<DataType>* out) {
TO_ARROW_TYPE_CASE(INT16, int16);
TO_ARROW_TYPE_CASE(INT32, int32);
TO_ARROW_TYPE_CASE(INT64, int64);
-#if (NPY_INT64 != NPY_LONGLONG)
- TO_ARROW_TYPE_CASE(LONGLONG, int64);
-#endif
TO_ARROW_TYPE_CASE(UINT8, uint8);
TO_ARROW_TYPE_CASE(UINT16, uint16);
TO_ARROW_TYPE_CASE(UINT32, uint32);
TO_ARROW_TYPE_CASE(UINT64, uint64);
-#if (NPY_UINT64 != NPY_ULONGLONG)
- TO_ARROW_CASE(ULONGLONG);
-#endif
TO_ARROW_TYPE_CASE(FLOAT16, float16);
TO_ARROW_TYPE_CASE(FLOAT32, float32);
TO_ARROW_TYPE_CASE(FLOAT64, float64);
diff --git a/cpp/src/arrow/python/numpy_interop.h
b/cpp/src/arrow/python/numpy_interop.h
index 0715c66..b62c4e6 100644
--- a/cpp/src/arrow/python/numpy_interop.h
+++ b/cpp/src/arrow/python/numpy_interop.h
@@ -80,6 +80,19 @@ inline int import_numpy() {
return 0;
}
+// See above about the missing Numpy integer type numbers
+inline int fix_numpy_type_num(int type_num) {
+#if !NPY_INT32_IS_INT && NPY_BITSOF_INT == 32
+ if (type_num == NPY_INT) return NPY_INT32;
+ if (type_num == NPY_UINT) return NPY_UINT32;
+#endif
+#if !NPY_INT64_IS_LONG_LONG && NPY_BITSOF_LONGLONG == 64
+ if (type_num == NPY_LONGLONG) return NPY_INT64;
+ if (type_num == NPY_ULONGLONG) return NPY_UINT64;
+#endif
+ return type_num;
+}
+
} // namespace py
} // namespace arrow
diff --git a/cpp/src/arrow/python/type_traits.h
b/cpp/src/arrow/python/type_traits.h
index ff39aad..d90517a 100644
--- a/cpp/src/arrow/python/type_traits.h
+++ b/cpp/src/arrow/python/type_traits.h
@@ -263,6 +263,8 @@ struct arrow_traits<Type::BINARY> {
};
static inline int NumPyTypeSize(int npy_type) {
+ npy_type = fix_numpy_type_num(npy_type);
+
switch (npy_type) {
case NPY_BOOL:
case NPY_INT8:
@@ -277,16 +279,6 @@ static inline int NumPyTypeSize(int npy_type) {
case NPY_INT64:
case NPY_UINT64:
return 8;
-#if !NPY_INT32_IS_INT
- case NPY_INT:
- case NPY_UINT:
- return NPY_BITSOF_INT / 8;
-#endif
-#if !NPY_INT64_IS_LONG_LONG
- case NPY_LONGLONG:
- case NPY_ULONGLONG:
- return NPY_BITSOF_LONGLONG / 8;
-#endif
case NPY_FLOAT16:
return 2;
case NPY_FLOAT32:
diff --git a/python/pyarrow/tests/test_convert_pandas.py
b/python/pyarrow/tests/test_convert_pandas.py
index 19ffb25..ceedf1a 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -569,6 +569,12 @@ class TestConvertPrimitiveTypes(object):
df = pd.DataFrame(data)
_check_pandas_roundtrip(df)
+ # Do the same with pa.array()
+ # (for some reason, it doesn't use the same code paths at all)
+ for np_arr in data.values():
+ arr = pa.array(np_arr)
+ assert arr.to_pylist() == np_arr.tolist()
+
def test_integer_with_nulls(self):
# pandas requires upcast to float dtype
