[
https://issues.apache.org/jira/browse/ARROW-1891?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16281973#comment-16281973
]
ASF GitHub Bot commented on ARROW-1891:
---------------------------------------
wesm closed pull request #1399: ARROW-1891: [Python] Always use NumPy NaT
sentinels to mark nulls when converting to array
URL: https://github.com/apache/arrow/pull/1399
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc
b/cpp/src/arrow/python/numpy_to_arrow.cc
index 798822c1b..f21b40ed3 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -421,7 +421,11 @@ class NumPyConverter {
using traits = internal::arrow_traits<ArrowType::type_id>;
const bool null_sentinels_possible =
- (use_pandas_null_sentinels_ && traits::supports_nulls);
+ // NumPy has a NaT type
+ (ArrowType::type_id == Type::TIMESTAMP || ArrowType::type_id ==
Type::DATE32) ||
+
+ // Observing pandas's null sentinels
+ ((use_pandas_null_sentinels_ && traits::supports_nulls));
if (mask_ != nullptr || null_sentinels_possible) {
RETURN_NOT_OK(InitNullBitmap());
@@ -631,8 +635,6 @@ inline Status
NumPyConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* d
auto date_dtype =
reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(dtype_->c_metadata);
if (dtype_->type_num == NPY_DATETIME) {
- const int64_t null_count = ValuesToBitmap<NPY_DATETIME>(arr_,
null_bitmap_data_);
-
// If we have inbound datetime64[D] data, this needs to be downcasted
// separately here from int64_t to int32_t, because this data is not
// supported in compute::Cast
@@ -642,6 +644,9 @@ inline Status
NumPyConverter::ConvertData<Date32Type>(std::shared_ptr<Buffer>* d
Status s = StaticCastBuffer<int64_t, int32_t>(**data, length_, pool_,
data);
RETURN_NOT_OK(s);
} else {
+ // TODO(wesm): This is redundant, and recomputed in VisitNative()
+ const int64_t null_count = ValuesToBitmap<NPY_DATETIME>(arr_,
null_bitmap_data_);
+
RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast<PyObject*>(dtype_),
&input_type));
if (!input_type->Equals(*type_)) {
RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_,
null_count,
diff --git a/python/pyarrow/tests/test_array.py
b/python/pyarrow/tests/test_array.py
index a4d781a33..92562da14 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -495,6 +495,14 @@ def test_array_conversions_no_sentinel_values():
assert arr3.null_count == 0
+def test_array_from_numpy_datetimeD():
+ arr = np.array([None, datetime.date(2017, 4, 4)], dtype='datetime64[D]')
+
+ result = pa.array(arr)
+ expected = pa.array([None, datetime.date(2017, 4, 4)], type=pa.date32())
+ assert result.equals(expected)
+
+
def test_array_from_numpy_ascii():
arr = np.array(['abcde', 'abc', ''], dtype='|S5')
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> [Python] NaT date32 values are only converted to nulls if from_pandas is used
> -----------------------------------------------------------------------------
>
> Key: ARROW-1891
> URL: https://issues.apache.org/jira/browse/ARROW-1891
> Project: Apache Arrow
> Issue Type: Bug
> Components: Python
> Reporter: Uwe L. Korn
> Assignee: Wes McKinney
> Labels: pull-request-available
> Fix For: 0.8.0
>
>
> {code}
> np.array([None, date(2017, 4, 4)], dtype='datetime64[D]')
> pa.array(expected, from_pandas=True) -> [null, 2017-4-4]
> pa.array(expected, from_pandas=True) -> [1970-1-1, 2017-4-4]
> {code}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
