[
https://issues.apache.org/jira/browse/ARROW-1892?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16280900#comment-16280900
]
ASF GitHub Bot commented on ARROW-1892:
---------------------------------------
wesm closed pull request #1394: ARROW-1892: [Python] Support binaries in lists
URL: https://github.com/apache/arrow/pull/1394
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc
b/cpp/src/arrow/python/arrow_to_pandas.cc
index b1825cbe6..1a1b71125 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -100,6 +100,7 @@ static inline bool ListTypeSupported(const DataType& type) {
case Type::UINT64:
case Type::FLOAT:
case Type::DOUBLE:
+ case Type::BINARY:
case Type::STRING:
case Type::TIMESTAMP:
// The above types are all supported.
@@ -695,6 +696,7 @@ class ObjectBlock : public PandasBlock {
CONVERTLISTSLIKE_CASE(TimestampType, TIMESTAMP)
CONVERTLISTSLIKE_CASE(FloatType, FLOAT)
CONVERTLISTSLIKE_CASE(DoubleType, DOUBLE)
+ CONVERTLISTSLIKE_CASE(BinaryType, BINARY)
CONVERTLISTSLIKE_CASE(StringType, STRING)
CONVERTLISTSLIKE_CASE(ListType, LIST)
default: {
@@ -1642,6 +1644,7 @@ class ArrowDeserializer {
CONVERTVALUES_LISTSLIKE_CASE(TimestampType, TIMESTAMP)
CONVERTVALUES_LISTSLIKE_CASE(FloatType, FLOAT)
CONVERTVALUES_LISTSLIKE_CASE(DoubleType, DOUBLE)
+ CONVERTVALUES_LISTSLIKE_CASE(BinaryType, BINARY)
CONVERTVALUES_LISTSLIKE_CASE(StringType, STRING)
CONVERTVALUES_LISTSLIKE_CASE(Decimal128Type, DECIMAL)
CONVERTVALUES_LISTSLIKE_CASE(ListType, LIST)
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc
b/cpp/src/arrow/python/numpy_to_arrow.cc
index 2316a796a..2f354dcfe 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -145,6 +145,52 @@ Status CheckFlatNumpyArray(PyArrayObject* numpy_array, int
np_type) {
} // namespace
+/// Append as many string objects from NumPy arrays to a `StringBuilder` as we
+/// can fit
+///
+/// \param[in] offset starting offset for appending
+/// \param[out] end_offset ending offset where we stopped appending. Will
+/// be length of arr if fully consumed
+/// \param[out] have_bytes true if we encountered any PyBytes object
+static Status AppendObjectBinaries(PyArrayObject* arr, PyArrayObject* mask,
+ int64_t offset, BinaryBuilder* builder,
+ int64_t* end_offset, bool* have_bytes) {
+ PyObject* obj;
+
+ Ndarray1DIndexer<PyObject*> objects(arr);
+ Ndarray1DIndexer<uint8_t> mask_values;
+
+ bool have_mask = false;
+ if (mask != nullptr) {
+ mask_values.Init(mask);
+ have_mask = true;
+ }
+
+ for (; offset < objects.size(); ++offset) {
+ OwnedRef tmp_obj;
+ obj = objects[offset];
+ if ((have_mask && mask_values[offset]) || PandasObjectIsNull(obj)) {
+ RETURN_NOT_OK(builder->AppendNull());
+ continue;
+ } else if (!PyBytes_Check(obj)) {
+ std::stringstream ss;
+ ss << "Error converting to Python objects to bytes: ";
+ RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
+ return Status::Invalid(ss.str());
+ }
+
+ const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
+ if (ARROW_PREDICT_FALSE(builder->value_data_length() + length >
kBinaryMemoryLimit)) {
+ break;
+ }
+ RETURN_NOT_OK(builder->Append(PyBytes_AS_STRING(obj), length));
+ }
+
+ // If we consumed the whole array, this will be the length of arr
+ *end_offset = offset;
+ return Status::OK();
+}
+
/// Append as many string objects from NumPy arrays to a `StringBuilder` as we
/// can fit
///
@@ -1192,6 +1238,59 @@ inline Status
NumPyConverter::ConvertTypedLists<NPY_OBJECT, NullType>(
return LoopPySequenceWithMasks(list, mask_values, have_mask, foreach_item);
}
+template <>
+inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, BinaryType>(
+ const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject*
list) {
+ PyAcquireGIL lock;
+ // TODO: If there are bytes involed, convert to Binary representation
+ bool have_bytes = false;
+
+ Ndarray1DIndexer<uint8_t> mask_values;
+
+ bool have_mask = false;
+ if (mask_ != nullptr) {
+ mask_values.Init(mask_);
+ have_mask = true;
+ }
+
+ auto value_builder = static_cast<BinaryBuilder*>(builder->value_builder());
+
+ auto foreach_item = [&](PyObject* object, bool mask) {
+ if (mask || PandasObjectIsNull(object)) {
+ return builder->AppendNull();
+ } else if (PyArray_Check(object)) {
+ auto numpy_array = reinterpret_cast<PyArrayObject*>(object);
+ RETURN_NOT_OK(builder->Append(true));
+
+ // TODO(uwe): Support more complex numpy array structures
+ RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
+
+ int64_t offset = 0;
+ RETURN_NOT_OK(AppendObjectBinaries(numpy_array, nullptr, 0,
value_builder, &offset,
+ &have_bytes));
+ if (offset < PyArray_SIZE(numpy_array)) {
+ return Status::Invalid("Array cell value exceeded 2GB");
+ }
+ return Status::OK();
+ } else if (PyList_Check(object)) {
+ int64_t size;
+ std::shared_ptr<DataType> inferred_type;
+ RETURN_NOT_OK(builder->Append(true));
+ RETURN_NOT_OK(InferArrowTypeAndSize(object, &size, &inferred_type));
+ if (inferred_type->id() != Type::NA && inferred_type->id() !=
Type::BINARY) {
+ std::stringstream ss;
+ ss << inferred_type->ToString() << " cannot be converted to BINARY.";
+ return Status::TypeError(ss.str());
+ }
+ return AppendPySequence(object, size, inferred_type, value_builder);
+ } else {
+ return Status::TypeError("Unsupported Python type for list items");
+ }
+ };
+
+ return LoopPySequenceWithMasks(list, mask_values, have_mask, foreach_item);
+}
+
template <>
inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, StringType>(
const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject*
list) {
@@ -1266,6 +1365,7 @@ Status NumPyConverter::ConvertLists(const
std::shared_ptr<DataType>& type,
LIST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
LIST_CASE(FLOAT, NPY_FLOAT, FloatType)
LIST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
+ LIST_CASE(BINARY, NPY_OBJECT, BinaryType)
LIST_CASE(STRING, NPY_OBJECT, StringType)
case Type::LIST: {
const auto& list_type = static_cast<const ListType&>(*type);
diff --git a/python/pyarrow/tests/pandas_examples.py
b/python/pyarrow/tests/pandas_examples.py
index c145e9634..f11da3c6c 100644
--- a/python/pyarrow/tests/pandas_examples.py
+++ b/python/pyarrow/tests/pandas_examples.py
@@ -110,6 +110,14 @@ def dataframe_with_lists(include_index=False):
[0.],
np.array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.] * 2)[::2],
]
+ fields.append(pa.field('bytes_list', pa.list_(pa.binary())))
+ arrays['bytes_list'] = [
+ [b"1", b"f"],
+ None,
+ [b"1"],
+ [b"1", b"2", b"3"],
+ [],
+ ]
fields.append(pa.field('str_list', pa.list_(pa.string())))
arrays['str_list'] = [
[u"1", u"ä"],
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> [Python] Unknown list item type: binary
> ---------------------------------------
>
> Key: ARROW-1892
> URL: https://issues.apache.org/jira/browse/ARROW-1892
> Project: Apache Arrow
> Issue Type: Bug
> Components: Python
> Affects Versions: 0.7.1
> Reporter: Uwe L. Korn
> Assignee: Uwe L. Korn
> Labels: pull-request-available
> Fix For: 0.8.0
>
>
> While strings are supported as list elements in the Arrow<->Pandas
> conversion, binary (bytes in Py3, str in Py2) are not supported.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
