[
https://issues.apache.org/jira/browse/ARROW-2141?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16424272#comment-16424272
]
ASF GitHub Bot commented on ARROW-2141:
---------------------------------------
BryanCutler closed pull request #1689: ARROW-2141: [Python] Support variable
length binary conversion from Pandas
URL: https://github.com/apache/arrow/pull/1689
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc
b/cpp/src/arrow/python/numpy_to_arrow.cc
index 71bf69fc1..eb0af8b08 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -195,16 +195,15 @@ int64_t MaskToBitmap(PyArrayObject* mask, int64_t length,
uint8_t* bitmap) {
} // namespace
-/// Append as many string objects from NumPy arrays to a `StringBuilder` as we
+/// Append as many string objects from NumPy arrays to a `BinaryBuilder` as we
/// can fit
///
/// \param[in] offset starting offset for appending
/// \param[out] end_offset ending offset where we stopped appending. Will
/// be length of arr if fully consumed
-/// \param[out] have_bytes true if we encountered any PyBytes object
static Status AppendObjectBinaries(PyArrayObject* arr, PyArrayObject* mask,
int64_t offset, BinaryBuilder* builder,
- int64_t* end_offset, bool* have_bytes) {
+ int64_t* end_offset) {
PyObject* obj;
Ndarray1DIndexer<PyObject*> objects(arr);
@@ -222,18 +221,26 @@ static Status AppendObjectBinaries(PyArrayObject* arr,
PyArrayObject* mask,
if ((have_mask && mask_values[offset]) ||
internal::PandasObjectIsNull(obj)) {
RETURN_NOT_OK(builder->AppendNull());
continue;
- } else if (!PyBytes_Check(obj)) {
+ } else if (PyBytes_Check(obj)) {
+ const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
+ if (ARROW_PREDICT_FALSE(builder->value_data_length() + length >
+ kBinaryMemoryLimit)) {
+ break;
+ }
+ RETURN_NOT_OK(builder->Append(PyBytes_AS_STRING(obj), length));
+ } else if (PyByteArray_Check(obj)) {
+ const int32_t length = static_cast<int32_t>(PyByteArray_GET_SIZE(obj));
+ if (ARROW_PREDICT_FALSE(builder->value_data_length() + length >
+ kBinaryMemoryLimit)) {
+ break;
+ }
+ RETURN_NOT_OK(builder->Append(PyByteArray_AS_STRING(obj), length));
+ } else {
std::stringstream ss;
ss << "Error converting from Python objects to bytes: ";
- RETURN_NOT_OK(InvalidConversion(obj, "str, bytes", &ss));
+ RETURN_NOT_OK(InvalidConversion(obj, "str, bytes, bytearray", &ss));
return Status::Invalid(ss.str());
}
-
- const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(obj));
- if (ARROW_PREDICT_FALSE(builder->value_data_length() + length >
kBinaryMemoryLimit)) {
- break;
- }
- RETURN_NOT_OK(builder->Append(PyBytes_AS_STRING(obj), length));
}
// If we consumed the whole array, this will be the length of arr
@@ -506,6 +513,7 @@ class NumPyConverter {
Status ConvertBooleans();
Status ConvertObjectStrings();
Status ConvertObjectFloats();
+ Status ConvertObjectBytes();
Status ConvertObjectFixedWidthBytes(const std::shared_ptr<DataType>& type);
Status ConvertObjectIntegers();
Status ConvertLists(const std::shared_ptr<DataType>& type);
@@ -988,6 +996,29 @@ Status NumPyConverter::ConvertObjectIntegers() {
return PushBuilderResult(&builder);
}
+Status NumPyConverter::ConvertObjectBytes() {
+ PyAcquireGIL lock;
+
+ BinaryBuilder builder(binary(), pool_);
+ RETURN_NOT_OK(builder.Resize(length_));
+
+ if (length_ == 0) {
+ // Produce an empty chunk
+ std::shared_ptr<Array> chunk;
+ RETURN_NOT_OK(builder.Finish(&chunk));
+ out_arrays_.emplace_back(std::move(chunk));
+ } else {
+ int64_t offset = 0;
+ while (offset < length_) {
+ RETURN_NOT_OK(AppendObjectBinaries(arr_, mask_, offset, &builder,
&offset));
+ std::shared_ptr<Array> chunk;
+ RETURN_NOT_OK(builder.Finish(&chunk));
+ out_arrays_.emplace_back(std::move(chunk));
+ }
+ }
+ return Status::OK();
+}
+
Status NumPyConverter::ConvertObjectFixedWidthBytes(
const std::shared_ptr<DataType>& type) {
PyAcquireGIL lock;
@@ -1088,6 +1119,8 @@ Status NumPyConverter::ConvertObjectsInfer() {
return ConvertTimes();
} else if (PyObject_IsInstance(obj, decimal_type_.obj()) == 1) {
return ConvertDecimals();
+ } else if (PyByteArray_Check(obj)) {
+ return ConvertObjectBytes();
} else if (PyList_Check(obj)) {
std::shared_ptr<DataType> inferred_type;
RETURN_NOT_OK(InferArrowType(obj, &inferred_type));
@@ -1105,7 +1138,7 @@ Status NumPyConverter::ConvertObjectsInfer() {
return ConvertLists(inferred_type);
} else {
const std::string supported_types =
- "string, bool, float, int, date, time, decimal, list, array";
+ "string, bool, float, int, date, time, decimal, bytearray, list,
array";
std::stringstream ss;
ss << "Error inferring Arrow type for Python object array. ";
RETURN_NOT_OK(InvalidConversion(obj, supported_types, &ss));
@@ -1154,6 +1187,8 @@ Status NumPyConverter::ConvertObjects() {
switch (type_->id()) {
case Type::STRING:
return ConvertObjectStrings();
+ case Type::BINARY:
+ return ConvertObjectBytes();
case Type::FIXED_SIZE_BINARY:
return ConvertObjectFixedWidthBytes(type_);
case Type::BOOL:
@@ -1339,8 +1374,6 @@ template <>
inline Status NumPyConverter::ConvertTypedLists<NPY_OBJECT, BinaryType>(
const std::shared_ptr<DataType>& type, ListBuilder* builder, PyObject*
list) {
PyAcquireGIL lock;
- // TODO: If there are bytes involed, convert to Binary representation
- bool have_bytes = false;
Ndarray1DIndexer<uint8_t> mask_values;
@@ -1363,8 +1396,8 @@ inline Status
NumPyConverter::ConvertTypedLists<NPY_OBJECT, BinaryType>(
RETURN_NOT_OK(CheckFlatNumpyArray(numpy_array, NPY_OBJECT));
int64_t offset = 0;
- RETURN_NOT_OK(AppendObjectBinaries(numpy_array, nullptr, 0,
value_builder, &offset,
- &have_bytes));
+ RETURN_NOT_OK(
+ AppendObjectBinaries(numpy_array, nullptr, 0, value_builder,
&offset));
if (offset < PyArray_SIZE(numpy_array)) {
return Status::Invalid("Array cell value exceeded 2GB");
}
diff --git a/python/pyarrow/tests/test_convert_pandas.py
b/python/pyarrow/tests/test_convert_pandas.py
index 45ec66dab..9ecc73c12 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1099,6 +1099,22 @@ def
test_fixed_size_bytes_does_not_accept_varying_lengths(self):
with pytest.raises(pa.ArrowInvalid):
pa.Table.from_pandas(df, schema=schema)
+ def test_variable_size_bytes(self):
+ s = pd.Series([b'123', b'', b'a', None])
+ arr = pa.Array.from_pandas(s, type=pa.binary())
+ assert arr.type == pa.binary()
+ _check_series_roundtrip(s, type_=pa.binary())
+
+ def test_binary_from_bytearray(self):
+ s = pd.Series([bytearray(b'123'), bytearray(b''), bytearray(b'a')])
+ # Explicitly set type
+ arr = pa.Array.from_pandas(s, type=pa.binary())
+ assert arr.type == pa.binary()
+ # Infer type from bytearrays
+ arr = pa.Array.from_pandas(s)
+ assert arr.type == pa.binary()
+ _check_series_roundtrip(s, type_=pa.binary())
+
def test_table_empty_str(self):
values = ['', '', '', '', '']
df = pd.DataFrame({'strings': values})
@@ -1693,7 +1709,7 @@ class TestConvertMisc(object):
# XXX unsupported
# (np.dtype([('a', 'i2')]), pa.struct([pa.field('a', pa.int16())])),
(np.object, pa.string()),
- # (np.object, pa.binary()), # XXX unsupported
+ (np.object, pa.binary()),
(np.object, pa.binary(10)),
(np.object, pa.list_(pa.int64())),
]
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> [Python] Conversion from Numpy object array to varsize binary unimplemented
> ---------------------------------------------------------------------------
>
> Key: ARROW-2141
> URL: https://issues.apache.org/jira/browse/ARROW-2141
> Project: Apache Arrow
> Issue Type: Improvement
> Components: Python
> Affects Versions: 0.8.0
> Reporter: Antoine Pitrou
> Assignee: Bryan Cutler
> Priority: Major
> Labels: pull-request-available
> Fix For: 0.10.0
>
>
> {code:python}
> >>> arr = np.array([b'xx'], dtype=np.object)
> >>> pa.array(arr, type=pa.binary(2))
> <pyarrow.lib.FixedSizeBinaryArray object at 0x7fe1ecaefa98>
> [
> b'xx'
> ]
> >>> pa.array(arr, type=pa.binary())
> Traceback (most recent call last):
> File "<ipython-input-12-e40948b94b33>", line 1, in <module>
> pa.array(arr, type=pa.binary())
> File "array.pxi", line 177, in pyarrow.lib.array
> File "error.pxi", line 77, in pyarrow.lib.check_status
> File "error.pxi", line 85, in pyarrow.lib.check_status
> ArrowNotImplementedError:
> /home/antoine/arrow/cpp/src/arrow/python/numpy_to_arrow.cc:1585 code:
> converter.Convert()
> /home/antoine/arrow/cpp/src/arrow/python/numpy_to_arrow.cc:1098 code:
> compute::Cast(&context, *arr, type_, options, &casted)
> /home/antoine/arrow/cpp/src/arrow/compute/kernels/cast.cc:1022 code:
> Cast(ctx, Datum(array.data()), out_type, options, &datum_out)
> /home/antoine/arrow/cpp/src/arrow/compute/kernels/cast.cc:1009 code:
> GetCastFunction(*value.type(), out_type, options, &func)
> No cast implemented from binary to binary
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
