[
https://issues.apache.org/jira/browse/ARROW-1992?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16342640#comment-16342640
]
ASF GitHub Bot commented on ARROW-1992:
---------------------------------------
xhochy closed pull request #1508: ARROW-1992: [C++/Python] Fix segfault when
string to categorical empty string array
URL: https://github.com/apache/arrow/pull/1508
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/cpp/src/arrow/compute/kernels/hash.cc
b/cpp/src/arrow/compute/kernels/hash.cc
index 1face78bd..8476c8b2d 100644
--- a/cpp/src/arrow/compute/kernels/hash.cc
+++ b/cpp/src/arrow/compute/kernels/hash.cc
@@ -406,12 +406,18 @@ class HashTableKernel<Type, Action,
enable_if_binary<Type>> : public HashTable {
}
Status Append(const ArrayData& arr) override {
+ constexpr uint8_t empty_value = 0;
if (!initialized_) {
RETURN_NOT_OK(Init());
}
const int32_t* offsets = GetValues<int32_t>(arr, 1);
- const uint8_t* data = GetValues<uint8_t>(arr, 2);
+ const uint8_t* data;
+ if (arr.buffers[2].get() == nullptr) {
+ data = &empty_value;
+ } else {
+ data = GetValues<uint8_t>(arr, 2);
+ }
auto action = static_cast<Action*>(this);
RETURN_NOT_OK(action->Reserve(arr.length));
diff --git a/python/pyarrow/tests/test_convert_pandas.py
b/python/pyarrow/tests/test_convert_pandas.py
index 5acb9c3db..fa265e55c 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -1237,6 +1237,21 @@ def test_decimal_metadata(self):
assert data_column['numpy_type'] == 'object'
assert data_column['metadata'] == {'precision': 26, 'scale': 11}
+ def test_table_empty_str(self):
+ values = ['', '', '', '', '']
+ df = pd.DataFrame({'strings': values})
+ field = pa.field('strings', pa.string())
+ schema = pa.schema([field])
+ table = pa.Table.from_pandas(df, schema=schema)
+
+ result1 = table.to_pandas(strings_to_categorical=False)
+ expected1 = pd.DataFrame({'strings': values})
+ tm.assert_frame_equal(result1, expected1, check_dtype=True)
+
+ result2 = table.to_pandas(strings_to_categorical=True)
+ expected2 = pd.DataFrame({'strings': pd.Categorical(values)})
+ tm.assert_frame_equal(result2, expected2, check_dtype=True)
+
def test_table_str_to_categorical_without_na(self):
values = ['a', 'a', 'b', 'b', 'c']
df = pd.DataFrame({'strings': values})
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> [Python] to_pandas crashes when using strings_to_categoricals on empty string
> cols on 0.8.0
> -------------------------------------------------------------------------------------------
>
> Key: ARROW-1992
> URL: https://issues.apache.org/jira/browse/ARROW-1992
> Project: Apache Arrow
> Issue Type: Bug
> Affects Versions: 0.8.0
> Environment: OS: Windows
> Python: PY36 x64
> Pandas: 0.22.0
> pyarrow: 0.8.0
> Reporter: Victor Uriarte
> Assignee: Licht Takeuchi
> Priority: Major
> Labels: pull-request-available
> Fix For: 0.9.0
>
>
> When trying to read back a table, Python crashes when pyarrow is used to
> read/convert a table that has a column of 0 length `strings and
> strings_to_categorical=True`. Example code below.
> This same test ran ok with pyarrow 0.7.1
> {code:none}
> import pandas as pd
> import pyarrow as pa
> df = pd.DataFrame({
> 'Foo': ['A', 'A', 'B', 'B', 'C'],
> 'Bar': ['A1', 'A2', 'B2', 'D3', ''],
> 'Baz': ['', '', '', '', ''],
> })
> table = pa.Table.from_pandas(df)
> df = table.to_pandas(strings_to_categorical=False) # Works
> print('Categoricals=False', len(df))
> df = table.to_pandas(strings_to_categorical=True) # Crashes
> print('Categoricals=True', len(df))
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
