jorisvandenbossche commented on code in PR #34559:
URL: https://github.com/apache/arrow/pull/34559#discussion_r1150345050
##########
python/pyarrow/tests/test_extension_type.py:
##########
@@ -1127,3 +1142,51 @@ def test_cpp_extension_in_python(tmpdir):
reconstructed_array = batch.column(0)
assert reconstructed_array.type == uuid_type
assert reconstructed_array == array
+
+
[email protected]
+def test_extension_to_pandas_storage_type(registered_period_type):
+ period_type, _ = registered_period_type
+ np_arr = np.array([1, 2, 3, 4], dtype='i8')
+ storage = pa.array([1, 2, 3, 4], pa.int64())
+ arr = pa.ExtensionArray.from_storage(period_type, storage)
+
+ if isinstance(period_type, PeriodTypeWithToPandasDtype):
+ pandas_dtype = period_type.to_pandas_dtype()
+ else:
+ pandas_dtype = np_arr.dtype
+
+ # Test arrays
+ result = arr.to_pandas()
+ assert result.dtype == pandas_dtype
+
+ # Test the change in ConvertChunkedArrayToPandas
Review Comment:
```suggestion
# Test chunked arrays
```
(later we won't know what this "change" is about)
##########
python/pyarrow/tests/test_extension_type.py:
##########
@@ -1127,3 +1142,51 @@ def test_cpp_extension_in_python(tmpdir):
reconstructed_array = batch.column(0)
assert reconstructed_array.type == uuid_type
assert reconstructed_array == array
+
+
[email protected]
+def test_extension_to_pandas_storage_type(registered_period_type):
+ period_type, _ = registered_period_type
+ np_arr = np.array([1, 2, 3, 4], dtype='i8')
+ storage = pa.array([1, 2, 3, 4], pa.int64())
+ arr = pa.ExtensionArray.from_storage(period_type, storage)
+
+ if isinstance(period_type, PeriodTypeWithToPandasDtype):
+ pandas_dtype = period_type.to_pandas_dtype()
+ else:
+ pandas_dtype = np_arr.dtype
+
+ # Test arrays
+ result = arr.to_pandas()
+ assert result.dtype == pandas_dtype
+
+ # Test the change in ConvertChunkedArrayToPandas
+ chunked_arr = pa.chunked_array([arr])
+ result = chunked_arr.to_numpy()
+ assert result.dtype == np_arr.dtype
+
+ result = chunked_arr.to_pandas()
+ assert result.dtype == pandas_dtype
+
+ # Test the change in ConvertTableToPandas
Review Comment:
```suggestion
# Test Table.to_pandas
```
##########
python/pyarrow/tests/test_extension_type.py:
##########
@@ -1127,3 +1142,51 @@ def test_cpp_extension_in_python(tmpdir):
reconstructed_array = batch.column(0)
assert reconstructed_array.type == uuid_type
assert reconstructed_array == array
+
+
[email protected]
+def test_extension_to_pandas_storage_type(registered_period_type):
+ period_type, _ = registered_period_type
+ np_arr = np.array([1, 2, 3, 4], dtype='i8')
+ storage = pa.array([1, 2, 3, 4], pa.int64())
+ arr = pa.ExtensionArray.from_storage(period_type, storage)
+
+ if isinstance(period_type, PeriodTypeWithToPandasDtype):
+ pandas_dtype = period_type.to_pandas_dtype()
+ else:
+ pandas_dtype = np_arr.dtype
+
+ # Test arrays
+ result = arr.to_pandas()
+ assert result.dtype == pandas_dtype
+
+ # Test the change in ConvertChunkedArrayToPandas
+ chunked_arr = pa.chunked_array([arr])
+ result = chunked_arr.to_numpy()
+ assert result.dtype == np_arr.dtype
+
+ result = chunked_arr.to_pandas()
+ assert result.dtype == pandas_dtype
+
+ # Test the change in ConvertTableToPandas
+ data = [
+ pa.array([1, 2, 3, 4]),
+ pa.array(['foo', 'bar', None, None]),
+ pa.array([True, None, True, False]),
+ arr
+ ]
+ my_schema = pa.schema([('f0', pa.int8()),
+ ('f1', pa.string()),
+ ('f2', pa.bool_()),
+ ('ext', period_type)])
+ table = pa.Table.from_arrays(data, schema=my_schema)
+ result = table.to_pandas()
+ assert result["ext"].dtype == pandas_dtype
+
+ import pandas as pd
+ if Version(pd.__version__) < Version("1.5.0"):
+ pytest.skip("ArrowDtype missing")
+
+ # Check the usage of types_mapper
+ result = table.to_pandas(types_mapper=pd.ArrowDtype)
+ assert isinstance(result["ext"].dtype, pd.ArrowDtype)
Review Comment:
```suggestion
# Check the usage of types_mapper
result = table.to_pandas(types_mapper=pd.ArrowDtype)
assert isinstance(result["ext"].dtype, pd.ArrowDtype)
```
This should be de-dented, so that this actually runs for latest pandas
versions?
(once you do that, you will also need to trigger crossbow to run the
test-conda-python-3.8-pandas-nightly build)
##########
python/pyarrow/tests/test_extension_type.py:
##########
@@ -1127,3 +1142,51 @@ def test_cpp_extension_in_python(tmpdir):
reconstructed_array = batch.column(0)
assert reconstructed_array.type == uuid_type
assert reconstructed_array == array
+
+
[email protected]
+def test_extension_to_pandas_storage_type(registered_period_type):
+ period_type, _ = registered_period_type
+ np_arr = np.array([1, 2, 3, 4], dtype='i8')
+ storage = pa.array([1, 2, 3, 4], pa.int64())
+ arr = pa.ExtensionArray.from_storage(period_type, storage)
+
+ if isinstance(period_type, PeriodTypeWithToPandasDtype):
+ pandas_dtype = period_type.to_pandas_dtype()
+ else:
+ pandas_dtype = np_arr.dtype
+
+ # Test arrays
+ result = arr.to_pandas()
+ assert result.dtype == pandas_dtype
+
+ # Test the change in ConvertChunkedArrayToPandas
+ chunked_arr = pa.chunked_array([arr])
+ result = chunked_arr.to_numpy()
+ assert result.dtype == np_arr.dtype
+
+ result = chunked_arr.to_pandas()
+ assert result.dtype == pandas_dtype
+
+ # Test the change in ConvertTableToPandas
+ data = [
+ pa.array([1, 2, 3, 4]),
+ pa.array(['foo', 'bar', None, None]),
+ pa.array([True, None, True, False]),
+ arr
+ ]
+ my_schema = pa.schema([('f0', pa.int8()),
+ ('f1', pa.string()),
+ ('f2', pa.bool_()),
+ ('ext', period_type)])
+ table = pa.Table.from_arrays(data, schema=my_schema)
+ result = table.to_pandas()
+ assert result["ext"].dtype == pandas_dtype
+
+ import pandas as pd
+ if Version(pd.__version__) < Version("1.5.0"):
+ pytest.skip("ArrowDtype missing")
Review Comment:
```suggestion
if Version(pd.__version__) >= Version("2.0.0.dev0"):
```
Alternative fix, it might be better to not mark the full test as "skipped",
but just only run this part for newer pandas
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
