This is an automated email from the ASF dual-hosted git repository.
AlenkaF pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 6f161d65250 GH-51302: [Python] Avoid deprecated .values call in
pandas->pyarrow conversion (#51484)
6f161d65250 is described below
commit 6f161d6525047a648f89a6320c2fb71debe2d184
Author: Joris Van den Bossche <[email protected]>
AuthorDate: Thu Oct 1 13:16:29 2026 +0200
GH-51302: [Python] Avoid deprecated .values call in pandas->pyarrow
conversion (#51484)
### Rationale for this change
Pandas has deprecated certain aspects of `.values` (when it looses
information), which our pandas->pyarrow conversion currently relies upon. So
updating that code path to avoid this usage.
### What changes are included in this PR?
Removed usage of `.values` for the deprecated cases (period, interval,
datetimetz). In the custom `get_values`, we already avoided this for period and
interval, added a special case for converting datetimetz to a numpy array. And
ensured that other cases were we did a direct `.values` goes through this
`get_values` helper.
### Are these changes tested?
Yes, I added more tests first to ensure full coverage of all cases that
might run into the deprecation (e.g. the `Schema.from_pandas(df)` or
`pa.array(EA)` with period/datetimetz dtypes were not currently covered)
### Are there any user-facing changes?
No
### Was AI used for this PR?
No
* GitHub Issue: #51302
Authored-by: Joris Van den Bossche <[email protected]>
Signed-off-by: AlenkaF <[email protected]>
---
python/pyarrow/array.pxi | 10 +++----
python/pyarrow/pandas-shim.pxi | 3 ++
python/pyarrow/pandas_compat.py | 15 ++++++----
python/pyarrow/tests/test_pandas.py | 57 ++++++++++++++++++++++++++++++-------
python/pyarrow/tests/test_schema.py | 9 ++++++
5 files changed, 72 insertions(+), 22 deletions(-)
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 223b77dd4fa..bf9a6d67228 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -5411,16 +5411,16 @@ cdef object get_array_class_from_type(
return _array_classes[data_type.id()]
-cdef object get_values(object obj, bint* is_series):
+cdef object get_values(object obj, bint* is_pandas_object):
if pandas_api.is_series(obj) or pandas_api.is_index(obj):
result = pandas_api.get_values(obj)
- is_series[0] = True
+ is_pandas_object[0] = True
elif isinstance(obj, np.ndarray):
result = obj
- is_series[0] = False
+ is_pandas_object[0] = False
else:
- result = pandas_api.series(obj, copy=False).values
- is_series[0] = False
+ result = pandas_api.get_values(pandas_api.series(obj, copy=False))
+ is_pandas_object[0] = False
return result
diff --git a/python/pyarrow/pandas-shim.pxi b/python/pyarrow/pandas-shim.pxi
index 603b2b09d40..bd29eb03222 100644
--- a/python/pyarrow/pandas-shim.pxi
+++ b/python/pyarrow/pandas-shim.pxi
@@ -239,6 +239,9 @@ cdef class _PandasAPIShim(object):
if isinstance(obj.dtype, (self.pd.api.types.IntervalDtype,
self.pd.api.types.PeriodDtype)):
return obj.array
+ elif isinstance(obj.dtype, self.pd.api.types.DatetimeTZDtype):
+ # get the underlying to numpy datetime64 UTC values
+ return obj.to_numpy("datetime64[{}]".format(obj.dtype.unit))
return obj.values
cdef _PandasAPIShim pandas_api = _PandasAPIShim()
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 36d336d5135..495df5645f3 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -578,7 +578,7 @@ def dataframe_to_types(df, preserve_index, columns=None):
types = []
# If pandas knows type, skip conversion
for c in columns_to_convert:
- values = c.values
+ values = _pandas_api.get_values(c)
if _pandas_api.is_categorical(values):
type_ = pa.array(c, from_pandas=True).type
elif _pandas_api.is_extension_array_dtype(values):
@@ -646,10 +646,13 @@ def dataframe_to_arrays(df, schema, preserve_index,
nthreads=1, columns=None,
f"had {result.null_count} null values")
return result
- def _can_definitely_zero_copy(arr):
- return (isinstance(arr, np.ndarray) and
- arr.flags.contiguous and
- issubclass(arr.dtype.type, np.integer))
+ def _can_definitely_zero_copy(ser):
+ if isinstance(ser.dtype, np.dtype):
+ arr = ser.values
+ return (isinstance(arr, np.ndarray) and
+ arr.flags.contiguous and
+ issubclass(arr.dtype.type, np.integer))
+ return False
if nthreads == 1:
arrays = [convert_column(c, f)
@@ -658,7 +661,7 @@ def dataframe_to_arrays(df, schema, preserve_index,
nthreads=1, columns=None,
arrays = []
with futures.ThreadPoolExecutor(nthreads) as executor:
for c, f in zip(columns_to_convert, convert_fields):
- if _can_definitely_zero_copy(c.values):
+ if _can_definitely_zero_copy(c):
arrays.append(convert_column(c, f))
else:
arrays.append(executor.submit(convert_column, c, f))
diff --git a/python/pyarrow/tests/test_pandas.py
b/python/pyarrow/tests/test_pandas.py
index dd20a0aa977..6fc9925eb01 100644
--- a/python/pyarrow/tests/test_pandas.py
+++ b/python/pyarrow/tests/test_pandas.py
@@ -3334,21 +3334,31 @@ def _fully_loaded_dataframe_example():
c1 = pd.date_range('2000-01-01', periods=10)
data = {
- 0: c1,
- 1: c1.tz_localize('utc'),
- 2: c1.tz_localize('US/Eastern'),
- 3: c1[::2].tz_localize('utc').repeat(2).astype('category'),
- 4: ['foo', 'bar'] * 5,
- 5: pd.Series(['foo', 'bar'] * 5).astype('category').values,
- 6: [True, False] * 5,
- 7: np.random.randn(10),
- 8: np.random.randint(0, 100, size=10),
- 9: pd.period_range('2013', periods=10, freq='M'),
- 10: pd.interval_range(start=1, freq=1, periods=10),
+ "col0": c1,
+ "col1": c1.tz_localize('utc'),
+ "col2": c1.tz_localize('US/Eastern'),
+ "col3": c1[::2].tz_localize('utc').repeat(2).astype('category'),
+ "col4": ['foo', 'bar'] * 5,
+ "col5": pd.Series(['foo', 'bar'] * 5).astype('category').values,
+ "col6": [True, False] * 5,
+ "col7": np.random.randn(10),
+ "col8": np.random.randint(0, 100, size=10),
+ "col9": pd.period_range('2013', periods=10, freq='M'),
+ "col10": pd.interval_range(start=1, freq=1, periods=10),
}
return pd.DataFrame(data, index=index)
+def test_roundtrip_fully_loaded_dataframe_example():
+ df = _fully_loaded_dataframe_example()
+ expected = df.copy()
+ expected["col3"] = df["col3"].cat.rename_categories(
+ df["col3"].cat.categories.tz_convert(None))
+ _check_pandas_roundtrip(df, preserve_index=None, expected=expected)
+ _check_pandas_roundtrip(df, preserve_index=None,
+ expected=expected, use_threads=True)
+
+
@pytest.mark.parametrize('columns', ([b'foo'], ['foo']))
def test_roundtrip_with_bytes_unicode(columns):
df = pd.DataFrame(columns=columns)
@@ -5327,3 +5337,28 @@ def test_json_unserializable_pd_df_attrs():
pd_metadata = json.loads(df_table.schema.metadata[b"pandas"])
assert not pd_metadata["attributes"]
+
+
+def test_pandas_array_likes_with_extension_arrays():
+ # https://github.com/apache/arrow/issues/51302
+ dtidx = pd.date_range("2025-01-01", periods=10)
+
+ arrays = [
+ (dtidx.array.tz_localize("UTC"), None),
+ (dtidx.array.tz_localize("Europe/Paris"), None),
+ (dtidx.array.tz_localize("Europe/Paris").as_unit("s"), None),
+ (pd.period_range('2013', periods=10, freq='M'), None),
+ (pd.interval_range(start=1, freq=1, periods=10), None),
+ (pd.array([1, 2, 3], dtype="Int32"), np.array([1, 2, 3],
dtype="int32")),
+ (pd.array([1.1, 2.2, None], dtype="Float64"),
+ np.array([1.1, 2.2, np.nan], dtype="float64")),
+ (pd.array([True, False, True], dtype="boolean"),
+ np.array([True, False, True], dtype="bool")),
+ (pd.array(["a", "b", None], dtype="string"),
+ np.array(["a", "b", None], dtype="object")),
+ ]
+
+ for arr in arrays:
+ for box in [pd.array, pd.Index, pd.Series]:
+ _check_array_roundtrip(box(arr[0]), expected=pd.Series(
+ arr[1]) if arr[1] is not None else None)
diff --git a/python/pyarrow/tests/test_schema.py
b/python/pyarrow/tests/test_schema.py
index 27b6419f9ad..93519fc0173 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -27,6 +27,7 @@ except ImportError:
import pyarrow as pa
import pyarrow.tests.util as test_util
+from pyarrow.tests.test_pandas import _fully_loaded_dataframe_example
def test_schema_constructor_errors():
@@ -719,6 +720,14 @@ def test_schema_from_pandas():
assert schema == expected
[email protected]
+def test_schema_from_pandas_all_types():
+ df = _fully_loaded_dataframe_example()
+ schema = pa.Schema.from_pandas(df)
+ expected = pa.Table.from_pandas(df).schema
+ assert schema == expected
+
+
def test_schema_sizeof():
schema = pa.schema([
pa.field('foo', pa.int32()),
