This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new b0b792ac88 GH-48808: [Python] Drop support for Pandas < 2.0.3 (#50444)
b0b792ac88 is described below
commit b0b792ac886fded79e26b0039aff074a6bd3dc95
Author: Raúl Cumplido <[email protected]>
AuthorDate: Thu Jul 9 19:30:12 2026 +0200
GH-48808: [Python] Drop support for Pandas < 2.0.3 (#50444)
### Rationale for this change
[Pandas 2.0.0](https://pypi.org/project/pandas/2.0.0/) was released on
April 3rd 2023.
The last version of Pandas 1 was [Pandas
1.5.3](https://pypi.org/project/pandas/1.5.3/) and it was released on January
19th 2023. That version only supports up to Python 3.11.
We are currently supporting up to Pandas 1.5.2 (first version supporting
3.11).
On the issue we discussed to bump our Pandas support to 2.0.0 but given
2.0.3 is the latest in the 2.0.x, was released on June 2023, only supports up
to Python 3.11 and bumping to it allows use to improve even further some of the
conditionals we are bumping to Pandas >= 2.0.3
### What changes are included in this PR?
Update minimal support check to Version(2.0.3) and remove all test skips,
conditionals and scenarios where we are checking for Pandas < 2.0.3.
We found a couple of minor things
### Are these changes tested?
Yes via CI
### Are there any user-facing changes?
Yes, Pandas < 2 won't be supported.
* GitHub Issue: #48808
Authored-by: Raúl Cumplido <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
.github/workflows/python.yml | 8 +-
dev/tasks/tasks.yml | 2 +-
docs/source/python/install.rst | 2 +-
python/pyarrow/array.pxi | 4 -
python/pyarrow/pandas-shim.pxi | 14 +--
python/pyarrow/pandas_compat.py | 2 -
python/pyarrow/src/arrow/python/arrow_to_pandas.cc | 2 +-
python/pyarrow/table.pxi | 4 -
.../pyarrow/tests/interchange/test_conversion.py | 89 ++++--------------
python/pyarrow/tests/parquet/test_pandas.py | 3 -
python/pyarrow/tests/test_compute.py | 18 +---
python/pyarrow/tests/test_pandas.py | 103 ++-------------------
python/pyarrow/tests/test_schema.py | 8 --
python/pyarrow/types.pxi | 4 +-
14 files changed, 47 insertions(+), 216 deletions(-)
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 30e6272fb3..713168545f 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -67,7 +67,7 @@ jobs:
name:
- conda-python-docs
- conda-python-3.12-nopandas
- - conda-python-3.11-pandas-1.5.2
+ - conda-python-3.11-pandas-2.0.3
- conda-python-3.14-pandas-latest
- conda-python-3.13-no-numpy
include:
@@ -79,11 +79,11 @@ jobs:
image: conda-python
title: AMD64 Conda Python 3.12 Without Pandas
python: "3.12"
- - name: conda-python-3.11-pandas-1.5.2
+ - name: conda-python-3.11-pandas-2.0.3
image: conda-python-pandas
- title: AMD64 Conda Python 3.11 Pandas 1.5.2
+ title: AMD64 Conda Python 3.11 Pandas 2.0.3
python: "3.11"
- pandas: "1.5.2"
+ pandas: "2.0.3"
numpy: "1.23.2"
- name: conda-python-3.14-pandas-latest
image: conda-python-pandas
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index c443c5a7e7..89d239a0a2 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -727,7 +727,7 @@ tasks:
############################## Integration tests ############################
-{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11",
"1.5.2", "1.23.2", True),
+{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11",
"2.0.3", "1.23.2", True),
("3.12",
"latest", "latest", False),
("3.13",
"latest", "1.26.2", False),
("3.13",
"latest", "latest", False),
diff --git a/docs/source/python/install.rst b/docs/source/python/install.rst
index 5fa9ab6c77..d076ca9643 100644
--- a/docs/source/python/install.rst
+++ b/docs/source/python/install.rst
@@ -74,7 +74,7 @@ Dependencies
Optional dependencies
* **NumPy 1.23.2** or higher.
-* **pandas 1.5.2** or higher,
+* **pandas 2.0.3** or higher,
* **cffi**.
Additional packages PyArrow is compatible with are :ref:`fsspec
<filesystem-fsspec>`
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index 5fc74969ab..e259a5e4fb 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -2383,10 +2383,6 @@ cdef _array_like_to_pandas(obj, options, types_mapper):
arr = dtype.__from_arrow__(obj)
return pandas_api.series(arr, name=name, copy=False)
- if pandas_api.is_v1():
- # ARROW-3789: Coerce date/timestamp types to datetime64[ns]
- c_options.coerce_temporal_nanoseconds = True
-
if isinstance(obj, Array):
with nogil:
check_status(ConvertArrayToPandas(c_options,
diff --git a/python/pyarrow/pandas-shim.pxi b/python/pyarrow/pandas-shim.pxi
index 3dbb97cd80..94802da6b0 100644
--- a/python/pyarrow/pandas-shim.pxi
+++ b/python/pyarrow/pandas-shim.pxi
@@ -38,7 +38,7 @@ cdef class _PandasAPIShim(object):
object _array_like_types, _is_extension_array_dtype, _lock
bint has_sparse
bint _pd024
- bint _is_v1, _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict
+ bint _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict
def __init__(self):
self._lock = Lock()
@@ -61,25 +61,23 @@ cdef class _PandasAPIShim(object):
self._pd = pd
self._version = pd.__version__
self._loose_version = Version(pd.__version__)
- self._is_v1 = False
- if self._loose_version < Version('1.0.0'):
+ if self._loose_version < Version('2.0.3'):
self._have_pandas = False
if raise_:
raise ImportError(
- f"pyarrow requires pandas 1.0.0 or above, pandas
{self._version} is "
+ f"pyarrow requires pandas 2.0.3 or above, pandas
{self._version} is "
"installed"
)
else:
warnings.warn(
- f"pyarrow requires pandas 1.0.0 or above, pandas
{self._version} is "
+ f"pyarrow requires pandas 2.0.3 or above, pandas
{self._version} is "
"installed. Therefore, pandas-specific integration is not "
"used.",
stacklevel=2
)
return
- self._is_v1 = self._loose_version < Version('2.0.0')
self._is_ge_v21 = self._loose_version >= Version('2.1.0')
self._is_ge_v23 = self._loose_version >= Version('2.3.0.dev0')
self._is_ge_v3 = self._loose_version >= Version('3.0.0.dev0')
@@ -166,10 +164,6 @@ cdef class _PandasAPIShim(object):
self._check_import()
return self._version
- def is_v1(self):
- self._check_import()
- return self._is_v1
-
def is_ge_v21(self):
self._check_import()
return self._is_ge_v21
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index d8fd383d31..ccb89fc05d 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -790,8 +790,6 @@ def _reconstruct_block(item, columns=None,
extension_columns=None, return_block=
def make_datetimetz(unit, tz):
- if _pandas_api.is_v1():
- unit = 'ns' # ARROW-3789: Coerce date/timestamp types to
datetime64[ns]
tz = pa.lib.string_to_tzinfo(tz, prefer_zoneinfo=_pandas_api.is_ge_v3())
return _pandas_api.datetimetz_type(unit, tz=tz)
diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc
b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc
index f163266f3b..348d352a04 100644
--- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc
+++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc
@@ -1293,7 +1293,7 @@ struct ObjectWriterVisitor {
PyObject** out) {
ARROW_DCHECK(internal::BorrowPandasDataOffsetType() != nullptr);
// DateOffset objects do not add nanoseconds component to pd.Timestamp.
- // as of Pandas 1.3.3
+ // as of Pandas 1.3.3
// (https://github.com/pandas-dev/pandas/issues/43892).
// So convert microseconds and remainder to preserve data
// but give users more expected results.
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index fc7c4fcfc8..17ca35e476 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -4070,10 +4070,6 @@ def table_to_blocks(options, Table table, categories,
extension_columns):
c_options.extension_columns = make_shared[unordered_set[c_string]](
unordered_set[c_string]({tobytes(col) for col in
extension_columns}))
- if pandas_api.is_v1():
- # ARROW-3789: Coerce date/timestamp types to datetime64[ns]
- c_options.coerce_temporal_nanoseconds = True
-
if c_options.self_destruct:
# Move the shared_ptr, table is now unsafe to use further
c_table = move(table.sp_table)
diff --git a/python/pyarrow/tests/interchange/test_conversion.py
b/python/pyarrow/tests/interchange/test_conversion.py
index 81713b94e6..cc105178ad 100644
--- a/python/pyarrow/tests/interchange/test_conversion.py
+++ b/python/pyarrow/tests/interchange/test_conversion.py
@@ -17,7 +17,6 @@
from datetime import datetime as dt
import pyarrow as pa
-from pyarrow.vendored.version import Version
import pytest
try:
@@ -122,9 +121,6 @@ def test_offset_of_sliced_array():
]
)
def test_pandas_roundtrip(uint, int, float, np_float_str):
- if Version(pd.__version__) < Version("1.5.0"):
- pytest.skip("__dataframe__ added to pandas in 1.5.0")
-
arr = [1, 2, 3]
table = pa.table(
{
@@ -153,9 +149,6 @@ def test_pandas_roundtrip(uint, int, float, np_float_str):
@pytest.mark.pandas
def test_pandas_roundtrip_string():
# See https://github.com/pandas-dev/pandas/issues/50554
- if Version(pd.__version__) < Version("1.6"):
- pytest.skip("Column.size() bug in pandas")
-
arr = ["a", "", "c"]
table = pa.table({"a": pa.array(arr)})
@@ -182,9 +175,6 @@ def test_pandas_roundtrip_string():
@pytest.mark.pandas
def test_pandas_roundtrip_large_string():
# See https://github.com/pandas-dev/pandas/issues/50554
- if Version(pd.__version__) < Version("1.6"):
- pytest.skip("Column.size() bug in pandas")
-
arr = ["a", "", "c"]
table = pa.table({"a_large": pa.array(arr, type=pa.large_string())})
@@ -192,36 +182,25 @@ def test_pandas_roundtrip_large_string():
from_dataframe as pandas_from_dataframe
)
- if Version(pd.__version__) >= Version("2.0.1"):
- pandas_df = pandas_from_dataframe(table)
- result = pi.from_dataframe(pandas_df)
-
- assert result["a_large"].to_pylist() == table["a_large"].to_pylist()
- assert pa.types.is_large_string(table["a_large"].type)
- assert pa.types.is_large_string(result["a_large"].type)
+ pandas_df = pandas_from_dataframe(table)
+ result = pi.from_dataframe(pandas_df)
- table_protocol = table.__dataframe__()
- result_protocol = result.__dataframe__()
+ assert result["a_large"].to_pylist() == table["a_large"].to_pylist()
+ assert pa.types.is_large_string(table["a_large"].type)
+ assert pa.types.is_large_string(result["a_large"].type)
- assert table_protocol.num_columns() == result_protocol.num_columns()
- assert table_protocol.num_rows() == result_protocol.num_rows()
- assert table_protocol.num_chunks() == result_protocol.num_chunks()
- assert table_protocol.column_names() == result_protocol.column_names()
+ table_protocol = table.__dataframe__()
+ result_protocol = result.__dataframe__()
- else:
- # large string not supported by pandas implementation for
- # older versions of pandas
- # https://github.com/pandas-dev/pandas/issues/52795
- with pytest.raises(AssertionError):
- pandas_from_dataframe(table)
+ assert table_protocol.num_columns() == result_protocol.num_columns()
+ assert table_protocol.num_rows() == result_protocol.num_rows()
+ assert table_protocol.num_chunks() == result_protocol.num_chunks()
+ assert table_protocol.column_names() == result_protocol.column_names()
@pytest.mark.pandas
def test_pandas_roundtrip_string_with_missing():
# See https://github.com/pandas-dev/pandas/issues/50554
- if Version(pd.__version__) < Version("1.6"):
- pytest.skip("Column.size() bug in pandas")
-
arr = ["a", "", "c", None]
table = pa.table({"a": pa.array(arr),
"a_large": pa.array(arr, type=pa.large_string())})
@@ -230,29 +209,20 @@ def test_pandas_roundtrip_string_with_missing():
from_dataframe as pandas_from_dataframe
)
- if Version(pd.__version__) >= Version("2.0.2"):
- pandas_df = pandas_from_dataframe(table)
- result = pi.from_dataframe(pandas_df)
+ pandas_df = pandas_from_dataframe(table)
+ result = pi.from_dataframe(pandas_df)
- assert result["a"].to_pylist() == table["a"].to_pylist()
- assert pa.types.is_string(table["a"].type)
- assert pa.types.is_large_string(result["a"].type)
+ assert result["a"].to_pylist() == table["a"].to_pylist()
+ assert pa.types.is_string(table["a"].type)
+ assert pa.types.is_large_string(result["a"].type)
- assert result["a_large"].to_pylist() == table["a_large"].to_pylist()
- assert pa.types.is_large_string(table["a_large"].type)
- assert pa.types.is_large_string(result["a_large"].type)
- else:
- # older versions of pandas do not have bitmask support
- # https://github.com/pandas-dev/pandas/issues/49888
- with pytest.raises(NotImplementedError):
- pandas_from_dataframe(table)
+ assert result["a_large"].to_pylist() == table["a_large"].to_pylist()
+ assert pa.types.is_large_string(table["a_large"].type)
+ assert pa.types.is_large_string(result["a_large"].type)
@pytest.mark.pandas
def test_pandas_roundtrip_categorical():
- if Version(pd.__version__) < Version("2.0.2"):
- pytest.skip("Bitmasks not supported in pandas interchange
implementation")
-
arr = ["Mon", "Tue", "Mon", "Wed", "Mon", "Thu", "Fri", "Sat", None]
table = pa.table(
{"weekday": pa.array(arr).dictionary_encode()}
@@ -299,8 +269,6 @@ def test_pandas_roundtrip_categorical():
@pytest.mark.pandas
@pytest.mark.parametrize("unit", ['s', 'ms', 'us', 'ns'])
def test_pandas_roundtrip_datetime(unit):
- if Version(pd.__version__) < Version("1.5.0"):
- pytest.skip("__dataframe__ added to pandas in 1.5.0")
from datetime import datetime as dt
# timezones not included as they are not yet supported in
@@ -308,12 +276,7 @@ def test_pandas_roundtrip_datetime(unit):
dt_arr = [dt(2007, 7, 13), dt(2007, 7, 14), dt(2007, 7, 15)]
table = pa.table({"a": pa.array(dt_arr, type=pa.timestamp(unit))})
- if Version(pd.__version__) < Version("1.6"):
- # pandas < 2.0 always creates datetime64 in "ns"
- # resolution
- expected = pa.table({"a": pa.array(dt_arr, type=pa.timestamp('ns'))})
- else:
- expected = table
+ expected = table
from pandas.api.interchange import (
from_dataframe as pandas_from_dataframe
@@ -337,9 +300,6 @@ def test_pandas_roundtrip_datetime(unit):
"np_float_str", ["float32", "float64"]
)
def test_pandas_to_pyarrow_with_missing(np_float_str):
- if Version(pd.__version__) < Version("1.5.0"):
- pytest.skip("__dataframe__ added to pandas in 1.5.0")
-
np_array = np.array([0, np.nan, 2], dtype=np.dtype(np_float_str))
datetime_array = [None, dt(2007, 7, 14), dt(2007, 7, 15)]
df = pd.DataFrame({
@@ -359,9 +319,6 @@ def test_pandas_to_pyarrow_with_missing(np_float_str):
@pytest.mark.pandas
def test_pandas_to_pyarrow_float16_with_missing():
- if Version(pd.__version__) < Version("1.5.0"):
- pytest.skip("__dataframe__ added to pandas in 1.5.0")
-
# np.float16 errors if ps.is_nan is used
# pyarrow.lib.ArrowNotImplementedError: Function 'is_nan' has no kernel
# matching input types (halffloat)
@@ -482,9 +439,6 @@ def test_nan_as_null():
@pytest.mark.pandas
def test_allow_copy_false():
- if Version(pd.__version__) < Version("1.5.0"):
- pytest.skip("__dataframe__ added to pandas in 1.5.0")
-
# Test that an error is raised when a copy is needed
# to create a bitmask
@@ -501,9 +455,6 @@ def test_allow_copy_false():
@pytest.mark.pandas
def test_allow_copy_false_bool_categorical():
- if Version(pd.__version__) < Version("1.5.0"):
- pytest.skip("__dataframe__ added to pandas in 1.5.0")
-
# Test that an error is raised for boolean
# and categorical dtype (copy is always made)
diff --git a/python/pyarrow/tests/parquet/test_pandas.py
b/python/pyarrow/tests/parquet/test_pandas.py
index 53864ff15e..8dbd5d71b9 100644
--- a/python/pyarrow/tests/parquet/test_pandas.py
+++ b/python/pyarrow/tests/parquet/test_pandas.py
@@ -524,9 +524,6 @@ def test_pandas_categorical_roundtrip():
@pytest.mark.pandas
def test_categories_with_string_pyarrow_dtype(tempdir):
# gh-33727: writing to parquet should not fail
- if Version(pd.__version__) < Version("1.3.0"):
- pytest.skip("PyArrow backed string data type introduced in pandas
1.3.0")
-
df1 = pd.DataFrame({"x": ["foo", "bar", "foo"]}, dtype="string[pyarrow]")
df1 = df1.astype("category")
diff --git a/python/pyarrow/tests/test_compute.py
b/python/pyarrow/tests/test_compute.py
index 39167e76c8..1e08e73668 100644
--- a/python/pyarrow/tests/test_compute.py
+++ b/python/pyarrow/tests/test_compute.py
@@ -2612,8 +2612,6 @@ def test_strftime():
def _check_datetime_components(timestamps, timezone=None):
- from pyarrow.vendored.version import Version
-
ts = pd.to_datetime(timestamps).tz_localize(
"UTC").tz_convert(timezone).to_series()
tsa = pa.array(ts, pa.timestamp("ns", tz=timezone))
@@ -2626,17 +2624,11 @@ def _check_datetime_components(timestamps,
timezone=None):
pa.field('iso_day_of_week', pa.int64())
]
- if Version(pd.__version__) < Version("1.1.0"):
- # https://github.com/pandas-dev/pandas/issues/33206
- iso_year = ts.map(lambda x: x.isocalendar()[0]).astype("int64")
- iso_week = ts.map(lambda x: x.isocalendar()[1]).astype("int64")
- iso_day = ts.map(lambda x: x.isocalendar()[2]).astype("int64")
- else:
- # Casting is required because pandas isocalendar returns int32
- # while arrow isocalendar returns int64.
- iso_year = ts.dt.isocalendar()["year"].astype("int64")
- iso_week = ts.dt.isocalendar()["week"].astype("int64")
- iso_day = ts.dt.isocalendar()["day"].astype("int64")
+ # Casting is required because pandas isocalendar returns int32
+ # while arrow isocalendar returns int64.
+ iso_year = ts.dt.isocalendar()["year"].astype("int64")
+ iso_week = ts.dt.isocalendar()["week"].astype("int64")
+ iso_day = ts.dt.isocalendar()["day"].astype("int64")
iso_calendar = pa.StructArray.from_arrays(
[iso_year, iso_week, iso_day],
diff --git a/python/pyarrow/tests/test_pandas.py
b/python/pyarrow/tests/test_pandas.py
index c42dc04cd1..2a782de164 100644
--- a/python/pyarrow/tests/test_pandas.py
+++ b/python/pyarrow/tests/test_pandas.py
@@ -488,14 +488,10 @@ class TestConvertMetadata:
@pytest.mark.parametrize('unit', ['us', 'ns'])
def test_datetimetz_column_index(self, unit):
- ext_kwargs = {}
- if Version(pd.__version__) >= Version("2.0.0"):
- # unit argument not supported on date_range for pandas < 2.0.0
- ext_kwargs = {'unit': unit}
df = pd.DataFrame(
[(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)],
columns=pd.date_range(
- start='2017-01-01', periods=3, tz='America/New_York',
**ext_kwargs
+ start='2017-01-01', periods=3, tz='America/New_York', unit=unit
)
)
t = pa.Table.from_pandas(df, preserve_index=True)
@@ -504,10 +500,7 @@ class TestConvertMetadata:
column_indexes, = js['column_indexes']
assert column_indexes['name'] is None
assert column_indexes['pandas_type'] == 'datetimetz'
- if ext_kwargs:
- assert column_indexes['numpy_type'] == f'datetime64[{unit}]'
- else:
- assert column_indexes['numpy_type'] == 'datetime64[ns]'
+ assert column_indexes['numpy_type'] == f'datetime64[{unit}]'
md = column_indexes['metadata']
assert md['timezone'] == 'America/New_York'
@@ -751,12 +744,8 @@ class TestConvertMetadata:
# It is possible that the metadata and actual schema is not fully
# matching (eg no timezone information for tz-aware column)
# -> to_pandas() conversion should not fail on that
- ext_kwargs = {}
- if Version(pd.__version__) >= Version("2.0.0"):
- # unit argument not supported on date_range for pandas < 2.0.0
- ext_kwargs = {'unit': 'ns'}
df = pd.DataFrame({"datetime": pd.date_range(
- "2020-01-01", periods=3, **ext_kwargs)})
+ "2020-01-01", periods=3, unit='ns')})
# OPTION 1: casting after conversion
table = pa.Table.from_pandas(df)
@@ -1156,8 +1145,6 @@ class TestConvertDateTimeLikeTypes:
@pytest.mark.parametrize('unit', ['s', 'ms', 'us', 'ns'])
def test_timestamps_with_timezone(self, unit):
- if Version(pd.__version__) < Version("2.0.0") and unit != 'ns':
- pytest.skip("pandas < 2.0 only supports nanosecond datetime64")
df = pd.DataFrame({
'datetime64': np.array([
'2007-07-13T01:23:34.123',
@@ -1347,10 +1334,6 @@ class TestConvertDateTimeLikeTypes:
expected_days = np.array(['2000-01-01', None, '1970-01-01',
'2040-02-26'], dtype='datetime64[D]')
- if Version(pd.__version__) < Version("2.0.0"):
- # ARROW-3789: Coerce date/timestamp types to datetime64[ns]
- expected_dtype = 'datetime64[ns]'
-
expected = np.array(['2000-01-01', None, '1970-01-01',
'2040-02-26'], dtype=expected_dtype)
@@ -1646,19 +1629,6 @@ class TestConvertDateTimeLikeTypes:
dtype='datetime64[s]')
_check_array_from_pandas_roundtrip(datetime64_s)
- def test_timestamp_to_pandas_coerces_to_ns(self):
- # non-ns timestamp gets cast to ns on conversion to pandas
- if Version(pd.__version__) >= Version("2.0.0"):
- pytest.skip("pandas >= 2.0 supports non-nanosecond datetime64")
-
- arr = pa.array([1, 2, 3], pa.timestamp('ms'))
- expected = pd.Series(pd.to_datetime([1, 2, 3], unit='ms'))
- s = arr.to_pandas()
- tm.assert_series_equal(s, expected)
- arr = pa.chunked_array([arr])
- s = arr.to_pandas()
- tm.assert_series_equal(s, expected)
-
def test_timestamp_to_pandas_out_of_bounds(self):
# ARROW-7758 check for out of bounds timestamps for non-ns timestamps
# that end up getting coerced into ns timestamps.
@@ -1735,8 +1705,6 @@ class TestConvertDateTimeLikeTypes:
@pytest.mark.parametrize("unit", ['s', 'ms', 'us', 'ns'])
def test_timedeltas_no_nulls(self, unit):
- if Version(pd.__version__) < Version("2.0.0"):
- unit = 'ns'
df = pd.DataFrame({
'timedelta64': np.array([0, 3600000000000, 7200000000000],
dtype=f'timedelta64[{unit}]')
@@ -1750,8 +1718,6 @@ class TestConvertDateTimeLikeTypes:
@pytest.mark.parametrize("unit", ['s', 'ms', 'us', 'ns'])
def test_timedeltas_nulls(self, unit):
- if Version(pd.__version__) < Version("2.0.0"):
- unit = 'ns'
df = pd.DataFrame({
'timedelta64': np.array([0, None, 7200000000000],
dtype=f'timedelta64[{unit}]')
@@ -2419,11 +2385,6 @@ class TestConvertListTypes:
tm.assert_series_equal(series, expected)
def test_to_list_of_maps_pandas(self):
- if ((Version(np.__version__) >= Version("1.25.0.dev0")) and
- (Version(pd.__version__) < Version("2.0.0"))):
- # TODO: regression in pandas with numpy 1.25dev
- # https://github.com/pandas-dev/pandas/issues/50360
- pytest.skip("Regression in pandas with numpy 1.25")
data = [
[[('foo', ['a', 'b']), ('bar', ['c', 'd'])]],
[[('baz', []), ('qux', None), ('quux', [None, 'e'])], [('quz',
['f', 'g'])]]
@@ -2443,13 +2404,6 @@ class TestConvertListTypes:
"""
A slightly more rigorous test for chunk/slice combinations
"""
-
- if ((Version(np.__version__) >= Version("1.25.0.dev0")) and
- (Version(pd.__version__) < Version("2.0.0"))):
- # TODO: regression in pandas with numpy 1.25dev
- # https://github.com/pandas-dev/pandas/issues/50360
- pytest.skip("Regression in pandas with numpy 1.25")
-
keys = pa.array(['ignore', 'foo', 'bar', 'baz',
'qux', 'quux', 'ignore']).slice(1, 5)
items = pa.array(
@@ -3799,9 +3753,7 @@ def test_table_from_pandas_schema_field_order_metadata():
result = table.to_pandas()
coerce_cols_to_types = {"float": "float32"}
- if Version(pd.__version__) >= Version("2.0.0"):
- # Pandas v2 now support non-nanosecond time units
- coerce_cols_to_types["datetime"] = "datetime64[s, UTC]"
+ coerce_cols_to_types["datetime"] = "datetime64[s, UTC]"
expected = df[["float", "datetime"]].astype(coerce_cols_to_types)
tm.assert_frame_equal(result, expected)
@@ -3969,10 +3921,7 @@ def test_to_pandas_split_blocks():
def _get_mgr(df):
- if Version(pd.__version__) < Version("1.1.0"):
- return df._data
- else:
- return df._mgr
+ return df._mgr
def _check_blocks_created(t, number):
@@ -4383,9 +4332,6 @@ def test_dictionary_from_pandas_specified_type():
def test_convert_categories_to_array_with_string_pyarrow_dtype():
# gh-33727: categories should be converted to pa.Array
- if Version(pd.__version__) < Version("1.3.0"):
- pytest.skip("PyArrow backed string data type introduced in pandas
1.3.0")
-
df = pd.DataFrame({"x": ["foo", "bar", "foo"]}, dtype="string[pyarrow]")
df = df.astype("category")
indices = pa.array(df['x'].cat.codes)
@@ -4524,12 +4470,8 @@ def test_convert_to_extension_array(monkeypatch):
tm.assert_frame_equal(result, df2)
# monkeypatch pandas Int64Dtype to *not* have the protocol method
- if Version(pd.__version__) < Version("1.3.0.dev"):
- monkeypatch.delattr(
- pd.core.arrays.integer._IntegerDtype, "__from_arrow__")
- else:
- monkeypatch.delattr(
- pd.core.arrays.integer.NumericDtype, "__from_arrow__")
+ monkeypatch.delattr(
+ pd.core.arrays.integer.NumericDtype, "__from_arrow__")
# Int64Dtype has no __from_arrow__ -> use normal conversion
result = table.to_pandas()
assert len(_get_mgr(result).blocks) == 1
@@ -4569,12 +4511,8 @@ def
test_conversion_extensiontype_to_extensionarray(monkeypatch):
# monkeypatch pandas Int64Dtype to *not* have the protocol method
# (remove the version added above and the actual version for recent pandas)
- if Version(pd.__version__) < Version("1.3.0.dev"):
- monkeypatch.delattr(
- pd.core.arrays.integer._IntegerDtype, "__from_arrow__")
- else:
- monkeypatch.delattr(
- pd.core.arrays.integer.NumericDtype, "__from_arrow__")
+ monkeypatch.delattr(
+ pd.core.arrays.integer.NumericDtype, "__from_arrow__")
result = arr.to_pandas()
assert _get_mgr(result).blocks[0].values.dtype == np.dtype("int64")
@@ -4611,9 +4549,6 @@ def test_to_pandas_extension_dtypes_mapping():
def test_to_pandas_extension_dtypes_mapping_complex_type():
- # https://github.com/apache/arrow/pull/44720
- if Version(pd.__version__) < Version("1.5.2"):
- pytest.skip("Test relies on pd.ArrowDtype")
pa_type = pa.struct(
[
pa.field("bar", pa.bool_(), nullable=False),
@@ -4636,9 +4571,6 @@ def
test_to_pandas_extension_dtypes_mapping_complex_type():
def test_array_to_pandas():
- if Version(pd.__version__) < Version("1.1"):
- pytest.skip("ExtensionDtype to_pandas method missing")
-
for arr in [pd.period_range("2012-01-01", periods=3, freq="D").array,
pd.interval_range(1, 4).array]:
result = pa.array(arr).to_pandas()
@@ -4653,20 +4585,11 @@ def test_array_to_pandas():
def test_roundtrip_empty_table_with_extension_dtype_index():
df = pd.DataFrame(index=pd.interval_range(start=0, end=3))
table = pa.table(df)
- if Version(pd.__version__) > Version("1.0"):
- tm.assert_index_equal(table.to_pandas().index, df.index)
- else:
- tm.assert_index_equal(table.to_pandas().index,
- pd.Index([{'left': 0, 'right': 1},
- {'left': 1, 'right': 2},
- {'left': 2, 'right': 3}],
- dtype='object'))
+ tm.assert_index_equal(table.to_pandas().index, df.index)
@pytest.mark.parametrize("index", ["a", ["a", "b"]])
def test_to_pandas_types_mapper_index(index):
- if Version(pd.__version__) < Version("1.5.0"):
- pytest.skip("ArrowDtype missing")
df = pd.DataFrame(
{
"a": [1, 2],
@@ -4683,9 +4606,6 @@ def test_to_pandas_types_mapper_index(index):
def test_array_to_pandas_types_mapper():
# https://issues.apache.org/jira/browse/ARROW-9664
- if Version(pd.__version__) < Version("1.2.0"):
- pytest.skip("Float64Dtype extension dtype missing")
-
data = pa.array([1, 2, 3], pa.int64())
# Test with mapper function
@@ -4707,9 +4627,6 @@ def test_array_to_pandas_types_mapper():
@pytest.mark.pandas
def test_chunked_array_to_pandas_types_mapper():
# https://issues.apache.org/jira/browse/ARROW-9664
- if Version(pd.__version__) < Version("1.2.0"):
- pytest.skip("Float64Dtype extension dtype missing")
-
data = pa.chunked_array([pa.array([1, 2, 3], pa.int64())])
assert isinstance(data, pa.ChunkedArray)
diff --git a/python/pyarrow/tests/test_schema.py
b/python/pyarrow/tests/test_schema.py
index 029e14ca16..db44ad3211 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -27,12 +27,6 @@ except ImportError:
import pyarrow as pa
import pyarrow.tests.util as test_util
-from pyarrow.vendored.version import Version
-
-try:
- import pandas as pd
-except ImportError:
- pass
def test_schema_constructor_errors():
@@ -55,8 +49,6 @@ def test_type_integers():
@pytest.mark.pandas
def test_type_to_pandas_dtype():
M8 = np.dtype('datetime64[ms]')
- if Version(pd.__version__) < Version("2.0.0"):
- M8 = np.dtype('datetime64[ns]')
cases = [
(pa.null(), np.object_),
(pa.bool_(), np.bool_),
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index ec1a5a2ba9..8eea34224a 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -186,9 +186,7 @@ def _get_pandas_tz_type(arrow_type, coerce_to_ns=False):
def _to_pandas_dtype(arrow_type, options=None):
- coerce_to_ns = (options and options.get('coerce_temporal_nanoseconds',
False)) or (
- _pandas_api.is_v1() and arrow_type.id in
- [_Type_DATE32, _Type_DATE64, _Type_TIMESTAMP, _Type_DURATION])
+ coerce_to_ns = bool(options and options.get('coerce_temporal_nanoseconds',
False))
if getattr(arrow_type, 'tz', None):
dtype = _get_pandas_tz_type(arrow_type, coerce_to_ns)
