This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 007c647d494a [SPARK-58115][PYTHON][TEST] Handle Pandas 3 in PyArrow
Array.to_pandas golden test
007c647d494a is described below
commit 007c647d494a2e75b6a0cbe06aaae0feaa4ebdf9
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Wed Jul 15 08:56:34 2026 +0800
[SPARK-58115][PYTHON][TEST] Handle Pandas 3 in PyArrow Array.to_pandas
golden test
### What changes were proposed in this pull request?
Keep the existing PyArrow Array.to_pandas() golden files unchanged and
patch the expected string conversion results in memory when running with pandas
3. PyArrow 24-specific overrides cover empty string arrays, whose conversion
changed separately from non-empty arrays.
### Why are the changes needed?
Pandas 3 uses its dedicated string dtype when converting non-empty Arrow
string arrays, while PyArrow 24 extends that behavior to empty string arrays.
The shared golden expects Series[object], causing the pandas 3 CI job to fail
with six mismatches for string and large_string arrays.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- pandas 2.3.3 / PyArrow 23.0.1:
conda run -n spark-dev-313 env PYTHONPATH=python python -m unittest
pyspark.tests.upstream.pyarrow.test_pyarrow_arrow_to_pandas_default
- pandas 3.0.2 / PyArrow 23.0.1:
conda run -n spark-dev-313-p3 env PYTHONPATH=python python -m unittest
pyspark.tests.upstream.pyarrow.test_pyarrow_arrow_to_pandas_default
- pandas 3.0.3 / PyArrow 24.0.0 / NumPy 2.5.1, matching the failed CI job:
conda run -n spark-dev-313-p3 env PYTHONPATH=/tmp/spark-pandas3-ci:python
python -m unittest
pyspark.tests.upstream.pyarrow.test_pyarrow_arrow_to_pandas_default
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #57237 from zhengruifeng/python/pandas3-pyarrow-golden-dev3.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../test_pyarrow_arrow_to_pandas_default.py | 33 ++++++++++++++++++++--
1 file changed, 30 insertions(+), 3 deletions(-)
diff --git
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_default.py
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_default.py
index 8caac302f2e6..566d10a6e35d 100644
---
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_default.py
+++
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_default.py
@@ -44,10 +44,10 @@ Set SPARK_GENERATE_GOLDEN_FILES=1 before running:
SPARK_GENERATE_GOLDEN_FILES=1 python -m pytest \\
python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_default.py
-## PyArrow Version Compatibility
+## PyArrow and pandas Version Compatibility
-The golden files capture behavior for a specific PyArrow version.
-Regenerate when upgrading PyArrow, as to_pandas() behavior may change.
+The golden files capture behavior for specific PyArrow and pandas versions.
+Regenerate when upgrading either dependency, as to_pandas() behavior may
change.
"""
import datetime
@@ -57,6 +57,7 @@ import unittest
from decimal import Decimal
from typing import Callable, List, Optional
+from pyspark.loose_version import LooseVersion
from pyspark.testing.utils import (
have_pyarrow,
have_pandas,
@@ -67,6 +68,11 @@ from pyspark.testing.utils import (
)
from pyspark.testing.goldenutils import GoldenFileTestMixin
+if have_pandas:
+ import pandas as pd
+if have_pyarrow:
+ import pyarrow as pa
+
@unittest.skipIf(
not have_pyarrow or not have_pandas or not have_numpy,
@@ -388,6 +394,26 @@ class
PyArrowArrayToPandasDefaultTests(GoldenFileTestMixin, unittest.TestCase):
row_names = list(sources.keys())
col_names = ["pyarrow array", "pandas series"]
+ overrides = {}
+ # Pandas 3 uses its dedicated string dtype for non-empty Arrow string
arrays.
+ if LooseVersion(pd.__version__) >= LooseVersion("3.0.0"):
+ overrides.update(
+ {
+ ("string:standard", "pandas series"): ("['hello', 'world',
'']@Series[str]"),
+ ("string:nullable", "pandas series"): ("['hello', nan,
'world']@Series[str]"),
+ ("large_string:standard", "pandas series"): ("['hello',
'world']@Series[str]"),
+ ("large_string:nullable", "pandas series"): "['hello',
nan]@Series[str]",
+ }
+ )
+ # PyArrow 24 extends the pandas string dtype conversion to empty
arrays.
+ if LooseVersion(pa.__version__) >= LooseVersion("24.0.0"):
+ overrides.update(
+ {
+ ("string:empty", "pandas series"): "[]@Series[str]",
+ ("large_string:empty", "pandas series"):
"[]@Series[str]",
+ }
+ )
+
def compute_cell(row_name, col_name):
arr = sources[row_name]
if col_name == "pyarrow array":
@@ -405,6 +431,7 @@ class PyArrowArrayToPandasDefaultTests(GoldenFileTestMixin,
unittest.TestCase):
compute_cell=compute_cell,
golden_file_prefix="golden_pyarrow_arrow_to_pandas_default",
index_name="test case",
+ overrides=overrides,
)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
