This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 56132f95cec5 [SPARK-58919][PYTHON][TEST] Add tests for
pa.Schema.from_pandas
56132f95cec5 is described below
commit 56132f95cec5e25427acde5baa39bb44e91edbd2
Author: Spenser Sun <[email protected]>
AuthorDate: Fri Aug 21 08:53:19 2026 +0800
[SPARK-58919][PYTHON][TEST] Add tests for pa.Schema.from_pandas
### What changes were proposed in this pull request?
Add golden-file tests for `pa.Schema.from_pandas` in
`test_pyarrow_dataframe_from_pandas.py`, alongside the existing
`RecordBatch.from_pandas` tests. The test sweeps `preserve_index`
(`None`/`False`/`True`) across frame shape and index kind (0-column,
single/multi-column, RangeIndex, named, unnamed, and MultiIndex), pinning each
inferred field's name, type, and nullability.
The index-aware input rendering (`_index_desc`/`_input_cell`) is lifted
from the RecordBatch class into the shared base (now a second consumer), and
`GoldenFileTestMixin` gains a `pa.Schema` repr (`repr_arrow_schema_value`) that
surfaces field nullability.
### Why are the changes needed?
Part of SPARK-54936 (monitor upstream library behavior changes). Spark
builds a `StructType` from `pa.Schema.from_pandas` at two sites that diverge on
`preserve_index`: classic `createDataFrame` passes `preserve_index=False`
(index dropped), while Spark Connect passes it bare/`None` (a named or
non-range index becomes an extra field). Both read each field's type and
nullability. Pinning this in a golden file catches an upstream behavior change
loudly instead of silently altering the i [...]
### Does this PR introduce _any_ user-facing change?
No. Test-only.
### How was this patch tested?
New golden-file tests, run without a Spark session. Validated across a
PyArrow 18-25 x pandas 2/3 sweep (16 combinations) with no version-specific
overrides.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 4.8)
Closes #58178 from Spenserrrr/schema-from-pandas-tests.
Authored-by: Spenser Sun <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/testing/goldenutils.py | 24 +++++
.../pyarrow/golden_pyarrow_schema_from_pandas.csv | 13 +++
.../pyarrow/golden_pyarrow_schema_from_pandas.md | 14 +++
.../pyarrow/test_pyarrow_dataframe_from_pandas.py | 106 +++++++++++++++++----
4 files changed, 141 insertions(+), 16 deletions(-)
diff --git a/python/pyspark/testing/goldenutils.py
b/python/pyspark/testing/goldenutils.py
index 8374d20eb422..df699a81b065 100644
--- a/python/pyspark/testing/goldenutils.py
+++ b/python/pyspark/testing/goldenutils.py
@@ -408,6 +408,27 @@ class GoldenFileTestMixin:
v_str, schema = cls._repr_arrow_columns(value, max_len)
return f"{v_str}@RecordBatch[{schema}]"
+ @classmethod
+ def repr_arrow_schema_value(cls, value: "pa.Schema", max_len: int = 32) ->
str:
+ """
+ Format a PyArrow Schema for golden file.
+
+ Renders each field as "name: type nullable=...". A Schema carries no
data, so
+ nullability is included (unlike the Table/RecordBatch schema string):
Spark reads
+ ``field.nullable`` to build its StructType, so a change there silently
alters the
+ inferred Spark schema.
+
+ Returns
+ -------
+ str
+ "[name: type nullable=True, ...]@Schema"
+ """
+ fields = [f"{f.name}: {cls.repr_type(f.type)} nullable={f.nullable}"
for f in value]
+ v_str = "[" + ", ".join(fields) + "]"
+ if max_len > 0:
+ v_str = v_str[:max_len]
+ return f"{v_str}@Schema"
+
@classmethod
def repr_pandas_value(cls, value: "pd.DataFrame", max_len: int = 32) ->
str:
"""
@@ -483,6 +504,7 @@ class GoldenFileTestMixin:
- PyArrow Array/ChunkedArray -> repr_arrow_value
- PyArrow Table -> repr_arrow_table_value
- PyArrow RecordBatch -> repr_arrow_record_batch_value
+ - PyArrow Schema -> repr_arrow_schema_value
- pandas DataFrame -> repr_pandas_value
- numpy ndarray -> repr_numpy_value
- Everything else -> repr_python_value
@@ -505,6 +527,8 @@ class GoldenFileTestMixin:
return cls.repr_arrow_table_value(value, max_len)
if have_pyarrow and isinstance(value, pa.RecordBatch):
return cls.repr_arrow_record_batch_value(value, max_len)
+ if have_pyarrow and isinstance(value, pa.Schema):
+ return cls.repr_arrow_schema_value(value, max_len)
if have_pandas and isinstance(value, pd.DataFrame):
return cls.repr_pandas_value(value, max_len)
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_schema_from_pandas.csv
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_schema_from_pandas.csv
new file mode 100644
index 000000000000..66f7bbaf691b
--- /dev/null
+++
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_schema_from_pandas.csv
@@ -0,0 +1,13 @@
+test case pandas dataframe preserve_index=None
preserve_index=False preserve_index=True
+0-columns:range-index {}@Dataframe[][index=RangeIndex[0:3:1]] []@Schema
[]@Schema [__index_level_0__: int64 nullable=True]@Schema
+0-columns:named-index {}@Dataframe[][index='idx':[100, 200, 300]] [idx:
int64 nullable=True]@Schema []@Schema [idx: int64
nullable=True]@Schema
+0-columns:unnamed-index {}@Dataframe[][index=None:[10, 20, 30]]
[__index_level_0__: int64 nullable=True]@Schema []@Schema
[__index_level_0__: int64 nullable=True]@Schema
+0-columns:empty {}@Dataframe[][index=RangeIndex[0:0:1]] []@Schema
[]@Schema [__index_level_0__: int64 nullable=True]@Schema
+single-column:range-index {'a': [1, 2, 3]}@Dataframe[a
int64][index=RangeIndex[0:3:1]] [a: int64 nullable=True]@Schema [a: int64
nullable=True]@Schema [a: int64 nullable=True, __index_level_0__: int64
nullable=True]@Schema
+single-column:named-index {'a': [1, 2, 3]}@Dataframe[a
int64][index='idx':[100, 200, 300]] [a: int64 nullable=True, idx: int64
nullable=True]@Schema [a: int64 nullable=True]@Schema [a: int64
nullable=True, idx: int64 nullable=True]@Schema
+single-column:unnamed-index {'a': [1, 2, 3]}@Dataframe[a
int64][index=None:[10, 20, 30]] [a: int64 nullable=True, __index_level_0__:
int64 nullable=True]@Schema [a: int64 nullable=True]@Schema [a: int64
nullable=True, __index_level_0__: int64 nullable=True]@Schema
+multi-column:standard {'i': [1, 2, 3], 'f': [1.5, 2.5, 3.5], 'b': [True,
False, True], 's': ['a', 'b', 'c'], 't': [Timestamp('2020-01-01 05:30:00'),
Timestamp('2020-01-01 05:30:00'), Timestamp('2020-01-01
05:30:00')]}@Dataframe[i int64, f float64, b bool, s object, t
datetime64[ns]][index=RangeIndex[0:3:1]] [i: int64 nullable=True, f: float64
nullable=True, b: bool nullable=True, s: string nullable=True, t: timestamp[ns]
nullable=True]@Schema [i: int64 nullable=True, f: float64 nullable= [...]
+multi-column:nullable {'f': [1.5, nan, 3.5], 'b': [True, None, False], 's':
['a', None, 'c'], 't': [Timestamp('2020-01-01 05:30:00'), NaT,
Timestamp('2020-01-01 05:30:00')]}@Dataframe[f float64, b object, s object, t
datetime64[ns]][index=RangeIndex[0:3:1]] [f: float64 nullable=True, b: bool
nullable=True, s: string nullable=True, t: timestamp[ns] nullable=True]@Schema
[f: float64 nullable=True, b: bool nullable=True, s: string nullable=True,
t: timestamp[ns] nullable=True]@Schema [f: fl [...]
+multi-column:no-rows {'i': [], 'f': [], 'b': [], 't': []}@Dataframe[i int64,
f float64, b bool, t datetime64[ns]][index=RangeIndex[0:0:1]] [i: int64
nullable=True, f: float64 nullable=True, b: bool nullable=True, t:
timestamp[ns] nullable=True]@Schema [i: int64 nullable=True, f: float64
nullable=True, b: bool nullable=True, t: timestamp[ns] nullable=True]@Schema
[i: int64 nullable=True, f: float64 nullable=True, b: bool nullable=True,
t: timestamp[ns] nullable=True, __index_level_0__: in [...]
+single-column:multiindex {'a': [1, 2, 3]}@Dataframe[a
int64][index=MultiIndex[names=['g', 'n']]] [a: int64 nullable=True, g: int64
nullable=True, n: int64 nullable=True]@Schema [a: int64 nullable=True]@Schema
[a: int64 nullable=True, g: int64 nullable=True, n: int64 nullable=True]@Schema
+single-column:multiindex-partial-name {'a': [1, 2]}@Dataframe[a
int64][index=MultiIndex[names=['g', None]]] [a: int64 nullable=True, g: int64
nullable=True, __index_level_1__: int64 nullable=True]@Schema [a: int64
nullable=True]@Schema [a: int64 nullable=True, g: int64 nullable=True,
__index_level_1__: int64 nullable=True]@Schema
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_schema_from_pandas.md
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_schema_from_pandas.md
new file mode 100644
index 000000000000..bcf6286bdae4
--- /dev/null
+++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_schema_from_pandas.md
@@ -0,0 +1,14 @@
+| test case | pandas dataframe
| preserve_index=None
|
preserve_index=False [...]
+|---------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|----------------------
[...]
+| 0-columns:range-index |
{}@Dataframe[][index=RangeIndex[0:3:1]]
| []@Schema
| []@Schema [...]
+| 0-columns:named-index | {}@Dataframe[][index='idx':[100,
200, 300]]
| [idx: int64 nullable=True]@Schema
|
[]@Schema [...]
+| 0-columns:unnamed-index | {}@Dataframe[][index=None:[10, 20,
30]]
| [__index_level_0__: int64 nullable=True]@Schema
|
[]@Schema [...]
+| 0-columns:empty |
{}@Dataframe[][index=RangeIndex[0:0:1]]
| []@Schema
| []@Schema [...]
+| single-column:range-index | {'a': [1, 2, 3]}@Dataframe[a
int64][index=RangeIndex[0:3:1]]
| [a: int64 nullable=True]@Schema
|
[a: int64 nullable=Tr [...]
+| single-column:named-index | {'a': [1, 2, 3]}@Dataframe[a
int64][index='idx':[100, 200, 300]]
| [a: int64 nullable=True, idx: int64 nullable=True]@Schema
|
[a: int64 nullable=Tr [...]
+| single-column:unnamed-index | {'a': [1, 2, 3]}@Dataframe[a
int64][index=None:[10, 20, 30]]
| [a: int64 nullable=True, __index_level_0__: int64
nullable=True]@Schema
| [a: int64 nullable=Tr [...]
+| multi-column:standard | {'i': [1, 2, 3], 'f': [1.5, 2.5,
3.5], 'b': [True, False, True], 's': ['a', 'b', 'c'], 't':
[Timestamp('2020-01-01 05:30:00'), Timestamp('2020-01-01 05:30:00'),
Timestamp('2020-01-01 05:30:00')]}@Dataframe[i int64, f float64, b bool, s
object, t datetime64[ns]][index=RangeIndex[0:3:1]] | [i: int64 nullable=True,
f: float64 nullable=True, b: bool nullable=True, s: string nullable=True, t:
timestamp[ns] nullable=True]@Schema | [i: int64 nullable=Tr [...]
+| multi-column:nullable | {'f': [1.5, nan, 3.5], 'b': [True,
None, False], 's': ['a', None, 'c'], 't': [Timestamp('2020-01-01 05:30:00'),
NaT, Timestamp('2020-01-01 05:30:00')]}@Dataframe[f float64, b object, s
object, t datetime64[ns]][index=RangeIndex[0:3:1]]
| [f: float64 nullable=True, b: bool nullable=True, s:
string nullable=True, t: timestamp[ns] nullable=True]@Schema
| [f: float64 nullable= [...]
+| multi-column:no-rows | {'i': [], 'f': [], 'b': [], 't':
[]}@Dataframe[i int64, f float64, b bool, t
datetime64[ns]][index=RangeIndex[0:0:1]]
| [i: int64 nullable=True, f:
float64 nullable=True, b: bool nullable=True, t: timestamp[ns]
nullable=True]@Schema | [i: int64 nullable=Tr [...]
+| single-column:multiindex | {'a': [1, 2, 3]}@Dataframe[a
int64][index=MultiIndex[names=['g', 'n']]]
| [a: int64 nullable=True, g: int64 nullable=True, n: int64
nullable=True]@Schema
| [a: int64 nullable=Tr [...]
+| single-column:multiindex-partial-name | {'a': [1, 2]}@Dataframe[a
int64][index=MultiIndex[names=['g', None]]]
| [a: int64 nullable=True, g: int64 nullable=True,
__index_level_1__: int64 nullable=True]@Schema
| [a: int64 nullable=Tr [...]
\ No newline at end of file
diff --git
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py
index 0b7af403fa07..b125e25d92d8 100644
---
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py
+++
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py
@@ -17,8 +17,8 @@
"""
Golden-file tests for the PyArrow ``from_pandas`` constructors that take a
whole pandas
-DataFrame: ``pa.RecordBatch.from_pandas`` (with ``pa.Table.from_pandas`` /
-``pa.Schema.from_pandas`` to follow in this file). These take a DataFrame,
unlike
+DataFrame: ``pa.RecordBatch.from_pandas`` and ``pa.Schema.from_pandas`` (with
+``pa.Table.from_pandas`` to follow in this file). These take a DataFrame,
unlike
``pa.Array.from_pandas`` which takes a Series (covered by
test_pyarrow_array_from_pandas_*).
Per-column type inference matches the Array tests, so these pin the
DataFrame-level
@@ -29,6 +29,13 @@ for RecordBatch -- num_rows preservation for a 0-column
DataFrame. Spark calls
the default ``preserve_index=None`` to carry num_rows via the index metadata
-- otherwise a
0-column relation loses its rows.
+``Schema.from_pandas`` is inspected to build a Spark schema, and the two prod
call sites
+diverge on ``preserve_index``: classic pandas/conversion.py:971 passes
``False`` (index
+dropped), Connect session.py:573 passes it bare/``None`` (a named or non-range
index becomes
+an extra field) -- so a named-index frame yields different field sets. Spark
reads each
+field's type AND nullability (conversion.py:989 / session.py:590), so the
schema test pins
+name/type/nullability across ``preserve_index``.
+
Regenerate with SPARK_GENERATE_GOLDEN_FILES=1.
"""
@@ -51,11 +58,25 @@ if have_pyarrow:
class _PyArrowFromPandasFrameTestBase(GoldenFileTestMixin, unittest.TestCase):
"""
- Shared machinery for the DataFrame-input ``from_pandas`` constructors
(RecordBatch
- here; Table/Schema as followups). Owns the source-frame inventory and
defines no
- ``test_*`` of its own.
+ Shared machinery for the DataFrame-input ``from_pandas`` constructors
(RecordBatch and
+ Schema here; Table as a followup). Owns the source-frame inventory and the
index-aware
+ input-cell rendering (both constructors depend on the index under
``preserve_index``);
+ defines no ``test_*`` of its own.
"""
+ @staticmethod
+ def _index_desc(index) -> str:
+ """Compact, deterministic description of a pandas index for the input
cell."""
+ if isinstance(index, pd.MultiIndex):
+ return f"MultiIndex[names={list(index.names)}]"
+ if isinstance(index, pd.RangeIndex):
+ return f"RangeIndex[{index.start}:{index.stop}:{index.step}]"
+ return f"{index.name!r}:{index.tolist()}"
+
+ def _input_cell(self, df) -> str:
+ """Input DataFrame repr, extended with its index (repr_value drops
it)."""
+ return f"{self.repr_value(df,
max_len=0)}[index={self._index_desc(df.index)}]"
+
def _build_source_frames(self):
"""Named pandas DataFrames covering shape x index-kind, plus a dtype
sample."""
dt = datetime.datetime(2020, 1, 1, 5, 30)
@@ -120,17 +141,6 @@ class _PyArrowFromPandasFrameTestBase(GoldenFileTestMixin,
unittest.TestCase):
class PyArrowRecordBatchFromPandasTests(_PyArrowFromPandasFrameTestBase):
"""Tests pa.RecordBatch.from_pandas() across preserve_index via golden
file comparison."""
- @staticmethod
- def _index_desc(index) -> str:
- """Compact, deterministic description of a pandas index for the input
cell."""
- if isinstance(index, pd.RangeIndex):
- return f"RangeIndex[{index.start}:{index.stop}:{index.step}]"
- return f"{index.name!r}:{index.tolist()}"
-
- def _input_cell(self, df) -> str:
- """Input DataFrame repr, extended with its index (repr_value drops
it)."""
- return f"{self.repr_value(df,
max_len=0)}[index={self._index_desc(df.index)}]"
-
def _from_pandas_cell(self, df, **kwargs) -> str:
"""
Convert ``df`` via RecordBatch.from_pandas(**kwargs) and append
num_rows -- the
@@ -173,6 +183,70 @@ class
PyArrowRecordBatchFromPandasTests(_PyArrowFromPandasFrameTestBase):
)
[email protected](
+ not have_pyarrow or not have_pandas,
+ pyarrow_requirement_message or pandas_requirement_message,
+)
+class PyArrowSchemaFromPandasTests(_PyArrowFromPandasFrameTestBase):
+ """Tests pa.Schema.from_pandas() across preserve_index via golden file
comparison."""
+
+ def _schema_source_frames(self):
+ """Shared frames plus two MultiIndex rows -- a MultiIndex has several
index levels,
+ each becoming its own field, so these pin multi-level index-to-field
naming at the
+ schema. Level values are integers (stable int64 on pandas 2 and 3;
strings drift)."""
+ frames = self._build_source_frames()
+ frames["single-column:multiindex"] = pd.DataFrame(
+ {"a": [1, 2, 3]},
+ index=pd.MultiIndex.from_tuples([(1, 10), (1, 20), (2, 30)],
names=["g", "n"]),
+ )
+ frames["single-column:multiindex-partial-name"] = pd.DataFrame(
+ {"a": [1, 2]},
+ index=pd.MultiIndex.from_tuples([(1, 10), (2, 20)], names=["g",
None]),
+ )
+ return frames
+
+ def _from_pandas_cell(self, df, **kwargs) -> str:
+ """
+ Infer the schema via Schema.from_pandas(**kwargs) and render its
fields with
+ nullability -- the name/type/nullable Spark reads to build its
StructType. Returns
+ ERR@<ExceptionClass> if inference raises; a formatting error is a test
bug.
+ """
+ try:
+ schema = pa.Schema.from_pandas(df, **kwargs)
+ except Exception as e:
+ return f"ERR@{type(e).__name__}"
+ return self.repr_value(schema, max_len=0)
+
+ def test_from_pandas(self):
+ """Test pa.Schema.from_pandas() across preserve_index against golden
file."""
+ sources = self._schema_source_frames()
+ row_names = list(sources.keys())
+ preserve = {
+ "preserve_index=None": None,
+ "preserve_index=False": False,
+ "preserve_index=True": True,
+ }
+ col_names = ["pandas dataframe", *preserve.keys()]
+
+ # Version-specific expected values go here, keyed by (row, col), for
known drift.
+ overrides: dict[tuple[str, str], str] = {}
+
+ def compute_cell(row_name, col_name):
+ df = sources[row_name]
+ if col_name == "pandas dataframe":
+ return self._input_cell(df)
+ return self._from_pandas_cell(df,
preserve_index=preserve[col_name])
+
+ self.compare_or_generate_golden_matrix(
+ row_names=row_names,
+ col_names=col_names,
+ compute_cell=compute_cell,
+ golden_file_prefix="golden_pyarrow_schema_from_pandas",
+ index_name="test case",
+ overrides=overrides,
+ )
+
+
if __name__ == "__main__":
from pyspark.testing import main
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
