This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new bf9848d231a0 [SPARK-59133][PYTHON][TEST] Add tests for
pa.Table.from_pandas
bf9848d231a0 is described below
commit bf9848d231a07220fe7af669ab4083cd186b6fb9
Author: Spenser Sun <[email protected]>
AuthorDate: Tue Sep 1 11:43:04 2026 +0800
[SPARK-59133][PYTHON][TEST] Add tests for pa.Table.from_pandas
### What changes were proposed in this pull request?
Adds `PyArrowTableFromPandasTests` to
`test_pyarrow_dataframe_from_pandas.py`, covering `pa.Table.from_pandas` across
`preserve_index` and pinning each column's chunk count. This completes the
DataFrame-input `from_pandas` constructors alongside the existing RecordBatch
and Schema tests.
### Why are the changes needed?
A Table column is a `ChunkedArray`, so `Table.from_pandas` accepts a
chunked-backed pandas column, while `RecordBatch.from_pandas` rejects the same
input with `TypeError: Cannot convert ChunkedArray to Array`. Neither that
difference nor the chunk counts a Table preserves was pinned anywhere.
### Does this PR introduce _any_ user-facing change?
No. Test-only.
### How was this patch tested?
New golden-file test `PyArrowTableFromPandasTests.test_from_pandas`,
verified across pyarrow 18-25 x pandas 2/3 with no version overrides needed.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 5)
Closes #58433 from Spenserrrr/table-from-pandas-tests.
Authored-by: Spenser Sun <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../pyarrow/golden_pyarrow_table_from_pandas.csv | 16 +++
.../pyarrow/golden_pyarrow_table_from_pandas.md | 17 +++
.../pyarrow/test_pyarrow_dataframe_from_pandas.py | 116 +++++++++++++++++++--
3 files changed, 142 insertions(+), 7 deletions(-)
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_from_pandas.csv
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_from_pandas.csv
new file mode 100644
index 000000000000..d5490e584fd6
--- /dev/null
+++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_from_pandas.csv
@@ -0,0 +1,16 @@
+test case pandas dataframe preserve_index=None
preserve_index=False preserve_index=True
+0-columns:range-index {}@Dataframe[][index=RangeIndex[0:3:1]]
{}@Table[][num_rows=3, num_chunks={}] {}@Table[][num_rows=0, num_chunks={}]
{__index_level_0__: [0, 1, 2]}@Table[__index_level_0__: int64][num_rows=3,
num_chunks={'__index_level_0__': 1}]
+0-columns:named-index {}@Dataframe[][index='idx':[100, 200, 300]] {idx:
[100, 200, 300]}@Table[idx: int64][num_rows=3, num_chunks={'idx': 1}]
{}@Table[][num_rows=0, num_chunks={}] {idx: [100, 200, 300]}@Table[idx:
int64][num_rows=3, num_chunks={'idx': 1}]
+0-columns:unnamed-index {}@Dataframe[][index=None:[10, 20, 30]]
{__index_level_0__: [10, 20, 30]}@Table[__index_level_0__: int64][num_rows=3,
num_chunks={'__index_level_0__': 1}] {}@Table[][num_rows=0, num_chunks={}]
{__index_level_0__: [10, 20, 30]}@Table[__index_level_0__: int64][num_rows=3,
num_chunks={'__index_level_0__': 1}]
+0-columns:empty {}@Dataframe[][index=RangeIndex[0:0:1]]
{}@Table[][num_rows=0, num_chunks={}] {}@Table[][num_rows=0, num_chunks={}]
{__index_level_0__: []}@Table[__index_level_0__: int64][num_rows=0,
num_chunks={'__index_level_0__': 1}]
+single-column:range-index {'a': [1, 2, 3]}@Dataframe[a
int64][index=RangeIndex[0:3:1]] {a: [1, 2, 3]}@Table[a: int64][num_rows=3,
num_chunks={'a': 1}] {a: [1, 2, 3]}@Table[a: int64][num_rows=3,
num_chunks={'a': 1}] {a: [1, 2, 3], __index_level_0__: [0, 1, 2]}@Table[a:
int64, __index_level_0__: int64][num_rows=3, num_chunks={'a': 1,
'__index_level_0__': 1}]
+single-column:named-index {'a': [1, 2, 3]}@Dataframe[a
int64][index='idx':[100, 200, 300]] {a: [1, 2, 3], idx: [100, 200,
300]}@Table[a: int64, idx: int64][num_rows=3, num_chunks={'a': 1, 'idx': 1}]
{a: [1, 2, 3]}@Table[a: int64][num_rows=3, num_chunks={'a': 1}] {a: [1, 2, 3],
idx: [100, 200, 300]}@Table[a: int64, idx: int64][num_rows=3, num_chunks={'a':
1, 'idx': 1}]
+single-column:unnamed-index {'a': [1, 2, 3]}@Dataframe[a
int64][index=None:[10, 20, 30]] {a: [1, 2, 3], __index_level_0__: [10, 20,
30]}@Table[a: int64, __index_level_0__: int64][num_rows=3, num_chunks={'a': 1,
'__index_level_0__': 1}] {a: [1, 2, 3]}@Table[a: int64][num_rows=3,
num_chunks={'a': 1}] {a: [1, 2, 3], __index_level_0__: [10, 20, 30]}@Table[a:
int64, __index_level_0__: int64][num_rows=3, num_chunks={'a': 1,
'__index_level_0__': 1}]
+multi-column:standard {'i': [1, 2, 3], 'f': [1.5, 2.5, 3.5], 'b': [True,
False, True], 's': ['a', 'b', 'c'], 't': [Timestamp('2020-01-01 05:30:00'),
Timestamp('2020-01-01 05:30:00'), Timestamp('2020-01-01
05:30:00')]}@Dataframe[i int64, f float64, b bool, s object, t
datetime64[ns]][index=RangeIndex[0:3:1]] {i: [1, 2, 3], f: [1.5, 2.5, 3.5], b:
[True, False, True], s: [a, b, c], t: [2020-01-01 05:30:00, 2020-01-01
05:30:00, 2020-01-01 05:30:00]}@Table[i: int64, f: float64, b: bool, s: st [...]
+multi-column:nullable {'f': [1.5, nan, 3.5], 'b': [True, None, False], 's':
['a', None, 'c'], 't': [Timestamp('2020-01-01 05:30:00'), NaT,
Timestamp('2020-01-01 05:30:00')]}@Dataframe[f float64, b object, s object, t
datetime64[ns]][index=RangeIndex[0:3:1]] {f: [1.5, None, 3.5], b: [True,
None, False], s: [a, None, c], t: [2020-01-01 05:30:00, None, 2020-01-01
05:30:00]}@Table[f: float64, b: bool, s: string, t: timestamp[ns]][num_rows=3,
num_chunks={'f': 1, 'b': 1, 's': 1, 't': 1}] {f: [ [...]
+multi-column:no-rows {'i': [], 'f': [], 'b': [], 't': []}@Dataframe[i int64,
f float64, b bool, t datetime64[ns]][index=RangeIndex[0:0:1]] {i: [], f: [],
b: [], t: []}@Table[i: int64, f: float64, b: bool, t:
timestamp[ns]][num_rows=0, num_chunks={'i': 1, 'f': 1, 'b': 1, 't': 1}] {i:
[], f: [], b: [], t: []}@Table[i: int64, f: float64, b: bool, t:
timestamp[ns]][num_rows=0, num_chunks={'i': 1, 'f': 1, 'b': 1, 't': 1}] {i:
[], f: [], b: [], t: [], __index_level_0__: []}@Table[i: int64, f: [...]
+single-column:arrow-backed {'a': [1, 2, 3]}@Dataframe[a
int64[pyarrow]][index=RangeIndex[0:3:1]][chunks={'a': 1}] {a: [1, 2,
3]}@Table[a: int64][num_rows=3, num_chunks={'a': 1}] {a: [1, 2, 3]}@Table[a:
int64][num_rows=3, num_chunks={'a': 1}] {a: [1, 2, 3], __index_level_0__: [0,
1, 2]}@Table[a: int64, __index_level_0__: int64][num_rows=3, num_chunks={'a':
1, '__index_level_0__': 1}]
+single-column:chunked {'a': [1, 2, 3]}@Dataframe[a
int64[pyarrow]][index=RangeIndex[0:3:1]][chunks={'a': 2}] {a: [1, 2,
3]}@Table[a: int64][num_rows=3, num_chunks={'a': 2}] {a: [1, 2, 3]}@Table[a:
int64][num_rows=3, num_chunks={'a': 2}] {a: [1, 2, 3], __index_level_0__: [0,
1, 2]}@Table[a: int64, __index_level_0__: int64][num_rows=3, num_chunks={'a':
2, '__index_level_0__': 1}]
+multi-column:chunked-uneven {'a': [1, 2, 3], 'b': [1, 2, 3], 'c': [7, 8,
9]}@Dataframe[a int64[pyarrow], b int64[pyarrow], c
int64][index=RangeIndex[0:3:1]][chunks={'a': 2, 'b': 3}] {a: [1, 2, 3],
b: [1, 2, 3], c: [7, 8, 9]}@Table[a: int64, b: int64, c: int64][num_rows=3,
num_chunks={'a': 2, 'b': 3, 'c': 1}] {a: [1, 2, 3], b: [1, 2, 3], c: [7, 8,
9]}@Table[a: int64, b: int64, c: int64][num_rows=3, num_chunks={'a': 2, 'b': 3,
'c': 1}] {a: [1, 2, 3], b: [1, 2, 3], c: [7, 8, 9], __index_lev [...]
+single-column:chunked-empty-chunk {'a': [1, 2, 3]}@Dataframe[a
int64[pyarrow]][index=RangeIndex[0:3:1]][chunks={'a': 3}] {a: [1, 2,
3]}@Table[a: int64][num_rows=3, num_chunks={'a': 3}] {a: [1, 2, 3]}@Table[a:
int64][num_rows=3, num_chunks={'a': 3}] {a: [1, 2, 3], __index_level_0__: [0,
1, 2]}@Table[a: int64, __index_level_0__: int64][num_rows=3, num_chunks={'a':
3, '__index_level_0__': 1}]
+single-column:chunked-string {'a': ['a', 'b', 'c']}@Dataframe[a
string[pyarrow]][index=RangeIndex[0:3:1]][chunks={'a': 2}] {a: [a, b,
c]}@Table[a: string][num_rows=3, num_chunks={'a': 2}] {a: [a, b,
c]}@Table[a: string][num_rows=3, num_chunks={'a': 2}] {a: [a, b, c],
__index_level_0__: [0, 1, 2]}@Table[a: string, __index_level_0__:
int64][num_rows=3, num_chunks={'a': 2, '__index_level_0__': 1}]
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_from_pandas.md
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_from_pandas.md
new file mode 100644
index 000000000000..ca18e314cc5c
--- /dev/null
+++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_from_pandas.md
@@ -0,0 +1,17 @@
+| test case | pandas dataframe
| preserve_index=None
[...]
+|-----------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------
[...]
+| 0-columns:range-index | {}@Dataframe[][index=RangeIndex[0:3:1]]
| {}@Table[][num_rows=3, num_chunks={}]
[...]
+| 0-columns:named-index | {}@Dataframe[][index='idx':[100, 200,
300]]
| {idx: [100, 200, 300]}@Table[idx: int64][num_rows=3,
num_chunks={'idx': 1}]
[...]
+| 0-columns:unnamed-index | {}@Dataframe[][index=None:[10, 20, 30]]
| {__index_level_0__: [10, 20, 30]}@Table[__index_level_0__:
int64][num_rows=3, num_chunks={'__index_level_0__': 1}]
[...]
+| 0-columns:empty | {}@Dataframe[][index=RangeIndex[0:0:1]]
| {}@Table[][num_rows=0, num_chunks={}]
[...]
+| single-column:range-index | {'a': [1, 2, 3]}@Dataframe[a
int64][index=RangeIndex[0:3:1]]
| {a: [1, 2, 3]}@Table[a: int64][num_rows=3, num_chunks={'a':
1}]
[...]
+| single-column:named-index | {'a': [1, 2, 3]}@Dataframe[a
int64][index='idx':[100, 200, 300]]
| {a: [1, 2, 3], idx: [100, 200, 300]}@Table[a: int64, idx:
int64][num_rows=3, num_chunks={'a': 1, 'idx': 1}]
[...]
+| single-column:unnamed-index | {'a': [1, 2, 3]}@Dataframe[a
int64][index=None:[10, 20, 30]]
| {a: [1, 2, 3], __index_level_0__: [10, 20, 30]}@Table[a:
int64, __index_level_0__: int64][num_rows=3, num_chunks={'a': 1,
'__index_level_0__': 1}] [...]
+| multi-column:standard | {'i': [1, 2, 3], 'f': [1.5, 2.5, 3.5],
'b': [True, False, True], 's': ['a', 'b', 'c'], 't': [Timestamp('2020-01-01
05:30:00'), Timestamp('2020-01-01 05:30:00'), Timestamp('2020-01-01
05:30:00')]}@Dataframe[i int64, f float64, b bool, s object, t
datetime64[ns]][index=RangeIndex[0:3:1]] | {i: [1, 2, 3], f: [1.5, 2.5, 3.5],
b: [True, False, True], s: [a, b, c], t: [2020-01-01 05:30:00, 2020-01-01
05:30:00, 2020-01-01 05:30:00]}@Table[i: int64, f: float [...]
+| multi-column:nullable | {'f': [1.5, nan, 3.5], 'b': [True, None,
False], 's': ['a', None, 'c'], 't': [Timestamp('2020-01-01 05:30:00'), NaT,
Timestamp('2020-01-01 05:30:00')]}@Dataframe[f float64, b object, s object, t
datetime64[ns]][index=RangeIndex[0:3:1]]
| {f: [1.5, None, 3.5], b: [True, None, False], s: [a, None, c], t:
[2020-01-01 05:30:00, None, 2020-01-01 05:30:00]}@Table[f: float64, b: bool, s:
string, t: timestam [...]
+| multi-column:no-rows | {'i': [], 'f': [], 'b': [], 't':
[]}@Dataframe[i int64, f float64, b bool, t
datetime64[ns]][index=RangeIndex[0:0:1]]
| {i: [], f: [], b: [], t:
[]}@Table[i: int64, f: float64, b: bool, t: timestamp[ns]][num_rows=0,
num_chunks={'i': 1, 'f': 1, 'b': 1, 't': 1}] [...]
+| single-column:arrow-backed | {'a': [1, 2, 3]}@Dataframe[a
int64[pyarrow]][index=RangeIndex[0:3:1]][chunks={'a': 1}]
| {a: [1, 2, 3]}@Table[a: int64][num_rows=3, num_chunks={'a':
1}]
[...]
+| single-column:chunked | {'a': [1, 2, 3]}@Dataframe[a
int64[pyarrow]][index=RangeIndex[0:3:1]][chunks={'a': 2}]
| {a: [1, 2, 3]}@Table[a: int64][num_rows=3, num_chunks={'a':
2}]
[...]
+| multi-column:chunked-uneven | {'a': [1, 2, 3], 'b': [1, 2, 3], 'c':
[7, 8, 9]}@Dataframe[a int64[pyarrow], b int64[pyarrow], c
int64][index=RangeIndex[0:3:1]][chunks={'a': 2, 'b': 3}]
| {a: [1, 2, 3], b: [1, 2, 3], c: [7, 8,
9]}@Table[a: int64, b: int64, c: int64][num_rows=3, num_chunks={'a': 2, 'b': 3,
'c': 1}] [...]
+| single-column:chunked-empty-chunk | {'a': [1, 2, 3]}@Dataframe[a
int64[pyarrow]][index=RangeIndex[0:3:1]][chunks={'a': 3}]
| {a: [1, 2, 3]}@Table[a: int64][num_rows=3, num_chunks={'a':
3}]
[...]
+| single-column:chunked-string | {'a': ['a', 'b', 'c']}@Dataframe[a
string[pyarrow]][index=RangeIndex[0:3:1]][chunks={'a': 2}]
| {a: [a, b, c]}@Table[a: string][num_rows=3, num_chunks={'a': 2}]
[...]
\ No newline at end of file
diff --git
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py
index 72c5fdcc097f..75540677b465 100644
---
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py
+++
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_dataframe_from_pandas.py
@@ -17,9 +17,9 @@
"""
Golden-file tests for the PyArrow ``from_pandas`` constructors that take a
whole pandas
-DataFrame: ``pa.RecordBatch.from_pandas`` and ``pa.Schema.from_pandas`` (with
-``pa.Table.from_pandas`` to follow in this file). These take a DataFrame,
unlike
-``pa.Array.from_pandas`` which takes a Series (covered by
test_pyarrow_array_from_pandas_*).
+DataFrame: ``pa.RecordBatch.from_pandas``, ``pa.Schema.from_pandas`` and
+``pa.Table.from_pandas``. These take a DataFrame, unlike
``pa.Array.from_pandas`` which
+takes a Series (covered by test_pyarrow_array_from_pandas_*).
Per-column type inference matches the Array tests, so these pin the
DataFrame-level
behavior instead: whole-frame assembly, the pandas index under
``preserve_index``, and --
@@ -36,6 +36,10 @@ an extra field) -- so a named-index frame yields different
field sets. Spark rea
field's type AND nullability (conversion.py:989 / session.py:590), so the
schema test pins
name/type/nullability across ``preserve_index``.
+``Table.from_pandas`` has no production call site and is covered for
completeness. A Table
+column is a ChunkedArray, so unlike RecordBatch it accepts a chunked-backed
column (which
+RecordBatch rejects with TypeError, SPARK-58625); the Table test also pins
chunk counts.
+
Regenerate with SPARK_GENERATE_GOLDEN_FILES=1.
"""
@@ -58,10 +62,10 @@ if have_pyarrow:
class _PyArrowFromPandasFrameTestBase(GoldenFileTestMixin, unittest.TestCase):
"""
- Shared machinery for the DataFrame-input ``from_pandas`` constructors
(RecordBatch and
- Schema here; Table as a followup). Owns the source-frame inventory and the
index-aware
- input-cell rendering (both constructors depend on the index under
``preserve_index``);
- defines no ``test_*`` of its own.
+ Shared machinery for the DataFrame-input ``from_pandas`` constructors
(RecordBatch,
+ Schema and Table). Owns the source-frame inventory and the index-aware
input-cell
+ rendering (all three depend on the index under ``preserve_index``);
defines no
+ ``test_*`` of its own.
"""
@staticmethod
@@ -247,6 +251,104 @@ class
PyArrowSchemaFromPandasTests(_PyArrowFromPandasFrameTestBase):
)
[email protected](
+ not have_pyarrow or not have_pandas,
+ pyarrow_requirement_message or pandas_requirement_message,
+)
+class PyArrowTableFromPandasTests(_PyArrowFromPandasFrameTestBase):
+ """Tests pa.Table.from_pandas() across preserve_index via golden file
comparison."""
+
+ def _table_source_frames(self):
+ """Shared frames plus arrow-backed rows: a Table column is a
ChunkedArray, so it
+ accepts the multi-chunk input RecordBatch.from_pandas rejects."""
+ frames = self._build_source_frames()
+
+ def chunked(*chunks, arrow_type=pa.int64()):
+ stored = pa.chunked_array(list(chunks), type=arrow_type)
+ return pd.arrays.ArrowExtensionArray(stored)
+
+ # Arrow storage always has a chunk count, even at 1; the numpy rows
have none.
+ frames["single-column:arrow-backed"] = pd.DataFrame(
+ {"a": pd.Series([1, 2, 3], dtype="int64[pyarrow]")}
+ )
+ frames["single-column:chunked"] = pd.DataFrame({"a": chunked([1, 2],
[3])})
+ # Chunk counts are per column, and a plain numpy column arrives as a
single chunk.
+ frames["multi-column:chunked-uneven"] = pd.DataFrame(
+ {
+ "a": chunked([1, 2], [3]),
+ "b": chunked([1], [2], [3]),
+ "c": pd.Series([7, 8, 9], dtype="int64"),
+ }
+ )
+ # An empty chunk is carried through rather than compacted away.
+ frames["single-column:chunked-empty-chunk"] = pd.DataFrame({"a":
chunked([1, 2], [], [3])})
+ frames["single-column:chunked-string"] = pd.DataFrame(
+ {"a": chunked(["a"], ["b", "c"], arrow_type=pa.string())}
+ )
+ return frames
+
+ def _input_cell(self, df) -> str:
+ """Base input cell plus source chunk counts -- the repr renders a
2-chunk and a
+ 1-chunk column identically. Only ChunkedArray-backed columns have a
count
+ (``Int64`` exports a plain Array), so the suffix is omitted when none
do."""
+ source_chunks = {}
+ for name in df.columns:
+ array = df[name].array
+ if not hasattr(array, "__arrow_array__"):
+ continue
+ stored = array.__arrow_array__()
+ if isinstance(stored, pa.ChunkedArray):
+ source_chunks[name] = stored.num_chunks
+ suffix = f"[chunks={source_chunks}]" if source_chunks else ""
+ return f"{super()._input_cell(df)}{suffix}"
+
+ def _from_pandas_cell(self, df, **kwargs) -> str:
+ """Convert ``df`` via Table.from_pandas(**kwargs), appending num_rows
and per-column
+ chunk counts -- the repr shows neither. Returns ERR@<ExceptionClass>
if the
+ conversion raises."""
+ try:
+ table = pa.Table.from_pandas(df, **kwargs)
+ except Exception as e:
+ return f"ERR@{type(e).__name__}"
+ # Every Table column is a ChunkedArray, so this always applies.
+ result_chunks = {
+ name: col.num_chunks for name, col in zip(table.schema.names,
table.columns)
+ }
+ return (
+ f"{self.repr_value(table, max_len=0)}"
+ f"[num_rows={table.num_rows}, num_chunks={result_chunks}]"
+ )
+
+ def test_from_pandas(self):
+ """Test pa.Table.from_pandas() across preserve_index against golden
file."""
+ sources = self._table_source_frames()
+ row_names = list(sources.keys())
+ preserve = {
+ "preserve_index=None": None,
+ "preserve_index=False": False,
+ "preserve_index=True": True,
+ }
+ col_names = ["pandas dataframe", *preserve.keys()]
+
+ # Version-specific expected values go here, keyed by (row, col), for
known drift.
+ overrides: dict[tuple[str, str], str] = {}
+
+ def compute_cell(row_name, col_name):
+ df = sources[row_name]
+ if col_name == "pandas dataframe":
+ return self._input_cell(df)
+ return self._from_pandas_cell(df,
preserve_index=preserve[col_name])
+
+ self.compare_or_generate_golden_matrix(
+ row_names=row_names,
+ col_names=col_names,
+ compute_cell=compute_cell,
+ golden_file_prefix="golden_pyarrow_table_from_pandas",
+ index_name="test case",
+ overrides=overrides,
+ )
+
+
if __name__ == "__main__":
from pyspark.testing import main
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
