This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new eab92f0ad2fc [SPARK-58765][PYTHON][TEST][FOLLOWUP] Add tests for
pa.Array.from_pandas with nested target types
eab92f0ad2fc is described below
commit eab92f0ad2fccaa92a500a9449bf2156cb260816
Author: Spenser Sun <[email protected]>
AuthorDate: Thu Aug 20 12:08:00 2026 +0800
[SPARK-58765][PYTHON][TEST][FOLLOWUP] Add tests for pa.Array.from_pandas
with nested target types
### What changes were proposed in this pull request?
Add `PyArrowArrayFromPandasTypeNestedTests` for
`pa.Array.from_pandas(series, type=..., safe=...)` with the nested target types
`to_arrow_type` builds (`list<element>`, `map`, `struct`), alongside the
existing scalar class. Four nested source rows are added to the shared
inventory (a child-overflow list and struct, and homogeneous dicts for the map
targets), so the default and mask goldens record their baseline.
### Why are the changes needed?
The nested `from_pandas` direction was uncovered. The pinned behavior:
`from_pandas` does not propagate `safe=` into a nested child, so an overflowing
child value raises under both `safe=True` and `safe=False`, whereas the same
overflow at the top level (the scalar tests) raises only under `safe=True`. The
safe and unsafe goldens therefore match; both are kept so a future pyarrow that
honors `safe=` in children fails loudly.
### Does this PR introduce any user-facing change?
No (test-only).
### How was this patch tested?
New golden tests. Swept pyarrow 18-25 x pandas 2/3: 16/16 pass, no
overrides.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 4.8)
Closes #58143 from Spenserrrr/from-pandas-type-nested-tests.
Authored-by: Spenser Sun <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../golden_pyarrow_array_from_pandas_default.csv | 4 +
.../golden_pyarrow_array_from_pandas_default.md | 4 +
.../golden_pyarrow_array_from_pandas_mask.csv | 4 +
.../golden_pyarrow_array_from_pandas_mask.md | 4 +
..._pyarrow_array_from_pandas_type_nested_safe.csv | 11 +++
...n_pyarrow_array_from_pandas_type_nested_safe.md | 12 +++
...yarrow_array_from_pandas_type_nested_unsafe.csv | 11 +++
...pyarrow_array_from_pandas_type_nested_unsafe.md | 12 +++
.../test_pyarrow_array_from_pandas_default.py | 8 ++
.../test_pyarrow_array_from_pandas_non_default.py | 97 ++++++++++++++++++++++
10 files changed, 167 insertions(+)
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_default.csv
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_default.csv
index 74a8cd3a3da2..7403a6618187 100644
---
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_default.csv
+++
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_default.csv
@@ -37,6 +37,10 @@ struct:standard [{'a': 1, 'b': 'x'}]@Series[object]
[[('a', 1), ('b', 'x')]]@str
struct:nullable [{'a': 1, 'b': 'x'}, None]@Series[object] [[('a',
1), ('b', 'x')], None]@struct<a: int64, b: string>
struct<struct>:standard [{'a': {'b': 1}}]@Series[object] [[('a',
{'b': 1})]]@struct<a: struct<b: int64>>
struct<list<int64>>:standard [{'a': [1, 2]}]@Series[object] [[('a', [1,
2])]]@struct<a: list<item: int64>>
+list<int64>:overflow [[300, 2], [3]]@Series[object] [[300, 2],
[3]]@list<item: int64>
+struct:overflow [{'a': 300, 'b': 'x'}]@Series[object] [[('a', 300),
('b', 'x')]]@struct<a: int64, b: string>
+struct<int64>:standard [{'a': 1, 'b': 2}]@Series[object] [[('a', 1),
('b', 2)]]@struct<a: int64, b: int64>
+struct<int64>:overflow [{'a': 300, 'b': 2}]@Series[object] [[('a', 300),
('b', 2)]]@struct<a: int64, b: int64>
datetime64[ns]:standard [Timestamp('2024-06-15
18:30:00')]@Series[datetime64[ns]] [2024-06-15 18:30:00]@timestamp[ns]
datetime64[ns]:nullable [Timestamp('2024-06-15 18:30:00'),
NaT]@Series[datetime64[ns]] [2024-06-15 18:30:00, None]@timestamp[ns]
datetime64[ns]:empty []@Series[datetime64[ns]] []@timestamp[ns]
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_default.md
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_default.md
index 0ca6395cdb11..a9acf237c129 100644
---
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_default.md
+++
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_default.md
@@ -38,6 +38,10 @@
| struct:nullable | [{'a': 1, 'b': 'x'}, None]@Series[object]
| [[('a', 1), ('b', 'x')],
None]@struct<a: int64, b: string> |
| struct<struct>:standard | [{'a': {'b': 1}}]@Series[object]
| [[('a', {'b': 1})]]@struct<a:
struct<b: int64>> |
| struct<list<int64>>:standard | [{'a': [1, 2]}]@Series[object]
| [[('a', [1, 2])]]@struct<a:
list<item: int64>> |
+| list<int64>:overflow | [[300, 2], [3]]@Series[object]
| [[300, 2], [3]]@list<item: int64>
|
+| struct:overflow | [{'a': 300, 'b': 'x'}]@Series[object]
| [[('a', 300), ('b',
'x')]]@struct<a: int64, b: string> |
+| struct<int64>:standard | [{'a': 1, 'b': 2}]@Series[object]
| [[('a', 1), ('b', 2)]]@struct<a:
int64, b: int64> |
+| struct<int64>:overflow | [{'a': 300, 'b': 2}]@Series[object]
| [[('a', 300), ('b', 2)]]@struct<a:
int64, b: int64> |
| datetime64[ns]:standard | [Timestamp('2024-06-15
18:30:00')]@Series[datetime64[ns]] | [2024-06-15
18:30:00]@timestamp[ns] |
| datetime64[ns]:nullable | [Timestamp('2024-06-15 18:30:00'),
NaT]@Series[datetime64[ns]] | [2024-06-15 18:30:00,
None]@timestamp[ns] |
| datetime64[ns]:empty | []@Series[datetime64[ns]]
| []@timestamp[ns]
|
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_mask.csv
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_mask.csv
index 9546f28e5c72..69debccb7824 100644
---
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_mask.csv
+++
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_mask.csv
@@ -37,6 +37,10 @@ struct:standard [{'a': 1, 'b': 'x'}]@Series[object]
[[('a', 1), ('b', 'x')]]@str
struct:nullable [{'a': 1, 'b': 'x'}, None]@Series[object] [[('a',
1), ('b', 'x')], None]@struct<a: int64, b: string> [[('a', 1), ('b',
'x')], None]@struct<a: int64, b: string>
struct<struct>:standard [{'a': {'b': 1}}]@Series[object] [[('a',
{'b': 1})]]@struct<a: struct<b: int64>> [[('a', {'b': 1})]]@struct<a: struct<b:
int64>>
struct<list<int64>>:standard [{'a': [1, 2]}]@Series[object] [[('a', [1,
2])]]@struct<a: list<item: int64>> [[('a', [1, 2])]]@struct<a: list<item:
int64>>
+list<int64>:overflow [[300, 2], [3]]@Series[object] [[300, 2],
[3]]@list<item: int64> [[300, 2], [3]]@list<item: int64>
+struct:overflow [{'a': 300, 'b': 'x'}]@Series[object] [[('a', 300),
('b', 'x')]]@struct<a: int64, b: string> [[('a', 300), ('b', 'x')]]@struct<a:
int64, b: string>
+struct<int64>:standard [{'a': 1, 'b': 2}]@Series[object] [[('a', 1),
('b', 2)]]@struct<a: int64, b: int64> [[('a', 1), ('b', 2)]]@struct<a:
int64, b: int64>
+struct<int64>:overflow [{'a': 300, 'b': 2}]@Series[object] [[('a', 300),
('b', 2)]]@struct<a: int64, b: int64> [[('a', 300), ('b', 2)]]@struct<a:
int64, b: int64>
datetime64[ns]:standard [Timestamp('2024-06-15
18:30:00')]@Series[datetime64[ns]] [2024-06-15 18:30:00]@timestamp[ns]
[2024-06-15 18:30:00]@timestamp[ns]
datetime64[ns]:nullable [Timestamp('2024-06-15 18:30:00'),
NaT]@Series[datetime64[ns]] [2024-06-15 18:30:00, None]@timestamp[ns]
[2024-06-15 18:30:00, None]@timestamp[ns]
datetime64[ns]:empty []@Series[datetime64[ns]] []@timestamp[ns]
[]@timestamp[ns]
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_mask.md
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_mask.md
index 1b01725a1854..b8085d64629e 100644
---
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_mask.md
+++
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_mask.md
@@ -38,6 +38,10 @@
| struct:nullable | [{'a': 1, 'b': 'x'}, None]@Series[object]
| [[('a', 1), ('b', 'x')],
None]@struct<a: int64, b: string> | [[('a', 1), ('b', 'x')],
None]@struct<a: int64, b: string> |
| struct<struct>:standard | [{'a': {'b': 1}}]@Series[object]
| [[('a', {'b': 1})]]@struct<a:
struct<b: int64>> | [[('a', {'b': 1})]]@struct<a: struct<b:
int64>> |
| struct<list<int64>>:standard | [{'a': [1, 2]}]@Series[object]
| [[('a', [1, 2])]]@struct<a:
list<item: int64>> | [[('a', [1, 2])]]@struct<a: list<item:
int64>> |
+| list<int64>:overflow | [[300, 2], [3]]@Series[object]
| [[300, 2], [3]]@list<item: int64>
| [[300, 2], [3]]@list<item: int64>
|
+| struct:overflow | [{'a': 300, 'b': 'x'}]@Series[object]
| [[('a', 300), ('b',
'x')]]@struct<a: int64, b: string> | [[('a', 300), ('b',
'x')]]@struct<a: int64, b: string> |
+| struct<int64>:standard | [{'a': 1, 'b': 2}]@Series[object]
| [[('a', 1), ('b', 2)]]@struct<a:
int64, b: int64> | [[('a', 1), ('b', 2)]]@struct<a: int64, b:
int64> |
+| struct<int64>:overflow | [{'a': 300, 'b': 2}]@Series[object]
| [[('a', 300), ('b', 2)]]@struct<a:
int64, b: int64> | [[('a', 300), ('b', 2)]]@struct<a: int64, b:
int64> |
| datetime64[ns]:standard | [Timestamp('2024-06-15
18:30:00')]@Series[datetime64[ns]] | [2024-06-15
18:30:00]@timestamp[ns] | [2024-06-15
18:30:00]@timestamp[ns] |
| datetime64[ns]:nullable | [Timestamp('2024-06-15 18:30:00'),
NaT]@Series[datetime64[ns]] | [2024-06-15 18:30:00,
None]@timestamp[ns] | [2024-06-15 18:30:00,
None]@timestamp[ns] |
| datetime64[ns]:empty | []@Series[datetime64[ns]]
| []@timestamp[ns]
| []@timestamp[ns]
|
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_nested_safe.csv
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_nested_safe.csv
new file mode 100644
index 000000000000..47e9f859dc1f
--- /dev/null
+++
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_nested_safe.csv
@@ -0,0 +1,11 @@
+source \ target list<element: int64> list<element: int8>
list<element: string> map<string, int64> map<string, int8>
struct<a: int64, b: string> struct<a: int8, b: string>
+list<int64>:standard [[1, 2], [3]]@list<element: int64> [[1, 2],
[3]]@list<element: int8> ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError
+list<int64>:overflow [[300, 2], [3]]@list<element: int64>
ERR@ArrowInvalid ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError
+list<int64>:nullable [[1, 2], None]@list<element: int64> [[1, 2],
None]@list<element: int8> ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError
+list<int64>:null-element [[1, None], [3]]@list<element: int64> [[1,
None], [3]]@list<element: int8> ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError
+list<string>:standard ERR@ArrowInvalid ERR@ArrowInvalid [['a',
'b'], ['c']]@list<element: string> ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError
+struct:standard ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowInvalid ERR@ArrowInvalid [[('a',
1), ('b', 'x')]]@struct<a: int64, b: string> [[('a', 1), ('b',
'x')]]@struct<a: int8, b: string>
+struct:overflow ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowInvalid ERR@ArrowInvalid [[('a',
300), ('b', 'x')]]@struct<a: int64, b: string> ERR@ArrowInvalid
+struct:nullable ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowInvalid ERR@ArrowInvalid [[('a',
1), ('b', 'x')], None]@struct<a: int64, b: string> [[('a', 1), ('b',
'x')], None]@struct<a: int8, b: string>
+struct<int64>:standard ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError [[('a', 1), ('b', 2)]]@map<string, int64> [[('a',
1), ('b', 2)]]@map<string, int8> ERR@ArrowTypeError
ERR@ArrowTypeError
+struct<int64>:overflow ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError [[('a', 300), ('b', 2)]]@map<string, int64>
ERR@ArrowInvalid ERR@ArrowTypeError ERR@ArrowInvalid
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_nested_safe.md
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_nested_safe.md
new file mode 100644
index 000000000000..398ce8fd2c9e
--- /dev/null
+++
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_nested_safe.md
@@ -0,0 +1,12 @@
+| source \ target | list<element: int64> |
list<element: int8> | list<element: string>
| map<string, int64> | map<string, int8>
| struct<a: int64, b: string> |
struct<a: int8, b: string> |
+|--------------------------|---------------------------------------|--------------------------------------|-------------------------------------------|---------------------------------------------|------------------------------------------|------------------------------------------------------------|-----------------------------------------------------------|
+| list<int64>:standard | [[1, 2], [3]]@list<element: int64> | [[1, 2],
[3]]@list<element: int8> | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowTypeError |
ERR@ArrowTypeError |
+| list<int64>:overflow | [[300, 2], [3]]@list<element: int64> |
ERR@ArrowInvalid | ERR@ArrowTypeError
| ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowTypeError |
ERR@ArrowTypeError |
+| list<int64>:nullable | [[1, 2], None]@list<element: int64> | [[1, 2],
None]@list<element: int8> | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowTypeError |
ERR@ArrowTypeError |
+| list<int64>:null-element | [[1, None], [3]]@list<element: int64> | [[1,
None], [3]]@list<element: int8> | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowTypeError |
ERR@ArrowTypeError |
+| list<string>:standard | ERR@ArrowInvalid |
ERR@ArrowInvalid | [['a', 'b'], ['c']]@list<element:
string> | ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowTypeError |
ERR@ArrowTypeError |
+| struct:standard | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowInvalid | ERR@ArrowInvalid
| [[('a', 1), ('b', 'x')]]@struct<a: int64, b: string> |
[[('a', 1), ('b', 'x')]]@struct<a: int8, b: string> |
+| struct:overflow | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowInvalid | ERR@ArrowInvalid
| [[('a', 300), ('b', 'x')]]@struct<a: int64, b: string> |
ERR@ArrowInvalid |
+| struct:nullable | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowInvalid | ERR@ArrowInvalid
| [[('a', 1), ('b', 'x')], None]@struct<a: int64, b: string> |
[[('a', 1), ('b', 'x')], None]@struct<a: int8, b: string> |
+| struct<int64>:standard | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| [[('a', 1), ('b', 2)]]@map<string, int64> | [[('a', 1), ('b',
2)]]@map<string, int8> | ERR@ArrowTypeError
| ERR@ArrowTypeError |
+| struct<int64>:overflow | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| [[('a', 300), ('b', 2)]]@map<string, int64> | ERR@ArrowInvalid
| ERR@ArrowTypeError |
ERR@ArrowInvalid |
\ No newline at end of file
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_nested_unsafe.csv
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_nested_unsafe.csv
new file mode 100644
index 000000000000..47e9f859dc1f
--- /dev/null
+++
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_nested_unsafe.csv
@@ -0,0 +1,11 @@
+source \ target list<element: int64> list<element: int8>
list<element: string> map<string, int64> map<string, int8>
struct<a: int64, b: string> struct<a: int8, b: string>
+list<int64>:standard [[1, 2], [3]]@list<element: int64> [[1, 2],
[3]]@list<element: int8> ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError
+list<int64>:overflow [[300, 2], [3]]@list<element: int64>
ERR@ArrowInvalid ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError
+list<int64>:nullable [[1, 2], None]@list<element: int64> [[1, 2],
None]@list<element: int8> ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError
+list<int64>:null-element [[1, None], [3]]@list<element: int64> [[1,
None], [3]]@list<element: int8> ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError
+list<string>:standard ERR@ArrowInvalid ERR@ArrowInvalid [['a',
'b'], ['c']]@list<element: string> ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowTypeError ERR@ArrowTypeError
+struct:standard ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowInvalid ERR@ArrowInvalid [[('a',
1), ('b', 'x')]]@struct<a: int64, b: string> [[('a', 1), ('b',
'x')]]@struct<a: int8, b: string>
+struct:overflow ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowInvalid ERR@ArrowInvalid [[('a',
300), ('b', 'x')]]@struct<a: int64, b: string> ERR@ArrowInvalid
+struct:nullable ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError ERR@ArrowInvalid ERR@ArrowInvalid [[('a',
1), ('b', 'x')], None]@struct<a: int64, b: string> [[('a', 1), ('b',
'x')], None]@struct<a: int8, b: string>
+struct<int64>:standard ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError [[('a', 1), ('b', 2)]]@map<string, int64> [[('a',
1), ('b', 2)]]@map<string, int8> ERR@ArrowTypeError
ERR@ArrowTypeError
+struct<int64>:overflow ERR@ArrowTypeError ERR@ArrowTypeError
ERR@ArrowTypeError [[('a', 300), ('b', 2)]]@map<string, int64>
ERR@ArrowInvalid ERR@ArrowTypeError ERR@ArrowInvalid
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_nested_unsafe.md
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_nested_unsafe.md
new file mode 100644
index 000000000000..398ce8fd2c9e
--- /dev/null
+++
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_array_from_pandas_type_nested_unsafe.md
@@ -0,0 +1,12 @@
+| source \ target | list<element: int64> |
list<element: int8> | list<element: string>
| map<string, int64> | map<string, int8>
| struct<a: int64, b: string> |
struct<a: int8, b: string> |
+|--------------------------|---------------------------------------|--------------------------------------|-------------------------------------------|---------------------------------------------|------------------------------------------|------------------------------------------------------------|-----------------------------------------------------------|
+| list<int64>:standard | [[1, 2], [3]]@list<element: int64> | [[1, 2],
[3]]@list<element: int8> | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowTypeError |
ERR@ArrowTypeError |
+| list<int64>:overflow | [[300, 2], [3]]@list<element: int64> |
ERR@ArrowInvalid | ERR@ArrowTypeError
| ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowTypeError |
ERR@ArrowTypeError |
+| list<int64>:nullable | [[1, 2], None]@list<element: int64> | [[1, 2],
None]@list<element: int8> | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowTypeError |
ERR@ArrowTypeError |
+| list<int64>:null-element | [[1, None], [3]]@list<element: int64> | [[1,
None], [3]]@list<element: int8> | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowTypeError |
ERR@ArrowTypeError |
+| list<string>:standard | ERR@ArrowInvalid |
ERR@ArrowInvalid | [['a', 'b'], ['c']]@list<element:
string> | ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowTypeError |
ERR@ArrowTypeError |
+| struct:standard | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowInvalid | ERR@ArrowInvalid
| [[('a', 1), ('b', 'x')]]@struct<a: int64, b: string> |
[[('a', 1), ('b', 'x')]]@struct<a: int8, b: string> |
+| struct:overflow | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowInvalid | ERR@ArrowInvalid
| [[('a', 300), ('b', 'x')]]@struct<a: int64, b: string> |
ERR@ArrowInvalid |
+| struct:nullable | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| ERR@ArrowInvalid | ERR@ArrowInvalid
| [[('a', 1), ('b', 'x')], None]@struct<a: int64, b: string> |
[[('a', 1), ('b', 'x')], None]@struct<a: int8, b: string> |
+| struct<int64>:standard | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| [[('a', 1), ('b', 2)]]@map<string, int64> | [[('a', 1), ('b',
2)]]@map<string, int8> | ERR@ArrowTypeError
| ERR@ArrowTypeError |
+| struct<int64>:overflow | ERR@ArrowTypeError |
ERR@ArrowTypeError | ERR@ArrowTypeError
| [[('a', 300), ('b', 2)]]@map<string, int64> | ERR@ArrowInvalid
| ERR@ArrowTypeError |
ERR@ArrowInvalid |
\ No newline at end of file
diff --git
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_from_pandas_default.py
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_from_pandas_default.py
index c3c2bb2b4f36..6b7db7ee8c18 100644
---
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_from_pandas_default.py
+++
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_from_pandas_default.py
@@ -187,6 +187,14 @@ class _PyArrowFromPandasTestBase(GoldenFileTestMixin,
unittest.TestCase):
sources["struct<struct>:standard"] = pd.Series([{"a": {"b": 1}}])
sources["struct<list<int64>>:standard"] = pd.Series([{"a": [1, 2]}])
+ # These feed the nested type tests (non_default): a child value that
overflows a
+ # narrower element type, and homogeneous-value dicts, which infer as
``struct`` but
+ # convert to ``map`` when the requested type asks for one.
+ sources["list<int64>:overflow"] = pd.Series([[300, 2], [3]])
+ sources["struct:overflow"] = pd.Series([{"a": 300, "b": "x"}])
+ sources["struct<int64>:standard"] = pd.Series([{"a": 1, "b": 2}])
+ sources["struct<int64>:overflow"] = pd.Series([{"a": 300, "b": 2}])
+
# =====================================================================
# Temporal types
# =====================================================================
diff --git
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_from_pandas_non_default.py
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_from_pandas_non_default.py
index 67191696207c..d9a2c88fc603 100644
---
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_from_pandas_non_default.py
+++
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_from_pandas_non_default.py
@@ -279,6 +279,103 @@ class PyArrowArrayFromPandasTypeScalarTests(
)
[email protected](
+ not have_pyarrow or not have_pandas or not have_numpy,
+ pyarrow_requirement_message or pandas_requirement_message or
numpy_requirement_message,
+)
+class PyArrowArrayFromPandasTypeNestedTests(
+ test_pyarrow_array_from_pandas_default._PyArrowFromPandasTestBase
+):
+ """
+ Tests pa.Array.from_pandas(series, type=..., safe=...) for NESTED target
types
+ (list / map / struct) via golden file comparison; scalar targets are
covered by
+ PyArrowArrayFromPandasTypeScalarTests.
+
+ Targets are the container types ``to_arrow_type`` builds, spelled to match
it
+ (``list<element: ...>``). The behavior recorded is that from_pandas does
NOT propagate
+ safe= into a nested child: an overflowing child value raises for both safe
settings,
+ whereas the same overflow at the top level (the scalar tests) raises only
under
+ safe=True. The safe and unsafe goldens therefore match; both are kept so
a future
+ pyarrow that honors safe= in children fails loudly. safe=True/False are
two methods and
+ two goldens; ``mask`` stays None.
+ """
+
+ @staticmethod
+ def _get_target_types():
+ """Nested to_arrow_type targets, each at a clean child type and a
narrower one it can
+ overflow, so the safe= non-propagation is observable against the
scalar goldens."""
+ return [
+ pa.list_(pa.field("element", pa.int64())),
+ pa.list_(pa.field("element", pa.int8())),
+ pa.list_(pa.field("element", pa.string())),
+ pa.map_(pa.string(), pa.int64()),
+ pa.map_(pa.string(), pa.int8()),
+ pa.struct([("a", pa.int64()), ("b", pa.string())]),
+ pa.struct([("a", pa.int8()), ("b", pa.string())]),
+ ]
+
+ def _type_source_arrays(self):
+ """
+ Nested source Series cherry-picked from the shared inventory: clean
list/struct rows,
+ a child-overflow row per container, and homogeneous dicts for the map
targets (a dict
+ Series infers as struct but converts to map when the type asks for
one).
+ """
+ pool = self._numpy_backed_sources()
+ selected = [
+ "list<int64>:standard",
+ "list<int64>:overflow",
+ "list<int64>:nullable",
+ "list<int64>:null-element",
+ "list<string>:standard",
+ "struct:standard",
+ "struct:overflow",
+ "struct:nullable",
+ "struct<int64>:standard",
+ "struct<int64>:overflow",
+ ]
+ return {name: pool[name] for name in selected}
+
+ def _compare_type_matrix(self, safe, golden_file_prefix, overrides):
+ sources = self._type_source_arrays()
+ target_types = self._get_target_types()
+ target_names = [self.repr_type(t) for t in target_types]
+ target_lookup = dict(zip(target_names, target_types))
+
+ self.compare_or_generate_golden_matrix(
+ row_names=list(sources.keys()),
+ col_names=target_names,
+ compute_cell=lambda src, tgt: self._from_pandas_cell(
+ sources[src], type=target_lookup[tgt], safe=safe
+ ),
+ golden_file_prefix=golden_file_prefix,
+ overrides=overrides,
+ )
+
+ def test_from_pandas_type_nested_safe(self):
+ """Test pa.Array.from_pandas(type=<nested>, safe=True) against golden
file."""
+ # No version-conditional cells: the nested sources are explicit
list/dict object
+ # Series (not inference-sensitive), so the matrix holds across the
pyarrow/pandas
+ # sweep. Entries would land here only if a future version changed a
cell.
+ overrides: dict[tuple[str, str], str] = {}
+ self._compare_type_matrix(
+ safe=True,
+
golden_file_prefix="golden_pyarrow_array_from_pandas_type_nested_safe",
+ overrides=overrides,
+ )
+
+ def test_from_pandas_type_nested_unsafe(self):
+ """Test pa.Array.from_pandas(type=<nested>, safe=False) against golden
file."""
+ # Same empty overrides as the safe method, and version-stable for the
same reason;
+ # since from_pandas does not propagate safe= into children, this
golden also matches
+ # the safe one.
+ overrides: dict[tuple[str, str], str] = {}
+ self._compare_type_matrix(
+ safe=False,
+
golden_file_prefix="golden_pyarrow_array_from_pandas_type_nested_unsafe",
+ overrides=overrides,
+ )
+
+
if __name__ == "__main__":
from pyspark.testing import main
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
