This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 3cd0205b5c47 [SPARK-58829][PYTHON][TEST] Add tests for pa.Table.cast
3cd0205b5c47 is described below
commit 3cd0205b5c47b4d66453f812e39339a0c41af273
Author: Spenser Sun <[email protected]>
AuthorDate: Thu Aug 20 16:01:46 2026 +0800
[SPARK-58829][PYTHON][TEST] Add tests for pa.Table.cast
### What changes were proposed in this pull request?
Add golden-file tests for `pa.Table.cast(target_schema)` under the
SPARK-54936 umbrella (monitor upstream PyArrow/pandas behavior). New file
`python/pyspark/tests/upstream/pyarrow/test_pyarrow_table_cast.py` with
`PyArrowTableCastTests`, covering both `safe=True` (default) and `safe=False`
via separate golden files, plus a `dev/sparktestsupport/modules.py`
registration.
### Why are the changes needed?
PySpark's conversion layer relies on `pa.Table.cast(schema)` in
`DataFrame.toArrow()` and `createDataFrame()` (both classic and Spark Connect).
Unlike `pa.Array.cast` (already covered by `test_pyarrow_array_cast.py`),
`Table.cast` pins genuinely Table-level behavior that the Array-level tests
cannot:
- multi-column, whole-schema casts;
- strict positional + name-equal column matching (a name mismatch, a pure
reorder, or a wrong field count all raise `ValueError` — which is why Spark
Connect pre-aligns names via `rename_columns()` before `cast()`);
- target-field nullability enforcement;
- temporal unit/timezone coercion;
- the empty-table edges (0 columns / 0 rows).
These tests fail loudly if an upstream version changes that behavior,
instead of PySpark silently returning wrong data.
### Does this PR introduce _any_ user-facing change?
No. Test-only.
### How was this patch tested?
New golden-file tests, run without a Spark session. Verified across PyArrow
18-25 x pandas 2/3 (16/16 combinations); regenerated goldens are byte-identical
and no version-specific overrides are needed (`Table.cast` is pandas-free).
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 4.8)
This pull request and its description were written by Isaac.
Closes #58056 from Spenserrrr/table-cast-tests.
Authored-by: Spenser Sun <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit ba82f4e3f8cefbceae5565aebac62760c1330f87)
Signed-off-by: Ruifeng Zheng <[email protected]>
---
dev/sparktestsupport/modules.py | 1 +
.../pyarrow/golden_pyarrow_table_cast_safe.csv | 21 ++
.../pyarrow/golden_pyarrow_table_cast_safe.md | 22 ++
.../pyarrow/golden_pyarrow_table_cast_unsafe.csv | 21 ++
.../pyarrow/golden_pyarrow_table_cast_unsafe.md | 22 ++
.../upstream/pyarrow/test_pyarrow_table_cast.py | 296 +++++++++++++++++++++
6 files changed, 383 insertions(+)
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index d5ef8e7b54ab..d097ad556531 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -545,6 +545,7 @@ pyspark_core = Module(
"pyspark.tests.upstream.pyarrow.test_pyarrow_ignore_timezone",
"pyspark.tests.upstream.pyarrow.test_pyarrow_scalar_type_coercion",
"pyspark.tests.upstream.pyarrow.test_pyarrow_scalar_type_inference",
+ "pyspark.tests.upstream.pyarrow.test_pyarrow_table_cast",
"pyspark.tests.upstream.pyarrow.test_pyarrow_table_to_pandas",
"pyspark.tests.upstream.pyarrow.test_pyarrow_type_coercion",
],
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_cast_safe.csv
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_cast_safe.csv
new file mode 100644
index 000000000000..1cdbb39f58d0
--- /dev/null
+++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_cast_safe.csv
@@ -0,0 +1,21 @@
+test case pyarrow table cast result
+types:downcast {a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a: int64, b: float64]
{a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a: int32, b: float32]
+types:upcast {a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a: int32, b: float32]
{a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a: int64, b: float64]
+overflow:int64->int32 {a: [1099511627776, 1]}@Table[a: int64] ERR@ArrowInvalid
+truncate:float->int {a: [1.9, -2.1]}@Table[a: float64] ERR@ArrowInvalid
+names:mismatch {a: [1, 2], b: [1.5, 2.5]}@Table[a: int64, b: float64]
ERR@ValueError
+names:reordered {a: [1, 2], b: [1.5, 2.5]}@Table[a: int64, b: float64]
ERR@ValueError
+names:field-count {a: [1, 2], b: [1.5, 2.5]}@Table[a: int64, b: float64]
ERR@ValueError
+nullable:false-with-nulls {a: [1, None]}@Table[a: int64] ERR@ValueError
+nullable:false-no-nulls {a: [1, 2]}@Table[a: int64] {a: [1,
2]}@Table[a: int32]
+timestamp:us->ns {ts: [1970-01-01 00:00:00, 1970-01-01
00:00:01]}@Table[ts: timestamp[us]] {ts: [1970-01-01 00:00:00, 1970-01-01
00:00:01]}@Table[ts: timestamp[ns]]
+timestamp:attach-tz {ts: [1970-01-01 00:00:00, 1970-01-01
00:00:01]}@Table[ts: timestamp[us]] {ts: [1970-01-01 00:00:00+00:00,
1970-01-01 00:00:01+00:00]}@Table[ts: timestamp[us, tz=UTC]]
+string->large_string {s: [hello, world, None]}@Table[s: string] {s:
[hello, world, None]}@Table[s: large_string]
+binary->large_binary {b: [b'x', b'yz', None]}@Table[b: binary] {b:
[b'x', b'yz', None]}@Table[b: large_binary]
+nested:list {lst: [[1, 2], [3], None]}@Table[lst: list<item: int64>]
{lst: [[1, 2], [3], None]}@Table[lst: list<item: int32>]
+nested:list-overflow {lst: [[1099511627776, 1]]}@Table[lst: list<item:
int64>] ERR@ArrowInvalid
+nested:struct {st: [[('x', 1), ('y', 'a')], None]}@Table[st: struct<x: int64,
y: string>] {st: [[('x', 1), ('y', 'a')], None]}@Table[st: struct<x: int32,
y: large_string>]
+nested:map {m: [[('k', 1), ('j', 2)], None]}@Table[m: map<string, int64>]
{m: [[('k', 1), ('j', 2)], None]}@Table[m: map<string, int32>]
+multi-chunk-column {a: [1, 2, 3, None]}@Table[a: int64] {a: [1, 2, 3,
None]}@Table[a: int32]
+empty:0-columns {}@Table[] {}@Table[]
+empty:columns-no-rows {i: [], s: []}@Table[i: int64, s: string] {i: [],
s: []}@Table[i: int32, s: large_string]
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_cast_safe.md
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_cast_safe.md
new file mode 100644
index 000000000000..397f4ebb9799
--- /dev/null
+++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_cast_safe.md
@@ -0,0 +1,22 @@
+| test case | pyarrow table
| cast result
|
+|---------------------------|-----------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------|
+| types:downcast | {a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a:
int64, b: float64] | {a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a:
int32, b: float32] |
+| types:upcast | {a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a:
int32, b: float32] | {a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a:
int64, b: float64] |
+| overflow:int64->int32 | {a: [1099511627776, 1]}@Table[a: int64]
| ERR@ArrowInvalid
|
+| truncate:float->int | {a: [1.9, -2.1]}@Table[a: float64]
| ERR@ArrowInvalid
|
+| names:mismatch | {a: [1, 2], b: [1.5, 2.5]}@Table[a: int64, b:
float64] | ERR@ValueError
|
+| names:reordered | {a: [1, 2], b: [1.5, 2.5]}@Table[a: int64, b:
float64] | ERR@ValueError
|
+| names:field-count | {a: [1, 2], b: [1.5, 2.5]}@Table[a: int64, b:
float64] | ERR@ValueError
|
+| nullable:false-with-nulls | {a: [1, None]}@Table[a: int64]
| ERR@ValueError
|
+| nullable:false-no-nulls | {a: [1, 2]}@Table[a: int64]
| {a: [1, 2]}@Table[a: int32]
|
+| timestamp:us->ns | {ts: [1970-01-01 00:00:00, 1970-01-01
00:00:01]}@Table[ts: timestamp[us]] | {ts: [1970-01-01 00:00:00, 1970-01-01
00:00:01]}@Table[ts: timestamp[ns]] |
+| timestamp:attach-tz | {ts: [1970-01-01 00:00:00, 1970-01-01
00:00:01]}@Table[ts: timestamp[us]] | {ts: [1970-01-01 00:00:00+00:00,
1970-01-01 00:00:01+00:00]}@Table[ts: timestamp[us, tz=UTC]] |
+| string->large_string | {s: [hello, world, None]}@Table[s: string]
| {s: [hello, world, None]}@Table[s: large_string]
|
+| binary->large_binary | {b: [b'x', b'yz', None]}@Table[b: binary]
| {b: [b'x', b'yz', None]}@Table[b: large_binary]
|
+| nested:list | {lst: [[1, 2], [3], None]}@Table[lst: list<item:
int64>] | {lst: [[1, 2], [3], None]}@Table[lst: list<item:
int32>] |
+| nested:list-overflow | {lst: [[1099511627776, 1]]}@Table[lst:
list<item: int64>] | ERR@ArrowInvalid
|
+| nested:struct | {st: [[('x', 1), ('y', 'a')], None]}@Table[st:
struct<x: int64, y: string>] | {st: [[('x', 1), ('y', 'a')], None]}@Table[st:
struct<x: int32, y: large_string>] |
+| nested:map | {m: [[('k', 1), ('j', 2)], None]}@Table[m:
map<string, int64>] | {m: [[('k', 1), ('j', 2)], None]}@Table[m:
map<string, int32>] |
+| multi-chunk-column | {a: [1, 2, 3, None]}@Table[a: int64]
| {a: [1, 2, 3, None]}@Table[a: int32]
|
+| empty:0-columns | {}@Table[]
| {}@Table[]
|
+| empty:columns-no-rows | {i: [], s: []}@Table[i: int64, s: string]
| {i: [], s: []}@Table[i: int32, s: large_string]
|
\ No newline at end of file
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_cast_unsafe.csv
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_cast_unsafe.csv
new file mode 100644
index 000000000000..f9065e14ff57
--- /dev/null
+++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_cast_unsafe.csv
@@ -0,0 +1,21 @@
+test case pyarrow table cast result
+types:downcast {a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a: int64, b: float64]
{a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a: int32, b: float32]
+types:upcast {a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a: int32, b: float32]
{a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a: int64, b: float64]
+overflow:int64->int32 {a: [1099511627776, 1]}@Table[a: int64] {a: [0,
1]}@Table[a: int32]
+truncate:float->int {a: [1.9, -2.1]}@Table[a: float64] {a: [1,
-2]}@Table[a: int64]
+names:mismatch {a: [1, 2], b: [1.5, 2.5]}@Table[a: int64, b: float64]
ERR@ValueError
+names:reordered {a: [1, 2], b: [1.5, 2.5]}@Table[a: int64, b: float64]
ERR@ValueError
+names:field-count {a: [1, 2], b: [1.5, 2.5]}@Table[a: int64, b: float64]
ERR@ValueError
+nullable:false-with-nulls {a: [1, None]}@Table[a: int64] ERR@ValueError
+nullable:false-no-nulls {a: [1, 2]}@Table[a: int64] {a: [1,
2]}@Table[a: int32]
+timestamp:us->ns {ts: [1970-01-01 00:00:00, 1970-01-01
00:00:01]}@Table[ts: timestamp[us]] {ts: [1970-01-01 00:00:00, 1970-01-01
00:00:01]}@Table[ts: timestamp[ns]]
+timestamp:attach-tz {ts: [1970-01-01 00:00:00, 1970-01-01
00:00:01]}@Table[ts: timestamp[us]] {ts: [1970-01-01 00:00:00+00:00,
1970-01-01 00:00:01+00:00]}@Table[ts: timestamp[us, tz=UTC]]
+string->large_string {s: [hello, world, None]}@Table[s: string] {s:
[hello, world, None]}@Table[s: large_string]
+binary->large_binary {b: [b'x', b'yz', None]}@Table[b: binary] {b:
[b'x', b'yz', None]}@Table[b: large_binary]
+nested:list {lst: [[1, 2], [3], None]}@Table[lst: list<item: int64>]
{lst: [[1, 2], [3], None]}@Table[lst: list<item: int32>]
+nested:list-overflow {lst: [[1099511627776, 1]]}@Table[lst: list<item:
int64>] {lst: [[0, 1]]}@Table[lst: list<item: int32>]
+nested:struct {st: [[('x', 1), ('y', 'a')], None]}@Table[st: struct<x: int64,
y: string>] {st: [[('x', 1), ('y', 'a')], None]}@Table[st: struct<x: int32,
y: large_string>]
+nested:map {m: [[('k', 1), ('j', 2)], None]}@Table[m: map<string, int64>]
{m: [[('k', 1), ('j', 2)], None]}@Table[m: map<string, int32>]
+multi-chunk-column {a: [1, 2, 3, None]}@Table[a: int64] {a: [1, 2, 3,
None]}@Table[a: int32]
+empty:0-columns {}@Table[] {}@Table[]
+empty:columns-no-rows {i: [], s: []}@Table[i: int64, s: string] {i: [],
s: []}@Table[i: int32, s: large_string]
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_cast_unsafe.md
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_cast_unsafe.md
new file mode 100644
index 000000000000..f1b1ddd381e1
--- /dev/null
+++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_table_cast_unsafe.md
@@ -0,0 +1,22 @@
+| test case | pyarrow table
| cast result
|
+|---------------------------|-----------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------|
+| types:downcast | {a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a:
int64, b: float64] | {a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a:
int32, b: float32] |
+| types:upcast | {a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a:
int32, b: float32] | {a: [1, 2, 3], b: [1.5, 2.5, 3.5]}@Table[a:
int64, b: float64] |
+| overflow:int64->int32 | {a: [1099511627776, 1]}@Table[a: int64]
| {a: [0, 1]}@Table[a: int32]
|
+| truncate:float->int | {a: [1.9, -2.1]}@Table[a: float64]
| {a: [1, -2]}@Table[a: int64]
|
+| names:mismatch | {a: [1, 2], b: [1.5, 2.5]}@Table[a: int64, b:
float64] | ERR@ValueError
|
+| names:reordered | {a: [1, 2], b: [1.5, 2.5]}@Table[a: int64, b:
float64] | ERR@ValueError
|
+| names:field-count | {a: [1, 2], b: [1.5, 2.5]}@Table[a: int64, b:
float64] | ERR@ValueError
|
+| nullable:false-with-nulls | {a: [1, None]}@Table[a: int64]
| ERR@ValueError
|
+| nullable:false-no-nulls | {a: [1, 2]}@Table[a: int64]
| {a: [1, 2]}@Table[a: int32]
|
+| timestamp:us->ns | {ts: [1970-01-01 00:00:00, 1970-01-01
00:00:01]}@Table[ts: timestamp[us]] | {ts: [1970-01-01 00:00:00, 1970-01-01
00:00:01]}@Table[ts: timestamp[ns]] |
+| timestamp:attach-tz | {ts: [1970-01-01 00:00:00, 1970-01-01
00:00:01]}@Table[ts: timestamp[us]] | {ts: [1970-01-01 00:00:00+00:00,
1970-01-01 00:00:01+00:00]}@Table[ts: timestamp[us, tz=UTC]] |
+| string->large_string | {s: [hello, world, None]}@Table[s: string]
| {s: [hello, world, None]}@Table[s: large_string]
|
+| binary->large_binary | {b: [b'x', b'yz', None]}@Table[b: binary]
| {b: [b'x', b'yz', None]}@Table[b: large_binary]
|
+| nested:list | {lst: [[1, 2], [3], None]}@Table[lst: list<item:
int64>] | {lst: [[1, 2], [3], None]}@Table[lst: list<item:
int32>] |
+| nested:list-overflow | {lst: [[1099511627776, 1]]}@Table[lst:
list<item: int64>] | {lst: [[0, 1]]}@Table[lst: list<item:
int32>] |
+| nested:struct | {st: [[('x', 1), ('y', 'a')], None]}@Table[st:
struct<x: int64, y: string>] | {st: [[('x', 1), ('y', 'a')], None]}@Table[st:
struct<x: int32, y: large_string>] |
+| nested:map | {m: [[('k', 1), ('j', 2)], None]}@Table[m:
map<string, int64>] | {m: [[('k', 1), ('j', 2)], None]}@Table[m:
map<string, int32>] |
+| multi-chunk-column | {a: [1, 2, 3, None]}@Table[a: int64]
| {a: [1, 2, 3, None]}@Table[a: int32]
|
+| empty:0-columns | {}@Table[]
| {}@Table[]
|
+| empty:columns-no-rows | {i: [], s: []}@Table[i: int64, s: string]
| {i: [], s: []}@Table[i: int32, s: large_string]
|
\ No newline at end of file
diff --git a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_table_cast.py
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_table_cast.py
new file mode 100644
index 000000000000..1012a97c9cdd
--- /dev/null
+++ b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_table_cast.py
@@ -0,0 +1,296 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Tests for PyArrow's pa.Table.cast() method using golden file comparison.
+
+Unlike pa.Array.cast() (covered by test_pyarrow_array_cast.py), Table.cast()
casts a
+whole table to a target *schema*. Per-column type conversion matches the Array
tests,
+so this file pins the genuinely Table-level behavior instead: multi-column
casts, strict
+field-name/order matching, target-field nullability enforcement, temporal
coercion, and
+the empty-table edges. Spark calls Table.cast() at
+python/pyspark/sql/pandas/conversion.py:499 and :1108 (classic toArrow /
createDataFrame)
+and, in Spark Connect, at python/pyspark/sql/connect/dataframe.py:1995 and
+python/pyspark/sql/connect/session.py:647,663.
+
+This suite covers both safe=True (default) and safe=False modes:
+- safe=True: checks for overflow/truncation, raising on unsafe conversions.
+- safe=False: allows unsafe conversions (overflow wrapping, truncation).
+Each mode has its own golden file. Field-name/order and nullability errors
raise in both
+modes -- they are checked before the per-column value cast.
+
+## Golden File Cell Format
+
+Each cell uses the value@type format:
+- pyarrow Table: "{col: [val1, val2, None], ...}@Table[name: type, ...]"
+- Error: "ERR@ExceptionClassName"
+
+## Regenerating Golden Files
+
+Set SPARK_GENERATE_GOLDEN_FILES=1 before running:
+
+ SPARK_GENERATE_GOLDEN_FILES=1 python -m pytest \\
+ python/pyspark/tests/upstream/pyarrow/test_pyarrow_table_cast.py
+
+If package tabulate (https://pypi.org/project/tabulate/) is installed,
+it will also regenerate the Markdown files.
+
+## PyArrow Version Compatibility
+
+The golden files capture behavior for a specific PyArrow version. Regenerate
when
+upgrading PyArrow, as cast support may change between versions. Table.cast() is
+pandas-free (PyArrow in, PyArrow out), so no pandas-version differences apply.
+"""
+
+import unittest
+
+from pyspark.testing.utils import (
+ have_pyarrow,
+ have_pandas,
+ pyarrow_requirement_message,
+ pandas_requirement_message,
+)
+from pyspark.testing.goldenutils import GoldenFileTestMixin
+
+if have_pyarrow:
+ import pyarrow as pa
+
+
+class _PyArrowTableCastTestBase(GoldenFileTestMixin, unittest.TestCase):
+ """Base class for pa.Table.cast() golden file tests. Defines no test_* of
its own."""
+
+ def _try_cast(self, table, target_schema, safe=True) -> str:
+ """
+ Cast ``table`` to ``target_schema`` and format the result as a golden
cell,
+ returning ``ERR@<ExceptionClass>`` if the cast raises. Only the cast
is guarded:
+ a formatting error is a test bug, not a cast signal, so it propagates.
+ """
+ try:
+ result = table.cast(target_schema, safe=safe)
+ except Exception as e:
+ return f"ERR@{type(e).__name__}"
+ return self.repr_value(result, max_len=0)
+
+ def _cast_scenarios(self):
+ """
+ Ordered {name: (source_table, target_schema)} pairs, each isolating one
+ Table.cast contract. Shared by the safe and unsafe test methods.
+ """
+ scenarios = {}
+
+ # =====================================================================
+ # Multi-column type cast (whole-schema assembly)
+ # =====================================================================
+ scenarios["types:downcast"] = (
+ pa.table(
+ {
+ "a": pa.array([1, 2, 3], pa.int64()),
+ "b": pa.array([1.5, 2.5, 3.5], pa.float64()),
+ }
+ ),
+ pa.schema([("a", pa.int32()), ("b", pa.float32())]),
+ )
+ scenarios["types:upcast"] = (
+ pa.table(
+ {
+ "a": pa.array([1, 2, 3], pa.int32()),
+ "b": pa.array([1.5, 2.5, 3.5], pa.float32()),
+ }
+ ),
+ pa.schema([("a", pa.int64()), ("b", pa.float64())]),
+ )
+
+ # =====================================================================
+ # safe axis: these flip between the safe and unsafe goldens
+ # =====================================================================
+ scenarios["overflow:int64->int32"] = (
+ pa.table({"a": pa.array([2**40, 1], pa.int64())}),
+ pa.schema([("a", pa.int32())]),
+ )
+ scenarios["truncate:float->int"] = (
+ pa.table({"a": pa.array([1.9, -2.1], pa.float64())}),
+ pa.schema([("a", pa.int64())]),
+ )
+
+ # =====================================================================
+ # Columns are matched positionally + name-equal (no match/reorder by
name), so
+ # a name mismatch, a pure reorder, and a wrong field count all raise
ValueError.
+ # =====================================================================
+ base_ab = pa.table(
+ {"a": pa.array([1, 2], pa.int64()), "b": pa.array([1.5, 2.5],
pa.float64())}
+ )
+ scenarios["names:mismatch"] = (
+ base_ab,
+ pa.schema([("x", pa.int32()), ("b", pa.float32())]),
+ )
+ scenarios["names:reordered"] = (
+ base_ab,
+ pa.schema([("b", pa.float64()), ("a", pa.int64())]),
+ )
+ scenarios["names:field-count"] = (
+ base_ab,
+ pa.schema([("a", pa.int32())]),
+ )
+
+ # =====================================================================
+ # Target-field nullability enforcement
+ # =====================================================================
+ scenarios["nullable:false-with-nulls"] = (
+ pa.table({"a": pa.array([1, None], pa.int64())}),
+ pa.schema([pa.field("a", pa.int32(), nullable=False)]),
+ )
+ scenarios["nullable:false-no-nulls"] = (
+ pa.table({"a": pa.array([1, 2], pa.int64())}),
+ pa.schema([pa.field("a", pa.int32(), nullable=False)]),
+ )
+
+ # =====================================================================
+ # Temporal unit / timezone coercion
+ # =====================================================================
+ ts_us = pa.table({"ts": pa.array([0, 1_000_000], pa.timestamp("us"))})
+ scenarios["timestamp:us->ns"] = (
+ ts_us,
+ pa.schema([("ts", pa.timestamp("ns"))]),
+ )
+ scenarios["timestamp:attach-tz"] = (
+ ts_us,
+ pa.schema([("ts", pa.timestamp("us", "UTC"))]),
+ )
+
+ # =====================================================================
+ # Variable-width widening
+ # =====================================================================
+ scenarios["string->large_string"] = (
+ pa.table({"s": pa.array(["hello", "world", None], pa.string())}),
+ pa.schema([("s", pa.large_string())]),
+ )
+ scenarios["binary->large_binary"] = (
+ pa.table({"b": pa.array([b"x", b"yz", None], pa.binary())}),
+ pa.schema([("b", pa.large_binary())]),
+ )
+
+ # =====================================================================
+ # Nested column types: cast recurses into the container and casts each
inner
+ # element, carrying safe= down (see nested:list-overflow). Kept name-
and
+ # order-matched so these stay clean successes on every PyArrow version.
+ # =====================================================================
+ scenarios["nested:list"] = (
+ pa.table({"lst": pa.array([[1, 2], [3], None],
pa.list_(pa.int64()))}),
+ pa.schema([("lst", pa.list_(pa.int32()))]),
+ )
+ scenarios["nested:list-overflow"] = (
+ pa.table({"lst": pa.array([[2**40, 1]], pa.list_(pa.int64()))}),
+ pa.schema([("lst", pa.list_(pa.int32()))]),
+ )
+ scenarios["nested:struct"] = (
+ pa.table(
+ {
+ "st": pa.array(
+ [{"x": 1, "y": "a"}, None],
+ pa.struct([("x", pa.int64()), ("y", pa.string())]),
+ )
+ }
+ ),
+ pa.schema([("st", pa.struct([("x", pa.int32()), ("y",
pa.large_string())]))]),
+ )
+ scenarios["nested:map"] = (
+ pa.table(
+ {"m": pa.array([[("k", 1), ("j", 2)], None],
pa.map_(pa.string(), pa.int64()))}
+ ),
+ pa.schema([("m", pa.map_(pa.string(), pa.int32()))]),
+ )
+
+ # =====================================================================
+ # Multi-chunk column: exercises pa.ChunkedArray.cast under Table.cast
+ # =====================================================================
+ scenarios["multi-chunk-column"] = (
+ pa.table({"a": pa.chunked_array([[1, 2], [3, None]], pa.int64())}),
+ pa.schema([("a", pa.int32())]),
+ )
+
+ # =====================================================================
+ # Empty edges
+ # =====================================================================
+ scenarios["empty:0-columns"] = (pa.table({}), pa.schema([]))
+ scenarios["empty:columns-no-rows"] = (
+ pa.table({"i": pa.array([], pa.int64()), "s": pa.array([],
pa.string())}),
+ pa.schema([("i", pa.int32()), ("s", pa.large_string())]),
+ )
+
+ return scenarios
+
+
[email protected](
+ not have_pyarrow or not have_pandas,
+ pyarrow_requirement_message or pandas_requirement_message,
+)
+class PyArrowTableCastTests(_PyArrowTableCastTestBase):
+ """
+ Tests pa.Table.cast(target_schema) with safe=True and safe=False via
golden files.
+
+ Pins Table-level cast behavior distinct from pa.Array.cast: whole-schema
casts,
+ strict field-name/order matching, target-field nullability enforcement,
temporal
+ coercion, and the empty-table edges.
+ """
+
+ def _run(self, safe, golden_file_prefix, overrides):
+ scenarios = self._cast_scenarios()
+ row_names = list(scenarios.keys())
+ col_names = ["pyarrow table", "cast result"]
+
+ def compute_cell(row_name, col_name):
+ source_table, target_schema = scenarios[row_name]
+ if col_name == "pyarrow table":
+ return self.repr_value(source_table, max_len=0)
+ elif col_name == "cast result":
+ return self._try_cast(source_table, target_schema, safe=safe)
+ else:
+ raise ValueError(f"unknown column: {col_name}")
+
+ self.compare_or_generate_golden_matrix(
+ row_names=row_names,
+ col_names=col_names,
+ compute_cell=compute_cell,
+ golden_file_prefix=golden_file_prefix,
+ index_name="test case",
+ overrides=overrides,
+ )
+
+ def test_table_cast_matrix(self):
+ """Test pa.Table.cast(target_schema) with safe=True (default)."""
+ # PyArrow-version-specific expected cells; empty at the pa24/pd2
baseline.
+ overrides: dict[tuple[str, str], str] = {}
+ self._run(
+ safe=True,
+ golden_file_prefix="golden_pyarrow_table_cast_safe",
+ overrides=overrides,
+ )
+
+ def test_table_cast_matrix_unsafe(self):
+ """Test pa.Table.cast(target_schema) with safe=False."""
+ overrides: dict[tuple[str, str], str] = {}
+ self._run(
+ safe=False,
+ golden_file_prefix="golden_pyarrow_table_cast_unsafe",
+ overrides=overrides,
+ )
+
+
+if __name__ == "__main__":
+ from pyspark.testing import main
+
+ main()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
