This is an automated email from the ASF dual-hosted git repository.
Yicong-Huang pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new c0e0237653ab [SPARK-58766][PYTHON][TEST][FOLLOWUP] Render non-UTF-8
binary->string casts in golden files and narrow the cell try/except
c0e0237653ab is described below
commit c0e0237653ab09f1f440228887336378f7f82a83
Author: Spenser Sun <[email protected]>
AuthorDate: Mon Aug 24 18:15:44 2026 +0000
[SPARK-58766][PYTHON][TEST][FOLLOWUP] Render non-UTF-8 binary->string casts
in golden files and narrow the cell try/except
### What changes were proposed in this pull request?
Follow-up to #58000 (SPARK-58766), which made `_scalar_str`
(`goldenutils.py`) render out-of-range temporal scalars instead of crashing.
This does the same for the binary case and narrows the cell try/except:
- **Binary repr:** an unsafe `binary`/`large_binary` ->
`string`/`large_string` cast relabels the buffer without validating UTF-8, so
`str(scalar)` raises `UnicodeDecodeError`. `_scalar_str` now renders the raw
bytes for such scalars (`scalar.as_buffer().to_pybytes()`, e.g. `b'\xff'`), so
the 4 affected `golden_pyarrow_scalar_cast_unsafe` cells render instead of
`ERRUnicodeDecodeError`. Raw bytes keep it a drift canary that tracks what
Arrow actually stores.
- **Narrowing:** move `repr_value` outside the `try` in the 3 cell helpers
(`_try_cast`, `_to_pandas_cell`, `_from_pandas_cell`), so only the conversion
is guarded and a formatter bug fails loudly.
### Why are the changes needed?
These goldens are drift canaries for the PyArrow behavior PySpark relies on
(SPARK-54936). `ERRUnicodeDecodeError` hid a real upstream behavior behind a
formatting failure.
### Does this PR introduce _any_ user-facing change?
No. Test-only.
### How was this patch tested?
Regenerated the golden (4 cells change) and swept PyArrow 18-25 x pandas
2/3; the full `upstream/pyarrow/` and `sql/tests/coercion/` golden tests pass.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 4.8)
Closes #58091 from Spenserrrr/binary-repr-narrowing.
Authored-by: Spenser Sun <[email protected]>
Signed-off-by: Yicong-Huang <[email protected]>
(cherry picked from commit f95006c1a2b47b7432b71d29677a876da761ccb4)
Signed-off-by: Yicong-Huang <[email protected]>
---
python/pyspark/testing/goldenutils.py | 21 ++++++++++++++++-----
.../pyarrow/golden_pyarrow_scalar_cast_unsafe.csv | 4 ++--
.../pyarrow/golden_pyarrow_scalar_cast_unsafe.md | 4 ++--
.../upstream/pyarrow/test_pyarrow_array_cast.py | 2 +-
.../test_pyarrow_array_from_pandas_default.py | 3 ++-
.../pyarrow/test_pyarrow_arrow_to_pandas_default.py | 3 ++-
6 files changed, 25 insertions(+), 12 deletions(-)
diff --git a/python/pyspark/testing/goldenutils.py
b/python/pyspark/testing/goldenutils.py
index df699a81b065..f9ba1f66fd98 100644
--- a/python/pyspark/testing/goldenutils.py
+++ b/python/pyspark/testing/goldenutils.py
@@ -316,10 +316,13 @@ class GoldenFileTestMixin:
"""
Render one PyArrow scalar for a golden cell via PyArrow's own
``str(scalar)``.
- A temporal value can be valid in Arrow yet outside Python's
``datetime`` range
- (e.g. a date32 past year 9999), where ``str`` builds a Python datetime
and
- raises ``OverflowError``. Record ``temporal overflow`` for those
instead of
- failing. A non-temporal ``OverflowError`` is unexpected, so it
propagates.
+ Some values are valid in Arrow yet unrenderable via ``str``:
+ - a temporal value past Python's ``datetime`` range (e.g. date32 past
year 9999)
+ raises ``OverflowError`` -> the marker ``temporal overflow``;
+ - an unsafe ``binary``->``string`` cast relabels bytes without UTF-8
validation,
+ so non-UTF-8 bytes raise ``UnicodeDecodeError`` -> the raw bytes, so
the golden
+ still tracks what Arrow stored for them.
+ An unexpected error (non-temporal overflow, non-string decode error)
propagates.
"""
try:
return str(scalar).replace("\x00", "\\0")
@@ -330,6 +333,14 @@ class GoldenFileTestMixin:
if pa.types.is_temporal(scalar.type):
return "temporal overflow"
raise
+ except UnicodeDecodeError:
+ # An unsafe binary->string cast relabels bytes as a string without
UTF-8
+ # validation, so str() cannot decode non-UTF-8 bytes; render the
raw bytes
+ # so the golden still tracks what Arrow stored for them. A
UnicodeDecodeError
+ # on a non-string type is unexpected, so re-raise.
+ if pa.types.is_string(scalar.type) or
pa.types.is_large_string(scalar.type):
+ return repr(scalar.as_buffer().to_pybytes())
+ raise
@classmethod
def repr_arrow_value(
@@ -339,7 +350,7 @@ class GoldenFileTestMixin:
Format a PyArrow Array/ChunkedArray for golden file.
Each element is rendered by ``_scalar_str`` (PyArrow's scalar
formatting, with
- an out-of-range temporal fallback).
+ fallbacks for values that are valid in Arrow but unrenderable via
``str``).
Parameters
----------
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_scalar_cast_unsafe.csv
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_scalar_cast_unsafe.csv
index eb413767be56..6e8bf7bf685c 100644
---
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_scalar_cast_unsafe.csv
+++
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_scalar_cast_unsafe.csv
@@ -35,8 +35,8 @@ string:unicode ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInval
large_string:numeric [0, 1, -1, None]@int8 [0, 1, -1, None]@int16 [0, 1,
-1, None]@int32 [0, 1, -1, None]@int64 ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid [0.0,
1.0, -1.0, None]@float16 [0.0, 1.0, -1.0, None]@float32 [0.0, 1.0, -1.0,
None]@float64 ERR@ArrowInvalid [0, 1, -1, None]@string [0, 1, -1,
None]@large_string [b'0', b'1', b'-1', None]@binary [b'0', b'1',
b'-1', None]@large_binary ERR@ArrowInvalid [0E-10, 1.0000000000,
-1.0000000000, None]@decimal128(38, 1 [...]
large_string:alpha ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid [abc, , None]@string [abc, , None]@large_string
[b'abc', b'', None]@binary [b'abc', b'', None]@large_binary
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@Arro [...]
large_string:unicode ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid [你好, مرحبا, 🎉, None]@string [你好, مرحبا, 🎉,
None]@large_string [b'\xe4\xbd\xa0\xe5\xa5\xbd',
b'\xd9\x85\xd8\xb1\xd8\xad\xd8\xa8\xd8\xa7', b'\xf0\x9f\x8e\x89', None]@binary
[b'\xe4\xbd\xa0\xe5\xa5\xbd', b'\xd9\x85\xd8\xb1\xd8\xad\xd8\xa8\xd8\xa7',
b'\xf0\x9f\x8e\x89', [...]
-binary:standard ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@UnicodeDecodeError ERR@UnicodeDecodeError
[b'\x00', b'\xff', b'hello', b'', None]@binary [b'\x00', b'\xff', b'hello',
b'', None]@large_binary ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowNotImplementedError
ERR@ArrowNotImplementedError ERR@ArrowNotImple [...]
-large_binary:standard ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@UnicodeDecodeError ERR@UnicodeDecodeError
[b'\x00', b'\xff', b'hello', b'', None]@binary [b'\x00', b'\xff', b'hello',
b'', None]@large_binary ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowNotImplementedError
ERR@ArrowNotImplementedError ERR@ArrowNo [...]
+binary:standard ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid [\0, b'\xff', hello, , None]@string [\0, b'\xff',
hello, , None]@large_string [b'\x00', b'\xff', b'hello', b'',
None]@binary [b'\x00', b'\xff', b'hello', b'', None]@large_binary
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowNotImplementedError ERR@ArrowNotIm [...]
+large_binary:standard ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowInvalid [\0, b'\xff', hello, , None]@string [\0, b'\xff',
hello, , None]@large_string [b'\x00', b'\xff', b'hello', b'',
None]@binary [b'\x00', b'\xff', b'hello', b'', None]@large_binary
ERR@ArrowInvalid ERR@ArrowInvalid ERR@ArrowInvalid
ERR@ArrowNotImplementedError ERR@Arro [...]
fixed_size_binary[16]:standard ERR@ArrowNotImplementedError
ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError
ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError
ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError
ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError
ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError
ERR@ArrowNotImplementedError [0123456789abcdef,
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0, None]@string [0123456789abcdef,
\0\0\0\0\0\0\0\0\0\0\0\0\0\0 [...]
decimal128(38, 10):standard [0, 1, -1, None]@int8 [0, 1, -1, None]@int16
[0, 1, -1, None]@int32 [0, 1, -1, None]@int64 [0, 1, 255, None]@uint8 [0, 1,
65535, None]@uint16 [0, 1, 4294967295, None]@uint32 [0, 1,
18446744073709551615, None]@uint64 ERR@ArrowNotImplementedError [0.0,
1.5, -1.5, None]@float32 [0.0, 1.5, -1.5, None]@float64
ERR@ArrowNotImplementedError [0E-10, 1.5000000000, -1.5000000000,
None]@string [0E-10, 1.5000000000, -1.5000000000, None]@large_string
ERR@ArrowNotImplemente [...]
decimal128(38, 10):large [-1, None]@int8 [-7169, None]@int16
[1410065407, None]@int32 [9999999999, None]@int64 [255,
None]@uint8 [58367, None]@uint16 [1410065407, None]@uint32
[9999999999, None]@uint64 ERR@ArrowNotImplementedError [10000000000.0,
None]@float32 [9999999999.0, None]@float64 ERR@ArrowNotImplementedError
[9999999999.0000000000, None]@string [9999999999.0000000000,
None]@large_string ERR@ArrowNotImplementedError
ERR@ArrowNotImplementedError ERR@ArrowNotImplementedError [99999 [...]
diff --git
a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_scalar_cast_unsafe.md
b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_scalar_cast_unsafe.md
index e35d32fa14f8..dac777b15df4 100644
--- a/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_scalar_cast_unsafe.md
+++ b/python/pyspark/tests/upstream/pyarrow/golden_pyarrow_scalar_cast_unsafe.md
@@ -36,8 +36,8 @@
| large_string:numeric | [0, 1, -1, None]@int8 |
[0, 1, -1, None]@int16 | [0, 1, -1, None]@int32
| [0, 1, -1, None]@int64
| ERR@ArrowInvalid | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| [0.0, 1.0, -1.0, None]@float16 | [0.0,
1.0, -1.0, None]@float32 [...]
| large_string:alpha | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| ERR@ArrowInvalid
| ERR@ArrowInvalid | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| ERR@ArrowInvalid |
ERR@ArrowInvalid [...]
| large_string:unicode | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| ERR@ArrowInvalid
| ERR@ArrowInvalid | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| ERR@ArrowInvalid |
ERR@ArrowInvalid [...]
-| binary:standard | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| ERR@ArrowInvalid
| ERR@ArrowInvalid | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| ERR@ArrowInvalid |
ERR@ArrowInvalid [...]
-| large_binary:standard | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| ERR@ArrowInvalid
| ERR@ArrowInvalid | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| ERR@ArrowInvalid |
ERR@ArrowInvalid [...]
+| binary:standard | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| ERR@ArrowInvalid
| ERR@ArrowInvalid | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| ERR@ArrowInvalid |
ERR@ArrowInvalid [...]
+| large_binary:standard | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| ERR@ArrowInvalid
| ERR@ArrowInvalid | ERR@ArrowInvalid |
ERR@ArrowInvalid | ERR@ArrowInvalid
| ERR@ArrowInvalid |
ERR@ArrowInvalid [...]
| fixed_size_binary[16]:standard | ERR@ArrowNotImplementedError |
ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError
| ERR@ArrowNotImplementedError
| ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError |
ERR@ArrowNotImplementedError | ERR@ArrowNotImplementedError
| ERR@ArrowNotImplementedError |
ERR@ArrowNotImplementedError [...]
| decimal128(38, 10):standard | [0, 1, -1, None]@int8 |
[0, 1, -1, None]@int16 | [0, 1, -1, None]@int32
| [0, 1, -1, None]@int64
| [0, 1, 255, None]@uint8 | [0, 1, 65535, None]@uint16 | [0, 1,
4294967295, None]@uint32 | [0, 1, 18446744073709551615, None]@uint64
| ERR@ArrowNotImplementedError | [0.0, 1.5,
-1.5, None]@float32 [...]
| decimal128(38, 10):large | [-1, None]@int8 |
[-7169, None]@int16 | [1410065407, None]@int32
| [9999999999, None]@int64
| [255, None]@uint8 | [58367, None]@uint16 |
[1410065407, None]@uint32 | [9999999999, None]@uint64
| ERR@ArrowNotImplementedError |
[10000000000.0, None]@float32 [...]
diff --git a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_cast.py
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_cast.py
index 128557fa6548..7b22bbd0b44f 100644
--- a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_cast.py
+++ b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_cast.py
@@ -127,9 +127,9 @@ class _PyArrowCastTestBase(GoldenFileTestMixin,
unittest.TestCase):
"""
try:
result = src_arr.cast(tgt_type, safe=safe)
- return self.repr_value(result, max_len=0)
except Exception as e:
return f"ERR@{type(e).__name__}"
+ return self.repr_value(result, max_len=0)
# ============================================================
diff --git
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_from_pandas_default.py
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_from_pandas_default.py
index 6b7db7ee8c18..a2f7b99afb38 100644
---
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_from_pandas_default.py
+++
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_from_pandas_default.py
@@ -120,9 +120,10 @@ class _PyArrowFromPandasTestBase(GoldenFileTestMixin,
unittest.TestCase):
as a golden-file cell, returning ``ERR@<ExceptionClass>`` if the
conversion raises.
"""
try:
- return self.repr_from_pandas_result(pa.Array.from_pandas(series,
**from_pandas_kwargs))
+ result = pa.Array.from_pandas(series, **from_pandas_kwargs)
except Exception as e:
return f"ERR@{type(e).__name__}"
+ return self.repr_from_pandas_result(result)
def _numpy_backed_sources(self):
"""
diff --git
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_default.py
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_default.py
index 68c9c4e5de51..fc154d70988d 100644
---
a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_default.py
+++
b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_default.py
@@ -92,9 +92,10 @@ class _PyArrowToPandasTestBase(GoldenFileTestMixin,
unittest.TestCase):
conversion raises.
"""
try:
- return self.repr_value(arr.to_pandas(**to_pandas_kwargs),
max_len=0)
+ result = arr.to_pandas(**to_pandas_kwargs)
except Exception as e:
return f"ERR@{type(e).__name__}"
+ return self.repr_value(result, max_len=0)
def _numeric_sources(self):
"""Integer, float, and boolean arrays."""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
