This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 5ecd27e [SPARK-61127][CONNECT][PYTHON] Make the rows of
DataFrame.collect and toLocalIterator honor
spark.sql.execution.pyspark.binaryAsBytes in the Python client as PySpark does
5ecd27e is described below
commit 5ecd27ed5fee1d78dc6a652d05dce938a4a9e752
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 06:25:15 2026 +0900
[SPARK-61127][CONNECT][PYTHON] Make the rows of DataFrame.collect and
toLocalIterator honor spark.sql.execution.pyspark.binaryAsBytes in the Python
client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61127](https://issues.apache.org/jira/browse/SPARK-61127): PySpark
Connect's `collect` and `toLocalIterator` convert the binary values of their
rows per the conf `spark.sql.execution.pyspark.binaryAsBytes`
(`DataFrame._get_binary_as_bytes`):
- `bytes` when the conf is `true`, case-insensitively, as by default
- `bytearray` otherwise
This applies to the binary values in arrays, maps and structs too, and to
`first`, `head`, `take` and `tail`, which collect.
The client now does the same. The rows of the PyO3 layer keep whether their
binary values are `bytes`, and so do the rows of their structs.
The client reads the conf only for a result with a binary value, the only
kind of result the conf changes. So a result without one makes no extra
request. PySpark reads the conf for every `collect`.
`foreachPartition` already read the conf, as PySpark's does.
### Why are the changes needed?
The client always gave `bytes`. So with the conf set to `false`,
`df.collect()` of a binary column gave `bytes`, where PySpark 4.3.0 gives
`bytearray`. Apache Spark's `ReadwriterParityTests.test_binary_type` failed on
this client.
### Does this PR introduce _any_ user-facing change?
Yes. With `spark.sql.execution.pyspark.binaryAsBytes` set to `false`, the
binary values of collected rows are `bytearray`, as with PySpark 4.3.0. By
default they are still `bytes`.
### How was this patch tested?
- A new live test checks the types of the binary values of `collect`,
`toLocalIterator` (with and without prefetching), `first`, `head`, `take` and
`tail`:
- in a column, an array, a struct and a map
- with the conf `false`, `TRUE` and `FALSE`, then unset
It passes with official PySpark 4.3.0, and fails without this change.
- Apache Spark's parity tests of binary values, run on this client against
a local server, pass. These are `test_binary_type` of
`test_parity_readwriter.py`, and the `binary_type` tests of
`test_parity_udf.py`, `test_parity_udtf.py` and
`arrow/test_parity_arrow_python_udf.py`. Without this change,
`test_binary_type` failed.
- The lib tests of both crates (590 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2178 passed, 8 skipped), and the
live drop-in and Arrow suites (175 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #320 from HyukjinKwon/SPARK-61127-binary-as-bytes.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 57 +++++++++++++++++++++++++++++++-------
crates/pyspark-rs/src/row.rs | 48 +++++++++++++++++++++++---------
python/tests/test_dropin_e2e.py | 33 ++++++++++++++++++++++
3 files changed, 115 insertions(+), 23 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 4b70df5..ff29ab6 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -2016,7 +2016,7 @@ impl PyDataFrame {
let (rows, schema) = py
.detach(|| self.dataframe.collect_with_schema())
.to_pyerr()?;
- rows_to_py(py, rows, schema.as_ref())
+ rows_to_py(py, &self.dataframe, rows, schema.as_ref())
}
/// Return an iterator over rows without materializing the entire result
in memory.
@@ -2208,7 +2208,8 @@ impl PyDataFrame {
let (row, schema) = py
.detach(|| self.dataframe.first_with_schema())
.to_pyerr()?;
- Ok(rows_to_py(py, row.into_iter().collect(), schema.as_ref())?.pop())
+ let rows = row.into_iter().collect();
+ Ok(rows_to_py(py, &self.dataframe, rows, schema.as_ref())?.pop())
}
/// Alias for first.
@@ -2223,7 +2224,7 @@ impl PyDataFrame {
let (rows, schema) = py
.detach(|| self.dataframe.limit(k).collect_with_schema())
.to_pyerr()?;
- rows_to_py(py, rows, schema.as_ref())?.into_py_any(py)
+ rows_to_py(py, &self.dataframe, rows,
schema.as_ref())?.into_py_any(py)
}
}
}
@@ -2233,7 +2234,7 @@ impl PyDataFrame {
let (rows, schema) = py
.detach(|| self.dataframe.take_with_schema(num))
.to_pyerr()?;
- rows_to_py(py, rows, schema.as_ref())
+ rows_to_py(py, &self.dataframe, rows, schema.as_ref())
}
/// Column names (a property, mirroring `DataFrame.columns`).
@@ -2247,7 +2248,7 @@ impl PyDataFrame {
let (rows, schema) = py
.detach(|| self.dataframe.tail(num).collect_with_schema())
.to_pyerr()?;
- rows_to_py(py, rows, schema.as_ref())
+ rows_to_py(py, &self.dataframe, rows, schema.as_ref())
}
/// Sample a fraction of rows, as PySpark's
`DataFrame.sample(withReplacement=None,
@@ -3989,19 +3990,46 @@ fn check_collectable(
Ok(())
}
-/// The collected `rows` of a result of `schema`, whose values of user-defined
types are
-/// deserialized when first read.
+/// Whether the binary values of a result of `schema` of `dataframe` are
`bytes`, as PySpark
+/// Connect's `DataFrame._get_binary_as_bytes` gives it: the conf
+/// `spark.sql.execution.pyspark.binaryAsBytes`, true unless it is set to
another value than
+/// `true`. The conf is read only for a result with a binary value, the only
one it changes.
+fn binary_as_bytes(
+ py: Python<'_>,
+ dataframe: &DataFrame,
+ schema: Option<&spark_connect::types::DataType>,
+) -> PyResult<bool> {
+ let binary =
+ |t: &spark_connect::types::DataType| matches!(t,
spark_connect::types::DataType::Binary);
+ if !schema.is_some_and(|schema| has_type(schema, &binary)) {
+ return Ok(true);
+ }
+ let value = py
+ .detach(|| {
+ dataframe
+ .spark_session()
+ .conf()
+ .get_with_default("spark.sql.execution.pyspark.binaryAsBytes",
Some("true"))
+ })
+ .to_pyerr()?;
+ Ok(value.is_some_and(|value| value.eq_ignore_ascii_case("true")))
+}
+
+/// The collected `rows` of a result of `schema` of `dataframe`, whose values
of user-defined
+/// types are deserialized when first read.
fn rows_to_py(
py: Python<'_>,
+ dataframe: &DataFrame,
rows: Vec<spark_connect::row::Row>,
schema: Option<&spark_connect::types::DataType>,
) -> PyResult<Vec<PyRow>> {
check_arrow_schema(py, schema)?;
check_collectable(py, schema)?;
let converters = udt_field_converters(py, schema)?;
+ let binary_as_bytes = binary_as_bytes(py, dataframe, schema)?;
Ok(rows
.into_iter()
- .map(|row| PyRow::with_converters(row, converters.clone()))
+ .map(|row| PyRow::with_converters(row, converters.clone(),
binary_as_bytes))
.collect())
}
@@ -4019,6 +4047,8 @@ pub struct PyLocalRowIterator {
/// The converters of the fields of user-defined types of the rows, once
the schema of the
/// result is known: with the first row.
converters: Option<Option<FieldConverters>>,
+ /// Whether the binary values of the rows are `bytes`, known with the
converters.
+ binary_as_bytes: bool,
}
impl PyLocalRowIterator {
@@ -4029,6 +4059,7 @@ impl PyLocalRowIterator {
iterator: None,
finished: false,
converters: None,
+ binary_as_bytes: true,
}
}
@@ -4067,13 +4098,19 @@ impl PyLocalRowIterator {
self.finished = true;
return Err(error);
}
- self.converters = Some(udt_field_converters(py,
Some(&schema))?);
+ let converters = udt_field_converters(py, Some(&schema))?;
+ self.binary_as_bytes = binary_as_bytes(py, &self.dataframe,
Some(&schema))?;
+ self.converters = Some(converters);
}
}
match next {
Some(Ok(row)) => {
let converters = self.converters.clone().flatten();
- Ok(Some(PyRow::with_converters(row, converters)))
+ Ok(Some(PyRow::with_converters(
+ row,
+ converters,
+ self.binary_as_bytes,
+ )))
}
// As PySpark raises the error of the execution, which ends the
iteration.
Some(Err(error)) => {
diff --git a/crates/pyspark-rs/src/row.rs b/crates/pyspark-rs/src/row.rs
index 4524877..75bd78a 100644
--- a/crates/pyspark-rs/src/row.rs
+++ b/crates/pyspark-rs/src/row.rs
@@ -4,7 +4,7 @@ use pyo3::basic::CompareOp;
use pyo3::exceptions::PyIndexError;
use pyo3::prelude::*;
use pyo3::sync::PyOnceLock;
-use pyo3::types::{PyBytes, PyDict, PyList, PySlice, PyString, PyTuple};
+use pyo3::types::{PyByteArray, PyBytes, PyDict, PyList, PySlice, PyString,
PyTuple};
use spark_connect::row::{Row, Value};
use crate::column::raise_pyspark;
@@ -34,6 +34,9 @@ enum RowData {
values: PyOnceLock<Py<PyTuple>>,
/// The converters of its fields, when one has a user-defined type.
converters: Option<FieldConverters>,
+ /// Whether its binary values are `bytes`, or `bytearray`, as PySpark
gives them per
+ /// `spark.sql.execution.pyspark.binaryAsBytes`.
+ binary_as_bytes: bool,
},
/// A row made in Python, which holds its Python values as they are, as
PySpark's does.
Python {
@@ -44,17 +47,23 @@ enum RowData {
impl PyRow {
pub fn new(row: Row) -> Self {
- Self::with_converters(row, None)
+ Self::with_converters(row, None, true)
}
/// A row that the server returned, whose values of user-defined types
`converters`
- /// deserializes when they are first read.
- pub(crate) fn with_converters(row: Row, converters:
Option<FieldConverters>) -> Self {
+ /// deserializes when they are first read, and whose binary values are
`bytes` if
+ /// `binary_as_bytes`, or else `bytearray`.
+ pub(crate) fn with_converters(
+ row: Row,
+ converters: Option<FieldConverters>,
+ binary_as_bytes: bool,
+ ) -> Self {
PyRow {
data: RowData::Spark {
row,
values: PyOnceLock::new(),
converters,
+ binary_as_bytes,
},
named: true,
}
@@ -85,6 +94,7 @@ impl PyRow {
row,
values,
converters,
+ binary_as_bytes,
} => {
let values = values.get_or_try_init(py, || ->
PyResult<Py<PyTuple>> {
let convert = |i: usize| {
@@ -98,7 +108,7 @@ impl PyRow {
.iter()
.enumerate()
.map(|(i, v)| {
- let value = value_to_py(py, v)?;
+ let value = value_to_py(py, v, *binary_as_bytes)?;
match convert(i) {
Some(converter) =>
converter.bind(py).call1((value,)),
None => Ok(value),
@@ -207,8 +217,13 @@ fn micros_timedelta(py: Python<'_>, micros: i64) ->
PyResult<Bound<'_, PyAny>> {
))
}
-/// Convert a Spark `Value` into a Python object (PyO3 0.28 `IntoPyObject`).
-pub fn value_to_py<'py>(py: Python<'py>, v: &Value) -> PyResult<Bound<'py,
PyAny>> {
+/// Convert a Spark `Value` into a Python object (PyO3 0.28 `IntoPyObject`),
whose binary values
+/// are `bytes` if `binary_as_bytes`, or else `bytearray`, also in arrays,
maps and structs.
+pub fn value_to_py<'py>(
+ py: Python<'py>,
+ v: &Value,
+ binary_as_bytes: bool,
+) -> PyResult<Bound<'py, PyAny>> {
// Propagate conversion failures as Python exceptions with `?` rather than
// panicking across the FFI boundary (the conversions are near-infallible,
// but a panic here would unwind into CPython).
@@ -222,7 +237,8 @@ pub fn value_to_py<'py>(py: Python<'py>, v: &Value) ->
PyResult<Bound<'py, PyAny
Value::Float(f) => (*f as f64).into_pyobject(py)?.into_any(),
Value::Double(f) => (*f).into_pyobject(py)?.into_any(),
Value::String(s) => s.as_str().into_pyobject(py)?.into_any(),
- Value::Binary(b) => PyBytes::new(py, b).into_any(),
+ Value::Binary(b) if binary_as_bytes => PyBytes::new(py, b).into_any(),
+ Value::Binary(b) => PyByteArray::new(py, b).into_any(),
Value::Date(d) => {
// Convert days-since-epoch (i32) to datetime.date
let date_mod = py.import("datetime")?;
@@ -288,7 +304,7 @@ pub fn value_to_py<'py>(py: Python<'py>, v: &Value) ->
PyResult<Bound<'py, PyAny
Value::List(items) => {
let list = PyList::empty(py);
for it in items {
- list.append(value_to_py(py, it)?)?;
+ list.append(value_to_py(py, it, binary_as_bytes)?)?;
}
list.into_any()
}
@@ -297,16 +313,22 @@ pub fn value_to_py<'py>(py: Python<'py>, v: &Value) ->
PyResult<Bound<'py, PyAny
// TypeError, as in PySpark.
let dict = PyDict::new(py);
for (k, val) in m {
- dict.set_item(value_to_py(py, k)?, value_to_py(py, val)?)?;
+ dict.set_item(
+ value_to_py(py, k, binary_as_bytes)?,
+ value_to_py(py, val, binary_as_bytes)?,
+ )?;
}
dict.into_any()
}
Value::Struct(row) => {
// A struct value materializes as a Row (named fields), matching
PySpark,
// rather than a plain dict. Cloning keeps shared names and
independent values.
- Py::new(py, PyRow::new(row.clone().into()))?
- .into_bound(py)
- .into_any()
+ Py::new(
+ py,
+ PyRow::with_converters(row.clone().into(), None,
binary_as_bytes),
+ )?
+ .into_bound(py)
+ .into_any()
}
Value::Variant { value, metadata } => {
// A VARIANT materializes as a VariantVal (toJson/toPython decode
lazily),
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index bea3e3e..19896c5 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1382,6 +1382,39 @@ def
test_collect_reads_a_result_across_stream_ends(session):
assert session.range(n).toArrow().num_rows == n
+def
test_binary_values_of_collected_rows_follow_binary_as_bytes_as_pyspark(session):
+ # As PySpark Connect's `DataFrame._get_binary_as_bytes`: the binary values
of the rows of
+ # collect and toLocalIterator, and so of first, head, take and tail, also
those in arrays,
+ # maps and structs, are bytes, unless
spark.sql.execution.pyspark.binaryAsBytes is set to
+ # another value than true, case-insensitively, which makes them bytearray.
+ df = session.sql(
+ "SELECT CAST('ab' AS BINARY) AS b, array(CAST('c' AS BINARY)) AS a, "
+ "named_struct('x', CAST('d' AS BINARY)) AS s, map('k', CAST('e' AS
BINARY)) AS m"
+ )
+
+ def types(row):
+ return [type(value) for value in (row.b, row.a[0], row.s.x,
row.m["k"])]
+
+ key = "spark.sql.execution.pyspark.binaryAsBytes"
+ try:
+ for value, expected in [("false", bytearray), ("TRUE", bytes),
("FALSE", bytearray)]:
+ session.conf.set(key, value)
+ assert types(df.collect()[0]) == [expected] * 4
+ assert types(next(df.toLocalIterator())) == [expected] * 4
+ assert types(next(df.toLocalIterator(prefetchPartitions=True))) ==
[expected] * 4
+ for row in [df.first(), df.head(), df.head(1)[0], df.take(1)[0],
df.tail(1)[0]]:
+ assert types(row) == [expected] * 4
+ assert df.collect() == [Row(b=b"ab", a=[b"c"], s=Row(x=b"d"),
m={"k": b"e"})]
+ # A result without a binary value is the same either way.
+ assert session.range(1).collect() == [Row(id=0)]
+ finally:
+ session.conf.unset(key)
+ assert types(df.collect()[0]) == [bytes] * 4
+ assert types(next(df.toLocalIterator())) == [bytes] * 4
+
+
+
+
def
test_internal_values_of_collected_rows_are_those_of_spark_as_pyspark(session):
# As PySpark's: the toInternal of a collected row is the internal value of
Spark that the
# server computes, the days and the microseconds since the epoch, and its
fromInternal is
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]