This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new efbd43e [SPARK-61014][CONNECT][PYTHON] Run foreach and
foreachPartition, and return the DataFrame of toJSON, as PySpark Connect does
efbd43e is described below
commit efbd43e3dfa32c9f7bc67ce964d371263f72392d
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 01:57:31 2026 +0900
[SPARK-61014][CONNECT][PYTHON] Run foreach and foreachPartition, and return
the DataFrame of toJSON, as PySpark Connect does
### What changes were proposed in this pull request?
[SPARK-61014](https://issues.apache.org/jira/browse/SPARK-61014): `foreach`
and `foreachPartition` now run as PySpark Connect's do, and `toJSON` returns a
DataFrame.
- `df.foreach(f)` selects a struct of the columns of each row, with a UDF
of no result that calls `f` with the row.
- `df.foreachPartition(f)` runs a `mapInArrow` function of no result that
calls `f` with an iterator of the rows of each partition. It converts them with
`spark.sql.execution.pyspark.binaryAsBytes`.
- The functions are pickled and run by the server's Python worker, so they
are Python, in `pyspark.sql._foreach`, as `pyspark.sql.connect.dataframe`
builds them.
- `df.toJSON()` returns the DataFrame of
`select(to_json(struct(col("*"))).alias("value"))`.
In the Rust API, `DataFrame::to_json_dataframe` builds the DataFrame of
`toJSON`, and `to_json` collects its strings.
### Why are the changes needed?
`foreach` and `foreachPartition` failed on the worker with `TypeError:
cannot unpack non-iterable function object`, because the command of their UDF
was the function rather than the pair of the function and its return type.
`toJSON()` raised `NotImplementedError`.
### Does this PR introduce _any_ user-facing change?
Yes. `foreach`, `foreachPartition` and `toJSON` work as in PySpark 4.3.0.
### How was this patch tested?
- A new live test checks the row that `foreach` gives `f` and the rows that
`foreachPartition` gives it, by errors that `f` raises on the server. It also
checks the DataFrame of `toJSON`. It passes with official PySpark 4.3.0, and
fails without this change.
- A new offline test unpickles the functions that are sent to a mock
Connect server, and calls them. The mock now serves `GetWithDefault` config
requests. The test fails without this change.
- A new lib test checks the plan of `to_json_dataframe`.
- A probe of `foreach`, `foreachPartition` (with `binaryAsBytes` on and
off) and `toJSON` matches official PySpark 4.3.0.
- The lib tests and every integration test, e2e included, against a local
Spark 4.3.0 server (672 passed, and 129 of the transport crate), the build of
all tests with `wasm-udf`, the offline Python suites (1625 passed, 8 skipped),
and the live drop-in and Arrow suites (95 passed) and live cases (10 passed).
Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #220 from HyukjinKwon/SPARK-61014-foreach.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 56 ++++++++++------------------
crates/spark-connect/src/dataframe.rs | 41 ++++++++++++++------
python/pyspark/sql/_foreach.py | 70 +++++++++++++++++++++++++++++++++++
python/tests/_mock_connect_server.py | 20 ++++++++--
python/tests/test_dropin_e2e.py | 34 +++++++++++++++++
python/tests/test_dropin_offline.py | 46 +++++++++++++++++++++++
6 files changed, 215 insertions(+), 52 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index a15eb44..b1dd09b 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -675,14 +675,11 @@ impl PyDataFrame {
PyDataFrame::new(self.dataframe.drop_duplicates_within_watermark(refs))
}
- /// `toJSON()` returns an RDD in classic PySpark; Spark Connect has no
RDD, so
- /// the official Connect client raises NotImplementedError - we match that
exactly.
- /// (Use `df.select(to_json(struct("*")))` for a JSON-string DataFrame.)
+ /// `DataFrame.toJSON`, as PySpark Connect's: a DataFrame of each row as a
JSON object
+ /// string, in a column `value`.
#[pyo3(name = "toJSON")]
- fn to_json(&self) -> PyResult<()> {
- Err(PyErr::new::<pyo3::exceptions::PyNotImplementedError, _>(
- "toJSON() is not supported for Spark Connect (it returns an RDD)",
- ))
+ fn to_json(&self) -> PyDataFrame {
+ PyDataFrame::new(self.dataframe.to_json_dataframe())
}
/// Whether this DataFrame is semantically equal to `other`.
@@ -1704,18 +1701,24 @@ impl PyDataFrame {
Ok(PyDataFrame::new(self.dataframe.map_in_arrow(udf, barrier)))
}
- /// `DataFrame.foreach` — run a function per row for side effects.
- #[pyo3(name = "foreach")]
- fn foreach(&self, py: Python<'_>, f: Bound<'_, PyAny>) -> PyResult<()> {
- let udf = build_side_effect_udf(py, "foreach", &f)?;
- self.dataframe.foreach(udf).to_pyerr()
+ /// `DataFrame.foreach`, as PySpark Connect's: a UDF over a struct of the
columns of each
+ /// row calls `f` with the row. The UDF is Python, as it runs on the server
+ /// (`pyspark.sql._foreach`).
+ fn foreach(slf: &Bound<'_, Self>, f: &Bound<'_, PyAny>) -> PyResult<()> {
+ slf.py()
+ .import("pyspark.sql._foreach")?
+ .call_method1("foreach", (slf, f))?;
+ Ok(())
}
- /// `DataFrame.foreachPartition` — run a function per partition for side
effects.
+ /// `DataFrame.foreachPartition`, as PySpark Connect's: a `mapInArrow`
function calls `f`
+ /// with an iterator of the rows of each partition.
#[pyo3(name = "foreachPartition")]
- fn foreach_partition(&self, py: Python<'_>, f: Bound<'_, PyAny>) ->
PyResult<()> {
- let udf = build_side_effect_udf(py, "foreachPartition", &f)?;
- self.dataframe.foreach_partition(udf).to_pyerr()
+ fn foreach_partition(slf: &Bound<'_, Self>, f: &Bound<'_, PyAny>) ->
PyResult<()> {
+ slf.py()
+ .import("pyspark.sql._foreach")?
+ .call_method1("foreachPartition", (slf, f))?;
+ Ok(())
}
/// `DataFrame.nearestByJoin`.
@@ -2903,27 +2906,6 @@ pub(crate) fn build_map_udf(
))
}
-/// Build a side-effect UDF (foreach / foreachPartition): no output schema.
-fn build_side_effect_udf(
- py: Python<'_>,
- name: &str,
- func: &Bound<'_, PyAny>,
-) -> PyResult<CommonInlineUserDefinedFunctionExpression> {
- let command = py_cloudpickle(py, func)?;
- let payload = PythonUDFPayload::new(
- spark_connect::types::DataType::Struct { fields: vec![] },
- eval_type::SQL_MAP_ARROW_ITER_UDF,
- command,
- py_version(py),
- );
- Ok(CommonInlineUserDefinedFunctionExpression::new(
- name.to_string(),
- true,
- vec![],
- payload,
- ))
-}
-
/// The converters of the fields of a result of `schema` that deserialize its
values of
/// user-defined types, as PySpark does when it collects them
/// (`pyspark.sql.types._udt_field_converters`); None without a user-defined
type.
diff --git a/crates/spark-connect/src/dataframe.rs
b/crates/spark-connect/src/dataframe.rs
index 98c79e3..1a708b4 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1734,19 +1734,18 @@ impl DataFrame {
}
}
- /// Convert each row to a JSON object string, mirroring
`DataFrame.toJSON()`.
- ///
- /// Reference pyspark produces `{"col":val,...}` per row by applying the
server's
- /// `to_json(struct(*))`, not a client-side row rendering (which
previously emitted
- /// Rust list syntax like `[1, a]`). Build that projection and collect the
strings.
+ /// Each row as a JSON object string, in a column `value`, mirroring
+ /// `pyspark.sql.DataFrame.toJSON` of PySpark Connect, which selects
+ /// `to_json(struct(col("*"))).alias("value")`.
+ pub fn to_json_dataframe(&self) -> DataFrame {
+ use crate::functions::{col, r#struct, to_json};
+ self.select([to_json(r#struct([col("*")])).alias("value")])
+ }
+
+ /// Collect each row as a JSON object string, the rows of
+ /// [`DataFrame::to_json_dataframe`].
pub fn to_json(&self) -> Result<Vec<String>> {
- let cols: Vec<Column> = self
- .columns()?
- .iter()
- .map(|c| crate::column::col(c))
- .collect();
- let json_col =
crate::functions::to_json(crate::functions::r#struct(cols));
- let rows = self.select(vec![json_col]).collect()?;
+ let rows = self.to_json_dataframe().collect()?;
Ok(rows
.iter()
.map(|r| {
@@ -3786,6 +3785,24 @@ mod plan_construction_tests {
assert_eq!(plan_ids_of(&two.range(1).unwrap().relation()), vec![1]);
}
+ #[test]
+ fn to_json_dataframe_selects_the_json_of_a_struct_of_every_column() {
+ let spark = session();
+ let df = spark.range(3).unwrap().to_json_dataframe();
+ match df.plan.as_ref() {
+ LogicalPlan::Project { columns, .. } => {
+ let rendered: Vec<String> =
+ columns.iter().map(|c| c.expression().render()).collect();
+ // As PySpark Connect's repr of the column.
+ assert_eq!(
+ rendered,
+ vec!["to_json(struct(unresolvedstar())) AS
value".to_string()]
+ );
+ }
+ _ => panic!("expected Project plan"),
+ }
+ }
+
#[test]
fn with_metadata_plan() {
let spark = session();
diff --git a/python/pyspark/sql/_foreach.py b/python/pyspark/sql/_foreach.py
new file mode 100644
index 0000000..fa88023
--- /dev/null
+++ b/python/pyspark/sql/_foreach.py
@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""``DataFrame.foreach`` and ``DataFrame.foreachPartition``, as PySpark
Connect's.
+
+The functions that call ``f`` are pickled with cloudpickle and run by the
server's Python
+worker, so they are Python, as PySpark's ``pyspark.sql.connect.dataframe``
builds them.
+"""
+from typing import TYPE_CHECKING, Any, Callable, Iterator
+
+from pyspark.sql import functions as F
+from pyspark.sql.conversion import ArrowTableToRowsConversion
+from pyspark.sql.types import Row, StructType, _create_row
+
+if TYPE_CHECKING:
+ import pyarrow as pa
+
+ from pyspark.sql.dataframe import DataFrame
+
+
+def foreach(df: "DataFrame", f: Callable[[Row], None]) -> None:
+ def foreach_func(row: Any) -> None:
+ f(row)
+
+ df.select(F.struct(*df.schema.fieldNames()).alias("row")).select(
+ F.udf(foreach_func, StructType())("row") # type: ignore[arg-type]
+ ).collect()
+
+
+def foreachPartition(df: "DataFrame", f: Callable[[Iterator[Row]], None]) ->
None:
+ schema = df.schema
+ conf_value =
df.sparkSession.conf.get("spark.sql.execution.pyspark.binaryAsBytes", "true")
+ binary_as_bytes = conf_value is not None and conf_value.lower() == "true"
+ field_converters = [
+ ArrowTableToRowsConversion._create_converter(
+ f.dataType, none_on_identity=False, binary_as_bytes=binary_as_bytes
+ )
+ for f in schema.fields
+ ]
+
+ def foreach_partition_func(itr: Iterator["pa.RecordBatch"]) ->
Iterator["pa.RecordBatch"]:
+ def flatten() -> Iterator[Row]:
+ for table in itr:
+ columnar_data = [
+ ArrowTableToRowsConversion._to_pylist(column) for column
in table.columns
+ ]
+ for i in range(0, table.num_rows):
+ values = [
+ field_converters[j](columnar_data[j][i]) # type:
ignore[misc]
+ for j in range(table.num_columns)
+ ]
+ yield _create_row(fields=schema.fieldNames(),
values=values)
+
+ f(flatten())
+ return iter([])
+
+ df.mapInArrow(foreach_partition_func, schema=StructType()).collect()
diff --git a/python/tests/_mock_connect_server.py
b/python/tests/_mock_connect_server.py
index 53a5e6c..452e9c3 100644
--- a/python/tests/_mock_connect_server.py
+++ b/python/tests/_mock_connect_server.py
@@ -161,8 +161,9 @@ def serve(connection):
# FetchErrorDetails returns `error_details` (raw
FetchErrorDetailsResponse fields) and
# records the error id asked for. Config Get returns the values in
`confs`, or else
# those of `default_confs`, the defaults of SQL configs of the server,
and fails for a
- # key without one, as the server does for an unset config; it records
the keys. Config
- # Set records the pairs and the silent flag of each request.
+ # key without one, as the server does for an unset config; it records
the keys, as
+ # does GetWithDefault, which returns the values in `confs` or else the
defaults that
+ # it is given. Config Set records the pairs and the silent flag of
each request.
error_details = b""
fetched_error_ids = []
confs = {}
@@ -232,8 +233,21 @@ def serve(connection):
pairs.append((pair[1][0].decode(), pair[2][0].decode()))
self.set_confs.append((pairs, set_request.get(2, [None])[0]))
return message_field(1, request[1][0])
+ if 3 in operation:
+ # Operation.get_with_default=3: GetWithDefault.pairs=1
(KeyValue.key=1,
+ # value=2), the default of each key that `confs` does not have.
+ pairs = []
+ for pair in parse_message(operation[3][0]).get(1, []):
+ pair = parse_message(pair)
+ key = pair[1][0].decode()
+ self.read_confs.append(key)
+ value = self.confs.get(key, pair.get(2, [b""])[0].decode())
+ pairs.append(message_field(2, message_field(1, key) +
message_field(2, value)))
+ return message_field(1, request[1][0]) + b"".join(pairs)
if 2 not in operation:
- context.abort(grpc.StatusCode.UNIMPLEMENTED, "only Get and Set
are served")
+ context.abort(
+ grpc.StatusCode.UNIMPLEMENTED, "only Get, GetWithDefault
and Set are served"
+ )
keys = [key.decode() for key in
parse_message(operation[2][0]).get(1, [])]
self.read_confs.extend(keys)
confs = {**self.default_confs, **self.confs}
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 96045ca..a60a249 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -436,6 +436,40 @@ def test_add_artifacts_reach_udfs_as_pyspark(session,
tmp_path):
assert session.range(1).select(read("id")).first()[0] == "from the
module|file content|inside"
+def test_foreach_foreach_partition_and_to_json_as_pyspark(session):
+ # As PySpark Connect's: foreach calls f with each row, and
foreachPartition with an
+ # iterator of the rows of each partition, on the server; toJSON is a
DataFrame of each row
+ # as a JSON object string.
+ import re
+
+ from pyspark.errors import PythonException
+
+ df = session.createDataFrame(
+ [(1, "a", b"x"), (2, None, None)], "id int, name string, bin binary"
+ )
+
+ def each(row):
+ raise ValueError(f"row:{row!r}")
+
+ with pytest.raises(PythonException, match=re.escape("row:Row(id=1,
name='a', bin=b'x')")):
+ df.where("id = 1").foreach(each)
+
+ def partition(rows):
+ raise ValueError(f"rows:{list(rows)!r}")
+
+ with pytest.raises(
+ PythonException,
+ match=re.escape("rows:[Row(id=1, name='a', bin=b'x'), Row(id=2,
name=None, bin=None)]"),
+ ):
+ df.coalesce(1).foreachPartition(partition)
+ assert df.foreach(lambda row: None) is None
+ assert df.foreachPartition(lambda rows: None) is None
+
+ json = df.toJSON()
+ assert json.columns == ["value"]
+ assert json.collect() == [Row(value='{"id":1,"name":"a","bin":"eA=="}'),
Row(value='{"id":2}')]
+
+
def test_execution_info_is_that_of_each_dataframe_as_pyspark(session):
a = session.range(3)
assert a.executionInfo is None
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 9959071..1b9c68e 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -6979,6 +6979,52 @@ def
test_to_local_iterator_executes_with_its_first_row_as_pyspark(mock_connect):
assert service.execute_calls == 1
+def
test_foreach_and_foreach_partition_send_python_functions_as_pyspark(mock_connect):
+ # As PySpark Connect's: foreach selects a struct of the columns of each
row with a UDF of
+ # no result that calls f with the row, and foreachPartition runs a
mapInArrow function of
+ # no result that calls f with an iterator of the rows of each partition.
They run on the
+ # server, so this calls the functions that are sent.
+ import pyarrow as pa
+
+ from pyspark import cloudpickle
+
+ spark, service = mock_connect
+ df = spark.range(1)
+
+ def relation():
+ # ExecutePlanRequest.plan=3, Plan.root=1.
+ return
parse_message(parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0])
+
+ def raising(row):
+ raise ValueError(row)
+
+ df.foreach(raising)
+ # Relation.project=3, Project.expressions=3;
Expression.common_inline_user_defined_function
+ # =15; CommonInlineUserDefinedFunction.function_name=1, python_udf=4;
PythonUDF.command=3.
+ udf =
parse_message(parse_message(parse_message(relation()[3][0])[3][0])[15][0])
+ assert udf[1][0].decode() == "foreach_func"
+ func, return_type = cloudpickle.loads(parse_message(udf[4][0])[3][0])
+ assert return_type == T.StructType()
+ with pytest.raises(ValueError) as error:
+ func(Row(id=1))
+ assert error.value.args == (Row(id=1),)
+
+ def consuming(rows):
+ raise ValueError(list(rows))
+
+ df.foreachPartition(consuming)
+ # Relation.map_partitions=28, MapPartitions.func=2; PythonUDF.eval_type=2,
of
+ # SQL_MAP_ARROW_ITER_UDF.
+ udf = parse_message(parse_message(relation()[28][0])[2][0])
+ assert parse_message(udf[4][0])[2][0] == 207
+ func, return_type = cloudpickle.loads(parse_message(udf[4][0])[3][0])
+ assert return_type == T.StructType()
+ with pytest.raises(ValueError) as error:
+ func(iter([pa.RecordBatch.from_pydict({"id": [1, 2]})]))
+ assert error.value.args == ([Row(id=1), Row(id=2)],)
+ assert "spark.sql.execution.pyspark.binaryAsBytes" in service.read_confs
+
+
def test_execute_plan_error_carries_operation_id(mock_connect):
from pyspark.errors import PySparkException
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]