This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new efbd43e  [SPARK-61014][CONNECT][PYTHON] Run foreach and 
foreachPartition, and return the DataFrame of toJSON, as PySpark Connect does
efbd43e is described below

commit efbd43e3dfa32c9f7bc67ce964d371263f72392d
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 01:57:31 2026 +0900

    [SPARK-61014][CONNECT][PYTHON] Run foreach and foreachPartition, and return 
the DataFrame of toJSON, as PySpark Connect does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61014](https://issues.apache.org/jira/browse/SPARK-61014): `foreach` 
and `foreachPartition` now run as PySpark Connect's do, and `toJSON` returns a 
DataFrame.
    
    - `df.foreach(f)` selects a struct of the columns of each row, with a UDF 
of no result that calls `f` with the row.
    - `df.foreachPartition(f)` runs a `mapInArrow` function of no result that 
calls `f` with an iterator of the rows of each partition. It converts them with 
`spark.sql.execution.pyspark.binaryAsBytes`.
    - The functions are pickled and run by the server's Python worker, so they 
are Python, in `pyspark.sql._foreach`, as `pyspark.sql.connect.dataframe` 
builds them.
    - `df.toJSON()` returns the DataFrame of 
`select(to_json(struct(col("*"))).alias("value"))`.
    
    In the Rust API, `DataFrame::to_json_dataframe` builds the DataFrame of 
`toJSON`, and `to_json` collects its strings.
    
    ### Why are the changes needed?
    
    `foreach` and `foreachPartition` failed on the worker with `TypeError: 
cannot unpack non-iterable function object`, because the command of their UDF 
was the function rather than the pair of the function and its return type. 
`toJSON()` raised `NotImplementedError`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `foreach`, `foreachPartition` and `toJSON` work as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks the row that `foreach` gives `f` and the rows that 
`foreachPartition` gives it, by errors that `f` raises on the server. It also 
checks the DataFrame of `toJSON`. It passes with official PySpark 4.3.0, and 
fails without this change.
    - A new offline test unpickles the functions that are sent to a mock 
Connect server, and calls them. The mock now serves `GetWithDefault` config 
requests. The test fails without this change.
    - A new lib test checks the plan of `to_json_dataframe`.
    - A probe of `foreach`, `foreachPartition` (with `binaryAsBytes` on and 
off) and `toJSON` matches official PySpark 4.3.0.
    - The lib tests and every integration test, e2e included, against a local 
Spark 4.3.0 server (672 passed, and 129 of the transport crate), the build of 
all tests with `wasm-udf`, the offline Python suites (1625 passed, 8 skipped), 
and the live drop-in and Arrow suites (95 passed) and live cases (10 passed). 
Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #220 from HyukjinKwon/SPARK-61014-foreach.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs    | 56 ++++++++++------------------
 crates/spark-connect/src/dataframe.rs | 41 ++++++++++++++------
 python/pyspark/sql/_foreach.py        | 70 +++++++++++++++++++++++++++++++++++
 python/tests/_mock_connect_server.py  | 20 ++++++++--
 python/tests/test_dropin_e2e.py       | 34 +++++++++++++++++
 python/tests/test_dropin_offline.py   | 46 +++++++++++++++++++++++
 6 files changed, 215 insertions(+), 52 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index a15eb44..b1dd09b 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -675,14 +675,11 @@ impl PyDataFrame {
         PyDataFrame::new(self.dataframe.drop_duplicates_within_watermark(refs))
     }
 
-    /// `toJSON()` returns an RDD in classic PySpark; Spark Connect has no 
RDD, so
-    /// the official Connect client raises NotImplementedError - we match that 
exactly.
-    /// (Use `df.select(to_json(struct("*")))` for a JSON-string DataFrame.)
+    /// `DataFrame.toJSON`, as PySpark Connect's: a DataFrame of each row as a 
JSON object
+    /// string, in a column `value`.
     #[pyo3(name = "toJSON")]
-    fn to_json(&self) -> PyResult<()> {
-        Err(PyErr::new::<pyo3::exceptions::PyNotImplementedError, _>(
-            "toJSON() is not supported for Spark Connect (it returns an RDD)",
-        ))
+    fn to_json(&self) -> PyDataFrame {
+        PyDataFrame::new(self.dataframe.to_json_dataframe())
     }
 
     /// Whether this DataFrame is semantically equal to `other`.
@@ -1704,18 +1701,24 @@ impl PyDataFrame {
         Ok(PyDataFrame::new(self.dataframe.map_in_arrow(udf, barrier)))
     }
 
-    /// `DataFrame.foreach` — run a function per row for side effects.
-    #[pyo3(name = "foreach")]
-    fn foreach(&self, py: Python<'_>, f: Bound<'_, PyAny>) -> PyResult<()> {
-        let udf = build_side_effect_udf(py, "foreach", &f)?;
-        self.dataframe.foreach(udf).to_pyerr()
+    /// `DataFrame.foreach`, as PySpark Connect's: a UDF over a struct of the 
columns of each
+    /// row calls `f` with the row. The UDF is Python, as it runs on the server
+    /// (`pyspark.sql._foreach`).
+    fn foreach(slf: &Bound<'_, Self>, f: &Bound<'_, PyAny>) -> PyResult<()> {
+        slf.py()
+            .import("pyspark.sql._foreach")?
+            .call_method1("foreach", (slf, f))?;
+        Ok(())
     }
 
-    /// `DataFrame.foreachPartition` — run a function per partition for side 
effects.
+    /// `DataFrame.foreachPartition`, as PySpark Connect's: a `mapInArrow` 
function calls `f`
+    /// with an iterator of the rows of each partition.
     #[pyo3(name = "foreachPartition")]
-    fn foreach_partition(&self, py: Python<'_>, f: Bound<'_, PyAny>) -> 
PyResult<()> {
-        let udf = build_side_effect_udf(py, "foreachPartition", &f)?;
-        self.dataframe.foreach_partition(udf).to_pyerr()
+    fn foreach_partition(slf: &Bound<'_, Self>, f: &Bound<'_, PyAny>) -> 
PyResult<()> {
+        slf.py()
+            .import("pyspark.sql._foreach")?
+            .call_method1("foreachPartition", (slf, f))?;
+        Ok(())
     }
 
     /// `DataFrame.nearestByJoin`.
@@ -2903,27 +2906,6 @@ pub(crate) fn build_map_udf(
     ))
 }
 
-/// Build a side-effect UDF (foreach / foreachPartition): no output schema.
-fn build_side_effect_udf(
-    py: Python<'_>,
-    name: &str,
-    func: &Bound<'_, PyAny>,
-) -> PyResult<CommonInlineUserDefinedFunctionExpression> {
-    let command = py_cloudpickle(py, func)?;
-    let payload = PythonUDFPayload::new(
-        spark_connect::types::DataType::Struct { fields: vec![] },
-        eval_type::SQL_MAP_ARROW_ITER_UDF,
-        command,
-        py_version(py),
-    );
-    Ok(CommonInlineUserDefinedFunctionExpression::new(
-        name.to_string(),
-        true,
-        vec![],
-        payload,
-    ))
-}
-
 /// The converters of the fields of a result of `schema` that deserialize its 
values of
 /// user-defined types, as PySpark does when it collects them
 /// (`pyspark.sql.types._udt_field_converters`); None without a user-defined 
type.
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index 98c79e3..1a708b4 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1734,19 +1734,18 @@ impl DataFrame {
         }
     }
 
-    /// Convert each row to a JSON object string, mirroring 
`DataFrame.toJSON()`.
-    ///
-    /// Reference pyspark produces `{"col":val,...}` per row by applying the 
server's
-    /// `to_json(struct(*))`, not a client-side row rendering (which 
previously emitted
-    /// Rust list syntax like `[1, a]`). Build that projection and collect the 
strings.
+    /// Each row as a JSON object string, in a column `value`, mirroring
+    /// `pyspark.sql.DataFrame.toJSON` of PySpark Connect, which selects
+    /// `to_json(struct(col("*"))).alias("value")`.
+    pub fn to_json_dataframe(&self) -> DataFrame {
+        use crate::functions::{col, r#struct, to_json};
+        self.select([to_json(r#struct([col("*")])).alias("value")])
+    }
+
+    /// Collect each row as a JSON object string, the rows of
+    /// [`DataFrame::to_json_dataframe`].
     pub fn to_json(&self) -> Result<Vec<String>> {
-        let cols: Vec<Column> = self
-            .columns()?
-            .iter()
-            .map(|c| crate::column::col(c))
-            .collect();
-        let json_col = 
crate::functions::to_json(crate::functions::r#struct(cols));
-        let rows = self.select(vec![json_col]).collect()?;
+        let rows = self.to_json_dataframe().collect()?;
         Ok(rows
             .iter()
             .map(|r| {
@@ -3786,6 +3785,24 @@ mod plan_construction_tests {
         assert_eq!(plan_ids_of(&two.range(1).unwrap().relation()), vec![1]);
     }
 
+    #[test]
+    fn to_json_dataframe_selects_the_json_of_a_struct_of_every_column() {
+        let spark = session();
+        let df = spark.range(3).unwrap().to_json_dataframe();
+        match df.plan.as_ref() {
+            LogicalPlan::Project { columns, .. } => {
+                let rendered: Vec<String> =
+                    columns.iter().map(|c| c.expression().render()).collect();
+                // As PySpark Connect's repr of the column.
+                assert_eq!(
+                    rendered,
+                    vec!["to_json(struct(unresolvedstar())) AS 
value".to_string()]
+                );
+            }
+            _ => panic!("expected Project plan"),
+        }
+    }
+
     #[test]
     fn with_metadata_plan() {
         let spark = session();
diff --git a/python/pyspark/sql/_foreach.py b/python/pyspark/sql/_foreach.py
new file mode 100644
index 0000000..fa88023
--- /dev/null
+++ b/python/pyspark/sql/_foreach.py
@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""``DataFrame.foreach`` and ``DataFrame.foreachPartition``, as PySpark 
Connect's.
+
+The functions that call ``f`` are pickled with cloudpickle and run by the 
server's Python
+worker, so they are Python, as PySpark's ``pyspark.sql.connect.dataframe`` 
builds them.
+"""
+from typing import TYPE_CHECKING, Any, Callable, Iterator
+
+from pyspark.sql import functions as F
+from pyspark.sql.conversion import ArrowTableToRowsConversion
+from pyspark.sql.types import Row, StructType, _create_row
+
+if TYPE_CHECKING:
+    import pyarrow as pa
+
+    from pyspark.sql.dataframe import DataFrame
+
+
+def foreach(df: "DataFrame", f: Callable[[Row], None]) -> None:
+    def foreach_func(row: Any) -> None:
+        f(row)
+
+    df.select(F.struct(*df.schema.fieldNames()).alias("row")).select(
+        F.udf(foreach_func, StructType())("row")  # type: ignore[arg-type]
+    ).collect()
+
+
+def foreachPartition(df: "DataFrame", f: Callable[[Iterator[Row]], None]) -> 
None:
+    schema = df.schema
+    conf_value = 
df.sparkSession.conf.get("spark.sql.execution.pyspark.binaryAsBytes", "true")
+    binary_as_bytes = conf_value is not None and conf_value.lower() == "true"
+    field_converters = [
+        ArrowTableToRowsConversion._create_converter(
+            f.dataType, none_on_identity=False, binary_as_bytes=binary_as_bytes
+        )
+        for f in schema.fields
+    ]
+
+    def foreach_partition_func(itr: Iterator["pa.RecordBatch"]) -> 
Iterator["pa.RecordBatch"]:
+        def flatten() -> Iterator[Row]:
+            for table in itr:
+                columnar_data = [
+                    ArrowTableToRowsConversion._to_pylist(column) for column 
in table.columns
+                ]
+                for i in range(0, table.num_rows):
+                    values = [
+                        field_converters[j](columnar_data[j][i])  # type: 
ignore[misc]
+                        for j in range(table.num_columns)
+                    ]
+                    yield _create_row(fields=schema.fieldNames(), 
values=values)
+
+        f(flatten())
+        return iter([])
+
+    df.mapInArrow(foreach_partition_func, schema=StructType()).collect()
diff --git a/python/tests/_mock_connect_server.py 
b/python/tests/_mock_connect_server.py
index 53a5e6c..452e9c3 100644
--- a/python/tests/_mock_connect_server.py
+++ b/python/tests/_mock_connect_server.py
@@ -161,8 +161,9 @@ def serve(connection):
         # FetchErrorDetails returns `error_details` (raw 
FetchErrorDetailsResponse fields) and
         # records the error id asked for. Config Get returns the values in 
`confs`, or else
         # those of `default_confs`, the defaults of SQL configs of the server, 
and fails for a
-        # key without one, as the server does for an unset config; it records 
the keys. Config
-        # Set records the pairs and the silent flag of each request.
+        # key without one, as the server does for an unset config; it records 
the keys, as
+        # does GetWithDefault, which returns the values in `confs` or else the 
defaults that
+        # it is given. Config Set records the pairs and the silent flag of 
each request.
         error_details = b""
         fetched_error_ids = []
         confs = {}
@@ -232,8 +233,21 @@ def serve(connection):
                     pairs.append((pair[1][0].decode(), pair[2][0].decode()))
                 self.set_confs.append((pairs, set_request.get(2, [None])[0]))
                 return message_field(1, request[1][0])
+            if 3 in operation:
+                # Operation.get_with_default=3: GetWithDefault.pairs=1 
(KeyValue.key=1,
+                # value=2), the default of each key that `confs` does not have.
+                pairs = []
+                for pair in parse_message(operation[3][0]).get(1, []):
+                    pair = parse_message(pair)
+                    key = pair[1][0].decode()
+                    self.read_confs.append(key)
+                    value = self.confs.get(key, pair.get(2, [b""])[0].decode())
+                    pairs.append(message_field(2, message_field(1, key) + 
message_field(2, value)))
+                return message_field(1, request[1][0]) + b"".join(pairs)
             if 2 not in operation:
-                context.abort(grpc.StatusCode.UNIMPLEMENTED, "only Get and Set 
are served")
+                context.abort(
+                    grpc.StatusCode.UNIMPLEMENTED, "only Get, GetWithDefault 
and Set are served"
+                )
             keys = [key.decode() for key in 
parse_message(operation[2][0]).get(1, [])]
             self.read_confs.extend(keys)
             confs = {**self.default_confs, **self.confs}
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 96045ca..a60a249 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -436,6 +436,40 @@ def test_add_artifacts_reach_udfs_as_pyspark(session, 
tmp_path):
     assert session.range(1).select(read("id")).first()[0] == "from the 
module|file content|inside"
 
 
+def test_foreach_foreach_partition_and_to_json_as_pyspark(session):
+    # As PySpark Connect's: foreach calls f with each row, and 
foreachPartition with an
+    # iterator of the rows of each partition, on the server; toJSON is a 
DataFrame of each row
+    # as a JSON object string.
+    import re
+
+    from pyspark.errors import PythonException
+
+    df = session.createDataFrame(
+        [(1, "a", b"x"), (2, None, None)], "id int, name string, bin binary"
+    )
+
+    def each(row):
+        raise ValueError(f"row:{row!r}")
+
+    with pytest.raises(PythonException, match=re.escape("row:Row(id=1, 
name='a', bin=b'x')")):
+        df.where("id = 1").foreach(each)
+
+    def partition(rows):
+        raise ValueError(f"rows:{list(rows)!r}")
+
+    with pytest.raises(
+        PythonException,
+        match=re.escape("rows:[Row(id=1, name='a', bin=b'x'), Row(id=2, 
name=None, bin=None)]"),
+    ):
+        df.coalesce(1).foreachPartition(partition)
+    assert df.foreach(lambda row: None) is None
+    assert df.foreachPartition(lambda rows: None) is None
+
+    json = df.toJSON()
+    assert json.columns == ["value"]
+    assert json.collect() == [Row(value='{"id":1,"name":"a","bin":"eA=="}'), 
Row(value='{"id":2}')]
+
+
 def test_execution_info_is_that_of_each_dataframe_as_pyspark(session):
     a = session.range(3)
     assert a.executionInfo is None
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 9959071..1b9c68e 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -6979,6 +6979,52 @@ def 
test_to_local_iterator_executes_with_its_first_row_as_pyspark(mock_connect):
     assert service.execute_calls == 1
 
 
+def 
test_foreach_and_foreach_partition_send_python_functions_as_pyspark(mock_connect):
+    # As PySpark Connect's: foreach selects a struct of the columns of each 
row with a UDF of
+    # no result that calls f with the row, and foreachPartition runs a 
mapInArrow function of
+    # no result that calls f with an iterator of the rows of each partition. 
They run on the
+    # server, so this calls the functions that are sent.
+    import pyarrow as pa
+
+    from pyspark import cloudpickle
+
+    spark, service = mock_connect
+    df = spark.range(1)
+
+    def relation():
+        # ExecutePlanRequest.plan=3, Plan.root=1.
+        return 
parse_message(parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0])
+
+    def raising(row):
+        raise ValueError(row)
+
+    df.foreach(raising)
+    # Relation.project=3, Project.expressions=3; 
Expression.common_inline_user_defined_function
+    # =15; CommonInlineUserDefinedFunction.function_name=1, python_udf=4; 
PythonUDF.command=3.
+    udf = 
parse_message(parse_message(parse_message(relation()[3][0])[3][0])[15][0])
+    assert udf[1][0].decode() == "foreach_func"
+    func, return_type = cloudpickle.loads(parse_message(udf[4][0])[3][0])
+    assert return_type == T.StructType()
+    with pytest.raises(ValueError) as error:
+        func(Row(id=1))
+    assert error.value.args == (Row(id=1),)
+
+    def consuming(rows):
+        raise ValueError(list(rows))
+
+    df.foreachPartition(consuming)
+    # Relation.map_partitions=28, MapPartitions.func=2; PythonUDF.eval_type=2, 
of
+    # SQL_MAP_ARROW_ITER_UDF.
+    udf = parse_message(parse_message(relation()[28][0])[2][0])
+    assert parse_message(udf[4][0])[2][0] == 207
+    func, return_type = cloudpickle.loads(parse_message(udf[4][0])[3][0])
+    assert return_type == T.StructType()
+    with pytest.raises(ValueError) as error:
+        func(iter([pa.RecordBatch.from_pydict({"id": [1, 2]})]))
+    assert error.value.args == ([Row(id=1), Row(id=2)],)
+    assert "spark.sql.execution.pyspark.binaryAsBytes" in service.read_confs
+
+
 def test_execute_plan_error_carries_operation_id(mock_connect):
     from pyspark.errors import PySparkException
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to