This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 2c4d8558 [SPARK-61189][CONNECT][PYTHON] Send the metadata edited in 
place of a StructType in the schemas of mapInPandas, applyInPandas and the 
other map functions, in the Python client as PySpark does
2c4d8558 is described below

commit 2c4d855800f3db8df7537e8182073bf0fabf6ed4
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 19:51:58 2026 +0900

    [SPARK-61189][CONNECT][PYTHON] Send the metadata edited in place of a 
StructType in the schemas of mapInPandas, applyInPandas and the other map 
functions, in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61189](https://issues.apache.org/jira/browse/SPARK-61189): Send the 
metadata edited in place of a StructType in the schemas of mapInPandas, 
mapInArrow, applyInPandas and transformWithState, as PySpark does.
    
    `resolve_datatype` read the base snapshot of a `StructType` (refreshed only 
by `add`), so an edit of a field's metadata after the struct was built, such as 
`st["a"].metadata["comment"] = "the a"`, was not sent to the server. It now 
converts the schema through `py_to_data_type`, which reads the live fields of a 
`StructType`, as the other schema arguments already do. Official PySpark keeps 
the edit in the schemas of these functions.
    
    ### Why are the changes needed?
    
    With `st = StructType([StructField("a", IntegerType())]); 
st["a"].metadata["comment"] = "the a"`, `df.mapInPandas(f, 
st).schema["a"].metadata` was `{}` (and `applyInPandas` and `mapInArrow` 
likewise), where official PySpark gives `{'comment': 'the a'}`. The data is 
unaffected; only the schema metadata differs.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. The schemas of mapInPandas, mapInArrow, applyInPandas and 
transformWithState carry the metadata edited in place after a StructType was 
built.
    
    ### How was this patch tested?
    
    - `test_map_schemas_keep_the_metadata_edited_in_place_as_pyspark` (offline, 
mock server): reads the schema sent for mapInPandas, mapInArrow and 
applyInPandas, for a struct edited after construction and after `add`, and 
checks the field metadata in the proto. Fails without the fix.
    - 
`test_map_schemas_keep_the_metadata_edited_in_place_on_the_server_as_pyspark` 
(live): the result schema of mapInPandas, mapInArrow and applyInPandas has the 
edited metadata. Fails without the fix.
    - The same cases give the same metadata with official PySpark 4.3.0.
    - The lib tests of both crates (599 passed), the build of all tests, also 
with `wasm-udf`, and the offline Python suites (2228 passed, 8 skipped), and 
the live tests of the map, grouped and stateful functions, schemas and metadata 
(45 passed) against a local Spark 4.3.0 server; rustfmt passes, and clippy 
reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #369 from HyukjinKwon/SPARK-61189-map-schemas-live-metadata.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  |  6 ++++--
 python/tests/test_dropin_e2e.py     | 13 +++++++++++++
 python/tests/test_dropin_offline.py | 39 +++++++++++++++++++++++++++++++++++++
 3 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 68bf6924..670766ed 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -3919,8 +3919,10 @@ pub(crate) fn resolve_datatype(
     session: &spark_connect::session::SparkSession,
     schema: &Bound<'_, PyAny>,
 ) -> PyResult<spark_connect::types::DataType> {
-    if let Ok(dt) = schema.extract::<PyRef<PyDataType>>() {
-        return Ok(dt.inner.clone());
+    // The live conversion, not the base snapshot: an edit in place of a 
StructType's fields or
+    // metadata after it was built is kept, as PySpark does.
+    if schema.extract::<PyRef<PyDataType>>().is_ok() {
+        return crate::types::py_to_data_type(schema);
     }
     if schema.is_instance_of::<pyo3::types::PyString>() {
         return crate::types::schema_data_type(py, session, schema);
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index aea3b595..65b4a85e 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -592,6 +592,19 @@ def 
test_udtf_handlers_that_capture_analysis_columns_run_as_pyspark(session):
         Partitioned().collect()
 
 
+def 
test_map_schemas_keep_the_metadata_edited_in_place_on_the_server_as_pyspark(session):
+    # As PySpark's: the schema of mapInPandas, mapInArrow and applyInPandas is 
the StructType
+    # with the metadata edited in place after it was built, so the result has 
it.
+    st = StructType([StructField("a", IntegerType()), StructField("b", 
StringType())])
+    st["a"].metadata["comment"] = "the a"
+    df = session.createDataFrame([(1, "x")], "a int, b string")
+    assert df.mapInPandas(lambda it: it, st).schema["a"].metadata == 
{"comment": "the a"}
+    assert df.mapInArrow(lambda it: it, st).schema["a"].metadata == 
{"comment": "the a"}
+    grouped = df.groupBy("a")
+    assert grouped.applyInPandas(lambda pdf: pdf, st).schema["a"].metadata == {
+        "comment": "the a"}
+
+
 def test_arrow_udtf_takes_and_returns_arrow_data_as_pyspark(session):
     # As PySpark's arrow_udtf(), a PyArrow-native UDTF: eval() takes a pyarrow 
array of the
     # argument, and returns pyarrow tables.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 3c7c9695..1d5f3d80 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -602,6 +602,45 @@ def 
test_data_types_convert_values_to_and_from_internal_as_pyspark(monkeypatch):
 
 
 
+def 
test_map_schemas_keep_the_metadata_edited_in_place_as_pyspark(mock_connect):
+    # As PySpark's, the schema that mapInPandas, mapInArrow and applyInPandas 
send is the
+    # StructType with the metadata of its fields as they are when the call is 
made, including
+    # an edit in place (st["a"].metadata[...] = ...) after the struct was 
built.
+    spark, service = mock_connect
+
+    def sent_fields(dataframe, relation_field, func_field):
+        # ExecutePlanRequest.plan=3, Plan.root=1; Relation.map_partitions=28 
(MapPartitions.func
+        # =2) or group_map=31 (GroupMap.func=3); 
CommonInlineUserDefinedFunction.python_udf=4,
+        # PythonUDF.output_type=1, DataType.struct=21, Struct.fields=1, 
StructField.name=1 and
+        # metadata=4.
+        dataframe.collect()
+        plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+        relation = parse_message(parse_message(plan)[relation_field][0])
+        udf = parse_message(parse_message(relation[func_field][0])[4][0])
+        struct = parse_message(parse_message(udf[1][0])[21][0])
+        fields = [parse_message(field) for field in struct[1]]
+        return [(f[1][0].decode(), json.loads(f[4][0]) if 4 in f else {}) for 
f in fields]
+
+    def edited():
+        st = T.StructType([T.StructField("a", T.IntegerType()), 
T.StructField("b", T.StringType())])
+        st["a"].metadata["comment"] = "the a"
+        return st
+
+    def added():
+        st = T.StructType().add("a", T.IntegerType()).add("b", T.StringType())
+        st["a"].metadata["comment"] = "the a"
+        return st
+
+    df = spark.range(1)
+    expected = [("a", {"comment": "the a"}), ("b", {})]
+    assert sent_fields(df.mapInPandas(lambda it: it, edited()), 28, 2) == 
expected
+    assert sent_fields(df.mapInArrow(lambda it: it, edited()), 28, 2) == 
expected
+    grouped = df.groupBy("id")
+    assert sent_fields(grouped.applyInPandas(lambda pdf: pdf, edited()), 31, 
3) == expected
+    assert sent_fields(df.mapInPandas(lambda it: it, added()), 28, 2) == 
expected
+    assert sent_fields(grouped.applyInPandas(lambda pdf: pdf, added()), 31, 3) 
== expected
+
+
 def test_datatype_eq_and_hash_match_pyspark():
     cases = _eq_cases()
     left = [f() for f in cases]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to