This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 2c4d8558 [SPARK-61189][CONNECT][PYTHON] Send the metadata edited in
place of a StructType in the schemas of mapInPandas, applyInPandas and the
other map functions, in the Python client as PySpark does
2c4d8558 is described below
commit 2c4d855800f3db8df7537e8182073bf0fabf6ed4
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 19:51:58 2026 +0900
[SPARK-61189][CONNECT][PYTHON] Send the metadata edited in place of a
StructType in the schemas of mapInPandas, applyInPandas and the other map
functions, in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61189](https://issues.apache.org/jira/browse/SPARK-61189): Send the
metadata edited in place of a StructType in the schemas of mapInPandas,
mapInArrow, applyInPandas and transformWithState, as PySpark does.
`resolve_datatype` read the base snapshot of a `StructType` (refreshed only
by `add`), so an edit of a field's metadata after the struct was built, such as
`st["a"].metadata["comment"] = "the a"`, was not sent to the server. It now
converts the schema through `py_to_data_type`, which reads the live fields of a
`StructType`, as the other schema arguments already do. Official PySpark keeps
the edit in the schemas of these functions.
### Why are the changes needed?
With `st = StructType([StructField("a", IntegerType())]);
st["a"].metadata["comment"] = "the a"`, `df.mapInPandas(f,
st).schema["a"].metadata` was `{}` (and `applyInPandas` and `mapInArrow`
likewise), where official PySpark gives `{'comment': 'the a'}`. The data is
unaffected; only the schema metadata differs.
### Does this PR introduce _any_ user-facing change?
Yes. The schemas of mapInPandas, mapInArrow, applyInPandas and
transformWithState carry the metadata edited in place after a StructType was
built.
### How was this patch tested?
- `test_map_schemas_keep_the_metadata_edited_in_place_as_pyspark` (offline,
mock server): reads the schema sent for mapInPandas, mapInArrow and
applyInPandas, for a struct edited after construction and after `add`, and
checks the field metadata in the proto. Fails without the fix.
-
`test_map_schemas_keep_the_metadata_edited_in_place_on_the_server_as_pyspark`
(live): the result schema of mapInPandas, mapInArrow and applyInPandas has the
edited metadata. Fails without the fix.
- The same cases give the same metadata with official PySpark 4.3.0.
- The lib tests of both crates (599 passed), the build of all tests, also
with `wasm-udf`, and the offline Python suites (2228 passed, 8 skipped), and
the live tests of the map, grouped and stateful functions, schemas and metadata
(45 passed) against a local Spark 4.3.0 server; rustfmt passes, and clippy
reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #369 from HyukjinKwon/SPARK-61189-map-schemas-live-metadata.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 6 ++++--
python/tests/test_dropin_e2e.py | 13 +++++++++++++
python/tests/test_dropin_offline.py | 39 +++++++++++++++++++++++++++++++++++++
3 files changed, 56 insertions(+), 2 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 68bf6924..670766ed 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -3919,8 +3919,10 @@ pub(crate) fn resolve_datatype(
session: &spark_connect::session::SparkSession,
schema: &Bound<'_, PyAny>,
) -> PyResult<spark_connect::types::DataType> {
- if let Ok(dt) = schema.extract::<PyRef<PyDataType>>() {
- return Ok(dt.inner.clone());
+ // The live conversion, not the base snapshot: an edit in place of a
StructType's fields or
+ // metadata after it was built is kept, as PySpark does.
+ if schema.extract::<PyRef<PyDataType>>().is_ok() {
+ return crate::types::py_to_data_type(schema);
}
if schema.is_instance_of::<pyo3::types::PyString>() {
return crate::types::schema_data_type(py, session, schema);
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index aea3b595..65b4a85e 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -592,6 +592,19 @@ def
test_udtf_handlers_that_capture_analysis_columns_run_as_pyspark(session):
Partitioned().collect()
+def
test_map_schemas_keep_the_metadata_edited_in_place_on_the_server_as_pyspark(session):
+ # As PySpark's: the schema of mapInPandas, mapInArrow and applyInPandas is
the StructType
+ # with the metadata edited in place after it was built, so the result has
it.
+ st = StructType([StructField("a", IntegerType()), StructField("b",
StringType())])
+ st["a"].metadata["comment"] = "the a"
+ df = session.createDataFrame([(1, "x")], "a int, b string")
+ assert df.mapInPandas(lambda it: it, st).schema["a"].metadata ==
{"comment": "the a"}
+ assert df.mapInArrow(lambda it: it, st).schema["a"].metadata ==
{"comment": "the a"}
+ grouped = df.groupBy("a")
+ assert grouped.applyInPandas(lambda pdf: pdf, st).schema["a"].metadata == {
+ "comment": "the a"}
+
+
def test_arrow_udtf_takes_and_returns_arrow_data_as_pyspark(session):
# As PySpark's arrow_udtf(), a PyArrow-native UDTF: eval() takes a pyarrow
array of the
# argument, and returns pyarrow tables.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 3c7c9695..1d5f3d80 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -602,6 +602,45 @@ def
test_data_types_convert_values_to_and_from_internal_as_pyspark(monkeypatch):
+def
test_map_schemas_keep_the_metadata_edited_in_place_as_pyspark(mock_connect):
+ # As PySpark's, the schema that mapInPandas, mapInArrow and applyInPandas
send is the
+ # StructType with the metadata of its fields as they are when the call is
made, including
+ # an edit in place (st["a"].metadata[...] = ...) after the struct was
built.
+ spark, service = mock_connect
+
+ def sent_fields(dataframe, relation_field, func_field):
+ # ExecutePlanRequest.plan=3, Plan.root=1; Relation.map_partitions=28
(MapPartitions.func
+ # =2) or group_map=31 (GroupMap.func=3);
CommonInlineUserDefinedFunction.python_udf=4,
+ # PythonUDF.output_type=1, DataType.struct=21, Struct.fields=1,
StructField.name=1 and
+ # metadata=4.
+ dataframe.collect()
+ plan =
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+ relation = parse_message(parse_message(plan)[relation_field][0])
+ udf = parse_message(parse_message(relation[func_field][0])[4][0])
+ struct = parse_message(parse_message(udf[1][0])[21][0])
+ fields = [parse_message(field) for field in struct[1]]
+ return [(f[1][0].decode(), json.loads(f[4][0]) if 4 in f else {}) for
f in fields]
+
+ def edited():
+ st = T.StructType([T.StructField("a", T.IntegerType()),
T.StructField("b", T.StringType())])
+ st["a"].metadata["comment"] = "the a"
+ return st
+
+ def added():
+ st = T.StructType().add("a", T.IntegerType()).add("b", T.StringType())
+ st["a"].metadata["comment"] = "the a"
+ return st
+
+ df = spark.range(1)
+ expected = [("a", {"comment": "the a"}), ("b", {})]
+ assert sent_fields(df.mapInPandas(lambda it: it, edited()), 28, 2) ==
expected
+ assert sent_fields(df.mapInArrow(lambda it: it, edited()), 28, 2) ==
expected
+ grouped = df.groupBy("id")
+ assert sent_fields(grouped.applyInPandas(lambda pdf: pdf, edited()), 31,
3) == expected
+ assert sent_fields(df.mapInPandas(lambda it: it, added()), 28, 2) ==
expected
+ assert sent_fields(grouped.applyInPandas(lambda pdf: pdf, added()), 31, 3)
== expected
+
+
def test_datatype_eq_and_hash_match_pyspark():
cases = _eq_cases()
left = [f() for f in cases]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]