This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 689bc53  [SPARK-61072][CONNECT][PYTHON] Convert a schema to DDL on the 
server in StructType.toDDL in the Python client as PySpark does
689bc53 is described below

commit 689bc5340a56142648bc97af4d622480afab142c
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 14:57:09 2026 +0900

    [SPARK-61072][CONNECT][PYTHON] Convert a schema to DDL on the server in 
StructType.toDDL in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61072](https://issues.apache.org/jira/browse/SPARK-61072): 
`StructType.toDDL()` now asks the server to convert the schema to DDL, as 
PySpark Connect does.
    
    PySpark Connect sends the JSON of the schema to the server of the active 
session, as the `JsonToDDL` of an `AnalyzePlanRequest`. So `toDDL()` returns 
the DDL of Spark, which quotes the names that need it and parses back:
    
    ```
    a INT,`b b` STRING NOT NULL,m MAP<STRING, STRUCT<x: DECIMAL(10,2)>>,c 
STRING COMMENT 'the c'
    ```
    
    Without an active session, it raises an `AssertionError`, as PySpark's does.
    
    The core adds `SparkSession::json_to_ddl`, and the session of the Python 
client adds `_to_ddl`, as PySpark Connect's.
    
    ### Why are the changes needed?
    
    The client wrote the DDL itself, as `a int,b b string NOT NULL,...`. It did 
not quote names, so the DDL of a name with a space did not parse back. It also 
wrote the types in lowercase, unlike Spark.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `StructType.toDDL()` returns the DDL of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks the DDL of a schema with a quoted name, `NOT 
NULL`, a nested map and struct, and a `COMMENT`, and parses it back with 
`createDataFrame`. It passes with official PySpark 4.3.0 and fails without this 
change.
    - A new offline test checks the JSON that `toDDL()` sends to a mock Connect 
server, and the error without an active session. The mock server now answers 
`JsonToDDL` requests. Two older offline tests checked the DDL that the client 
wrote, without a session; their `toDDL` checks are removed.
    - A new e2e test checks `SparkSession::json_to_ddl` against the server.
    - On a local server, 11 schemas give the same DDL as official PySpark 
4.3.0, including collations, char and varchar, intervals, variant, time, and an 
empty schema.
    - The lib tests of both crates (589 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1978 passed, 8 skipped), and the 
live drop-in and Arrow suites (134 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #275 from HyukjinKwon/SPARK-61072-to-ddl.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/session.rs                  | 10 ++++++
 crates/pyspark-rs/src/types.rs                    | 17 ++++++----
 crates/spark-connect/src/session.rs               | 22 +++++++++++++
 crates/spark-connect/tests/e2e_session_catalog.rs | 38 +++++++++++++++++++++++
 python/tests/_mock_connect_server.py              |  7 +++++
 python/tests/test_dropin_e2e.py                   | 21 +++++++++++++
 python/tests/test_dropin_offline.py               | 26 ++++++++++++----
 7 files changed, 129 insertions(+), 12 deletions(-)

diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 2c3063d..abf51d6 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1729,6 +1729,16 @@ impl PySparkSession {
         set_default_and_active(py, &session);
     }
 
+    /// The DDL of the schema `struct`, as PySpark Connect's `_to_ddl`: the 
server converts
+    /// its JSON.
+    #[pyo3(name = "_to_ddl")]
+    fn to_ddl(&self, py: Python<'_>, r#struct: &Bound<'_, PyAny>) -> 
PyResult<String> {
+        let json: String = r#struct.call_method0("json")?.extract()?;
+        let session = self.session.clone();
+        py.detach(|| session.json_to_ddl(&json).map_err(Box::new))
+            .to_pyerr()
+    }
+
     /// The tags of the current thread, a set, as PySpark's `getTags`.
     #[pyo3(name = "getTags")]
     fn get_tags<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, 
pyo3::types::PySet>> {
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 6b16bf1..201077e 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -1872,14 +1872,19 @@ impl PyStructType {
         self.fields.iter().map(|f| f.name.clone()).collect()
     }
 
-    /// DDL string. Mirrors `StructType.toDDL()`.
+    /// DDL string. Mirrors `StructType.toDDL()`, which asks the server of the 
active session to
+    /// convert the JSON of the schema, as Spark writes DDL.
     #[pyo3(name = "toDDL")]
-    fn __st_to_ddl(&self) -> PyResult<String> {
-        DataType::Struct {
-            fields: self.fields.clone(),
+    fn __st_to_ddl(slf: &Bound<'_, Self>) -> PyResult<String> {
+        let py = slf.py();
+        let session = py
+            .import("pyspark.sql")?
+            .getattr("SparkSession")?
+            .call_method0("getActiveSession")?;
+        if session.is_none() {
+            return Err(pyo3::exceptions::PyAssertionError::new_err(()));
         }
-        .to_ddl()
-        .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
+        session.call_method1("_to_ddl", (slf,))?.extract()
     }
 
     /// Tree string. Mirrors `StructType.treeString(maxDepth)`.
diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index 0b57655..ce7b3a4 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -331,6 +331,28 @@ impl SparkSession {
         }
     }
 
+    /// Convert the JSON of a schema to DDL on the server, mirroring PySpark 
Connect's
+    /// `_to_ddl`: the `JsonToDDL` of an `AnalyzePlanRequest`, as Spark writes 
it.
+    pub fn json_to_ddl(&self, json: &str) -> Result<String> {
+        let request = proto::AnalyzePlanRequest {
+            session_id: self.client().session_id().to_string(),
+            user_context: Some(proto::UserContext::default()),
+            analyze: Some(proto::analyze_plan_request::Analyze::JsonToDdl(
+                proto::analyze_plan_request::JsonToDdl {
+                    json_string: json.to_string(),
+                },
+            )),
+            ..Default::default()
+        };
+        let response = block_on(self.client().analyze_plan(request))?;
+        match response.result {
+            Some(proto::analyze_plan_response::Result::JsonToDdl(r)) => 
Ok(r.ddl_string),
+            _ => Err(SparkError::connect_msg(
+                "AnalyzePlan response did not contain a DDL string",
+            )),
+        }
+    }
+
     /// Execute a SQL statement and return a DataFrame.
     ///
     /// Mirrors `pyspark.sql.SparkSession.sql(sqlQuery)`: the statement is 
sent at once, so a
diff --git a/crates/spark-connect/tests/e2e_session_catalog.rs 
b/crates/spark-connect/tests/e2e_session_catalog.rs
index d1e1616..a548c32 100644
--- a/crates/spark-connect/tests/e2e_session_catalog.rs
+++ b/crates/spark-connect/tests/e2e_session_catalog.rs
@@ -380,3 +380,41 @@ fn parse_ddl_parses_on_the_server() {
     assert_eq!(error.error_class, "PARSE_SYNTAX_ERROR");
     let _ = spark.stop();
 }
+
+/// `json_to_ddl` converts the JSON of a schema to DDL on the server, as 
PySpark Connect's
+/// `_to_ddl`: the DDL of Spark, with its quoted names, NOT NULL and COMMENT.
+#[test]
+fn json_to_ddl_converts_on_the_server() {
+    if !should_run() {
+        return;
+    }
+    use spark_connect::types::{DataType, StructField};
+    let spark = session();
+    let field = |name: &str, data_type: DataType, nullable: bool| StructField {
+        name: name.to_string(),
+        data_type,
+        nullable,
+        metadata: Default::default(),
+    };
+    let mut commented = field("c", DataType::Long, true);
+    commented.metadata.insert(
+        "comment".to_string(),
+        serde_json::Value::String("the c".to_string()),
+    );
+    let schema = DataType::Struct {
+        fields: vec![
+            field("a", DataType::Integer, true),
+            field(
+                "b b",
+                DataType::String {
+                    collation: "UTF8_BINARY".to_string(),
+                },
+                false,
+            ),
+            commented,
+        ],
+    };
+    let ddl = spark.json_to_ddl(&schema.json()).expect("ddl");
+    assert_eq!(ddl, "a INT,`b b` STRING NOT NULL,c BIGINT COMMENT 'the c'");
+    let _ = spark.stop();
+}
diff --git a/python/tests/_mock_connect_server.py 
b/python/tests/_mock_connect_server.py
index 1699501..ec2c4da 100644
--- a/python/tests/_mock_connect_server.py
+++ b/python/tests/_mock_connect_server.py
@@ -282,6 +282,8 @@ def serve(connection):
             self.read_confs = []
             self.set_confs = []
             self.clone_requests = []
+            # The DDL of the JSON of each JsonToDDL.
+            self.ddl_string = ""
 
         def analyze(self, request, context):
             self.calls += 1
@@ -305,6 +307,11 @@ def serve(connection):
                 ddl = 
parse_message(parse_message(request)[11][0])[1][0].decode()
                 parsed = ddl_data_type(ddl) or self.data_type
                 return message_field(9, message_field(1, parsed))
+            # base.proto: AnalyzePlanRequest.json_to_ddl=18 -> 
JsonToDDL.json_string=1;
+            # AnalyzePlanResponse.json_to_ddl=16 -> JsonToDDL.ddl_string=1: 
`ddl_string`, which
+            # a test sets.
+            if 18 in parse_message(request):
+                return message_field(16, message_field(1, 
self.ddl_string.encode()))
             if self.failure == "no-result":
                 return b""
             if self.failure == "missing-schema":
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 35be445..1a2a99d 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -2062,6 +2062,27 @@ def 
test_window_frames_return_the_rows_of_pyspark(session):
         sums(o.rowsBetween(-(1 << 31) - 1, 0))
 
 
+def test_struct_types_convert_to_the_ddl_of_spark_as_pyspark(session):
+    # As PySpark Connect's StructType.toDDL: the server writes the DDL of the 
schema, which
+    # createDataFrame parses back.
+    from pyspark.sql import types as T
+
+    st = T.StructType([
+        T.StructField("a", T.IntegerType()),
+        T.StructField("b b", T.StringType(), False),
+        T.StructField("m", T.MapType(
+            T.StringType(), T.StructType([T.StructField("x", T.DecimalType(10, 
2))]))),
+        T.StructField("c", T.StringType(), metadata={"comment": "the c"}),
+    ])
+    ddl = st.toDDL()
+    assert ddl == (
+        "a INT,`b b` STRING NOT NULL,m MAP<STRING, STRUCT<x: DECIMAL(10,2)>>,"
+        "c STRING COMMENT 'the c'")
+    assert session.createDataFrame([], ddl).schema == st
+
+
+
+
 def test_maps_keep_their_entries_in_order_with_keys_of_their_type(session):
     row = session.range(1).selectExpr(
         "map(2, 'b', 1, 'a') AS ints",
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index f834631..a3a1480 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1595,7 +1595,6 @@ def test_structtype_object_model():
     ])
     assert st.typeName() == "struct"
     assert st.fieldNames() == ["a", "b"]
-    assert st.toDDL() == "a int,b string NOT NULL"
     assert "root" in st.treeString() and "a: integer (nullable = true)" in 
st.treeString()
     # toNullable makes every field nullable.
     assert st.toNullable().simpleString() == "struct<a:int,b:string>"
@@ -3220,6 +3219,26 @@ def test_typed_columns_exist():
     assert type(F.current_date()).__name__ == "Column"
 
 
+def 
test_struct_types_convert_to_ddl_on_the_server_of_the_active_session_as_pyspark(mock_connect):
+    # As PySpark Connect's StructType.toDDL: the server of the active session 
converts the JSON
+    # of the schema (AnalyzePlanRequest.json_to_ddl=18 -> 
JsonToDDL.json_string=1), and there
+    # is no DDL without an active session.
+    spark, service = mock_connect
+    service.ddl_string = "a INT,`b b` STRING NOT NULL"
+    st = T.StructType([
+        T.StructField("a", T.IntegerType()),
+        T.StructField("b b", T.StringType(), False),
+    ])
+    assert st.toDDL() == "a INT,`b b` STRING NOT NULL"
+    request = parse_message(service.analyze_requests[-1])
+    assert parse_message(request[18][0])[1][0].decode() == st.json()
+    spark.stop()
+    with pytest.raises(AssertionError):
+        st.toDDL()
+
+
+
+
 def test_struct_type_schema_methods():
     """Test StructType schema introspection methods."""
     st = T.StructType([
@@ -3227,11 +3246,6 @@ def test_struct_type_schema_methods():
         T.StructField("age", T.IntegerType(), True),
     ])
 
-    # Test toDDL
-    ddl = st.toDDL()
-    assert "name" in ddl
-    assert "age" in ddl
-
     # Test treeString
     tree = st.treeString()
     assert "root" in tree


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to