This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new c2ef841  [SPARK-61040][CONNECT][PYTHON] Parse the DDL strings of 
DataType.fromDDL and of the return types of UDFs on the server in the Python 
client as PySpark does
c2ef841 is described below

commit c2ef84112accc4aac25aab56d960eee99c89acec
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:50:15 2026 +0900

    [SPARK-61040][CONNECT][PYTHON] Parse the DDL strings of DataType.fromDDL 
and of the return types of UDFs on the server in the Python client as PySpark 
does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61040](https://issues.apache.org/jira/browse/SPARK-61040): 
`DataType.fromDDL` and the DDL return types of UDFs are now parsed on the 
server, as PySpark Connect does.
    
    `DataType.fromDDL(ddl)` and `StructType.fromDDL` now parse with the new 
`pyspark.sql.types._parse_datatype_string(s)`. That asks the server of the 
active session to parse the string (the new `SparkSession._parse_ddl`, the 
`DDLParse` of an `AnalyzePlanRequest`), as PySpark Connect's do. So does a UDF 
of a DDL return type, as `udf(f, "a INT NOT NULL")`, whose `returnType` 
resolves through `DataType.fromDDL`. They take any DDL that Spark does:
    - `NOT NULL` and `COMMENT` clauses
    - nested types, such as `"st struct<a:int,b:double>"`
    - collations, such as `"s STRING COLLATE UTF8_LCASE"`
    
    They raise the errors of Spark's parser, such as `PARSE_SYNTAX_ERROR` for 
`"a INT,"` and `UNSUPPORTED_DATATYPE` for an unknown type, and 
`NO_ACTIVE_OR_DEFAULT_SESSION` without a session.
    
    ### Why are the changes needed?
    
    The client parsed them on the client:
    - It raised `ValueError: [INVALID_DATATYPE_FORMAT]` for `NOT NULL` and 
nested types.
    - It took `"s STRING COLLATE UTF8_LCASE"` as a string of the collation 
`collate utf8_lcase`, and `"a INT,"` as `a INT`.
    - It had no `pyspark.sql.types._parse_datatype_string`, which PySpark code 
imports.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These parse the DDL of PySpark 4.3.0, and raise its errors.
    
    ### How was this patch tested?
    
    - A new live test parses DDL strings with `DataType.fromDDL`, 
`StructType.fromDDL` and `_parse_datatype_string`, runs a UDF of a nested DDL 
return type, and checks the errors. It passes with official PySpark 4.3.0, and 
fails without this change.
    - On a local server, 12 cases of these behave as in official PySpark 4.3.0.
    - The lib tests of both crates (571 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1810 passed, 8 skipped), and the 
live drop-in and Arrow suites (110 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #237 from HyukjinKwon/SPARK-61040-fromddl-server.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/session.rs    |  9 +++++++++
 crates/pyspark-rs/src/types.rs      | 14 +++++++++-----
 python/pyspark/sql/types.py         | 10 ++++++++++
 python/tests/test_dropin_e2e.py     | 24 ++++++++++++++++++++++++
 python/tests/test_dropin_offline.py | 14 +++++++-------
 5 files changed, 59 insertions(+), 12 deletions(-)

diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 7f2cc5d..05f8960 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1415,6 +1415,15 @@ impl PySparkSession {
         active_session(py)
     }
 
+    /// The type of the DDL string `ddl`, as the server parses it, as PySpark 
Connect's
+    /// `SparkSession._parse_ddl`, of a `DDLParse` of an `AnalyzePlanRequest`.
+    fn _parse_ddl(&self, py: Python<'_>, ddl: &str) -> PyResult<Py<PyAny>> {
+        let data_type = py
+            .detach(|| self.session.parse_ddl(ddl).map_err(Box::new))
+            .to_pyerr()?;
+        crate::types::data_type_to_py(py, &data_type)
+    }
+
     /// The active session of this thread, or else the default session; errors 
if neither.
     /// Mirrors `SparkSession.active`, which raises 
`NO_ACTIVE_OR_DEFAULT_SESSION`.
     #[staticmethod]
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 033f006..f5b8fc7 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -196,14 +196,18 @@ impl PyDataType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
+    /// The type of the DDL string `ddl`, as PySpark's `DataType.fromDDL`, 
which the server of
+    /// the active session parses (`_parse_datatype_string`): the concrete 
type, a StructType
+    /// for a list of fields. The subclasses inherit this classmethod, so that 
any of them
+    /// parses DDL the same way.
     #[classmethod]
     #[pyo3(name = "fromDDL")]
     fn __obj_from_ddl(cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<Py<PyAny>> {
-        let dt = DataType::from_ddl(ddl)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))?;
-        // The concrete type, as pyspark returns (a StructType for a list of 
fields). The
-        // subclasses inherit this classmethod, so that any of them parses DDL 
the same way.
-        data_type_to_py(cls.py(), &dt)
+        Ok(cls
+            .py()
+            .import("pyspark.sql.types")?
+            .call_method1("_parse_datatype_string", (ddl,))?
+            .unbind())
     }
     #[pyo3(name = "typeName")]
     fn __dt_type_name(&self) -> String {
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index fa8cace..a5f9d88 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -287,6 +287,16 @@ __all__ = [
 ]
 
 
+def _parse_datatype_string(s):
+    """The DataType of the DDL string `s`, as the server of the active session 
parses it, as
+    PySpark Connect's `_parse_datatype_string`: a schema of fields, as "a INT, 
b STRING", or a
+    type, as "decimal(10, 2)". It raises NO_ACTIVE_OR_DEFAULT_SESSION without 
a session, and
+    the errors of Spark's parser."""
+    from pyspark.sql import SparkSession
+
+    return SparkSession.active()._parse_ddl(s)
+
+
 def _parse_datatype_json_string(json_string):
     """Reconstruct a DataType from its Spark JSON (client-side; UDF workers 
use the
     official pyspark function of the same name). Handles the full recursive 
grammar:
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index e588adc..6042f4d 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1508,6 +1508,30 @@ def 
test_create_dataframe_takes_the_ddl_of_spark_as_pyspark(session):
     assert mapped.collect() == [Row(id=0), Row(id=1)]
 
 
+
+def test_ddl_types_are_parsed_by_the_server_as_pyspark(session):
+    # As PySpark Connect's DataType.fromDDL and _parse_datatype_string, which 
ask the server
+    # of the active session to parse a DDL string, as does a UDF of a DDL 
return type.
+    from pyspark.errors import ParseException
+    from pyspark.sql import types as T
+    from pyspark.sql.types import _parse_datatype_string
+
+    field = T.DataType.fromDDL("a INT NOT NULL COMMENT 'c'").fields[0]
+    assert (field.nullable, field.metadata) == (False, {"comment": "c"})
+    assert T.StructType.fromDDL("st struct<a:int,b:double>").simpleString() == 
(
+        "struct<st:struct<a:int,b:double>>")
+    assert T.DataType.fromDDL("s STRING COLLATE UTF8_LCASE").simpleString() == 
(
+        "struct<s:string collate UTF8_LCASE>")
+    assert _parse_datatype_string("decimal(10, 2)") == T.DecimalType(10, 2)
+    for ddl, condition in [("a INT,", "PARSE_SYNTAX_ERROR"), ("a BOGUS", 
"UNSUPPORTED_DATATYPE")]:
+        with pytest.raises(ParseException) as error:
+            T.DataType.fromDDL(ddl)
+        assert error.value.getCondition() == condition
+    pair = F.udf(lambda x: (x, 1.5), "st struct<a:bigint,b:double>")
+    assert pair.returnType.simpleString() == 
"struct<st:struct<a:bigint,b:double>>"
+    result = session.range(1).select(F.udf(lambda x: (x, 1.5), 
"struct<a:bigint,b:double>")("id"))
+    assert [tuple(r[0]) for r in result.collect()] == [(0, 1.5)]
+
 def test_functions_send_their_optional_arguments_as_pyspark(session):
     # The values of official PySpark 4.3.0; each optional argument used to be 
dropped.
     df = session.sql(
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index b0bcb74..40a3e62 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -289,7 +289,7 @@ def test_datatype_eq_and_hash_match_pyspark():
     assert type(T.LongType()).__eq__ is not object.__eq__
 
 
-def test_datatype_eq_across_construction_paths():
+def test_datatype_eq_across_construction_paths(mock_connect):
     import json
 
     st = T.StructType([T.StructField("a", T.IntegerType())])
@@ -1142,7 +1142,7 @@ def test_time_values_are_sent_as_pyspark(mock_connect):
     assert parse_message(parse_message(expression[1][0])[26][0]) == {1: 
[nanos], 2: [6]}
 
 
-def test_time_types_parse_as_pyspark():
+def test_time_types_parse_as_pyspark(mock_connect):
     # As PySpark's: a bare time is TimeType(), of the default precision 6, in 
a DDL string and
     # in the JSON of a type.
     assert T.DataType.fromDDL("t time") == T.StructType([T.StructField("t", 
T.TimeType(6))])
@@ -1446,7 +1446,7 @@ def test_user_defined_type_repr_is_its_class_name():
      "StructType([StructField('b', StringType(), True), StructField('a', 
IntegerType(), True)])"),
     ("int", T.IntegerType, "IntegerType()"),
 ])
-def test_datatype_from_ddl_returns_the_concrete_type(ddl, cls, expected):
+def test_datatype_from_ddl_returns_the_concrete_type(ddl, cls, expected, 
mock_connect):
     parsed = T.DataType.fromDDL(ddl)
     assert repr(parsed) == expected
     assert isinstance(parsed, cls)
@@ -1454,7 +1454,7 @@ def test_datatype_from_ddl_returns_the_concrete_type(ddl, 
cls, expected):
     assert repr(T.LongType.fromDDL(ddl)) == expected
 
 
-def test_datatype_object_model():
+def test_datatype_object_model(mock_connect):
     # json / jsonValue / typeName / simpleString / needConversion match 
official pyspark.
     assert T.IntegerType().typeName() == "integer"
     assert T.IntegerType().simpleString() == "int"
@@ -4562,7 +4562,7 @@ def test_udf_string_arg_is_column_name_not_literal():
     assert repr(c) == repr(u(col("id")))
 
 
-def test_udf_and_pandas_udf_decorator_forms():
+def test_udf_and_pandas_udf_decorator_forms(mock_connect):
     # @udf(returnType) / @pandas_udf(returnType) return a decorator that wraps 
a function.
     from pyspark.sql.functions import udf, pandas_udf
     from pyspark.sql.types import IntegerType
@@ -4580,7 +4580,7 @@ def test_udf_and_pandas_udf_decorator_forms():
     assert isinstance(p._unwrapped, UserDefinedFunction) and p.__name__ == "p"
 
 
-def test_udf_return_type_normalized_to_concrete_datatype():
+def test_udf_return_type_normalized_to_concrete_datatype(mock_connect):
     # A DDL-string returnType must become a concrete, picklable DataType so the
     # cloudpickled command carries a real type (the pandas/Arrow worker needs 
it).
     import pickle
@@ -4595,7 +4595,7 @@ def 
test_udf_return_type_normalized_to_concrete_datatype():
     pickle.dumps(f.returnType)  # concrete types pickle
 
 
-def test_pandas_udf_eval_type_inferred_from_type_hints():
+def test_pandas_udf_eval_type_inferred_from_type_hints(mock_connect):
     # @pandas_udf("type") infers eval type from the function's pandas type 
hints:
     # Series->Series = scalar(200), Iterator->Iterator = scalar_iter(204),
     # Series->scalar = grouped_agg(202).


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to