This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new c2ef841 [SPARK-61040][CONNECT][PYTHON] Parse the DDL strings of
DataType.fromDDL and of the return types of UDFs on the server in the Python
client as PySpark does
c2ef841 is described below
commit c2ef84112accc4aac25aab56d960eee99c89acec
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:50:15 2026 +0900
[SPARK-61040][CONNECT][PYTHON] Parse the DDL strings of DataType.fromDDL
and of the return types of UDFs on the server in the Python client as PySpark
does
### What changes were proposed in this pull request?
[SPARK-61040](https://issues.apache.org/jira/browse/SPARK-61040):
`DataType.fromDDL` and the DDL return types of UDFs are now parsed on the
server, as PySpark Connect does.
`DataType.fromDDL(ddl)` and `StructType.fromDDL` now parse with the new
`pyspark.sql.types._parse_datatype_string(s)`. That asks the server of the
active session to parse the string (the new `SparkSession._parse_ddl`, the
`DDLParse` of an `AnalyzePlanRequest`), as PySpark Connect's do. So does a UDF
of a DDL return type, as `udf(f, "a INT NOT NULL")`, whose `returnType`
resolves through `DataType.fromDDL`. They take any DDL that Spark does:
- `NOT NULL` and `COMMENT` clauses
- nested types, such as `"st struct<a:int,b:double>"`
- collations, such as `"s STRING COLLATE UTF8_LCASE"`
They raise the errors of Spark's parser, such as `PARSE_SYNTAX_ERROR` for
`"a INT,"` and `UNSUPPORTED_DATATYPE` for an unknown type, and
`NO_ACTIVE_OR_DEFAULT_SESSION` without a session.
### Why are the changes needed?
The client parsed them on the client:
- It raised `ValueError: [INVALID_DATATYPE_FORMAT]` for `NOT NULL` and
nested types.
- It took `"s STRING COLLATE UTF8_LCASE"` as a string of the collation
`collate utf8_lcase`, and `"a INT,"` as `a INT`.
- It had no `pyspark.sql.types._parse_datatype_string`, which PySpark code
imports.
### Does this PR introduce _any_ user-facing change?
Yes. These parse the DDL of PySpark 4.3.0, and raise its errors.
### How was this patch tested?
- A new live test parses DDL strings with `DataType.fromDDL`,
`StructType.fromDDL` and `_parse_datatype_string`, runs a UDF of a nested DDL
return type, and checks the errors. It passes with official PySpark 4.3.0, and
fails without this change.
- On a local server, 12 cases of these behave as in official PySpark 4.3.0.
- The lib tests of both crates (571 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1810 passed, 8 skipped), and the
live drop-in and Arrow suites (110 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #237 from HyukjinKwon/SPARK-61040-fromddl-server.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/session.rs | 9 +++++++++
crates/pyspark-rs/src/types.rs | 14 +++++++++-----
python/pyspark/sql/types.py | 10 ++++++++++
python/tests/test_dropin_e2e.py | 24 ++++++++++++++++++++++++
python/tests/test_dropin_offline.py | 14 +++++++-------
5 files changed, 59 insertions(+), 12 deletions(-)
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 7f2cc5d..05f8960 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1415,6 +1415,15 @@ impl PySparkSession {
active_session(py)
}
+ /// The type of the DDL string `ddl`, as the server parses it, as PySpark
Connect's
+ /// `SparkSession._parse_ddl`, of a `DDLParse` of an `AnalyzePlanRequest`.
+ fn _parse_ddl(&self, py: Python<'_>, ddl: &str) -> PyResult<Py<PyAny>> {
+ let data_type = py
+ .detach(|| self.session.parse_ddl(ddl).map_err(Box::new))
+ .to_pyerr()?;
+ crate::types::data_type_to_py(py, &data_type)
+ }
+
/// The active session of this thread, or else the default session; errors
if neither.
/// Mirrors `SparkSession.active`, which raises
`NO_ACTIVE_OR_DEFAULT_SESSION`.
#[staticmethod]
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 033f006..f5b8fc7 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -196,14 +196,18 @@ impl PyDataType {
fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
obj
}
+ /// The type of the DDL string `ddl`, as PySpark's `DataType.fromDDL`,
which the server of
+ /// the active session parses (`_parse_datatype_string`): the concrete
type, a StructType
+ /// for a list of fields. The subclasses inherit this classmethod, so that
any of them
+ /// parses DDL the same way.
#[classmethod]
#[pyo3(name = "fromDDL")]
fn __obj_from_ddl(cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) ->
PyResult<Py<PyAny>> {
- let dt = DataType::from_ddl(ddl)
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError,
_>(e.to_string()))?;
- // The concrete type, as pyspark returns (a StructType for a list of
fields). The
- // subclasses inherit this classmethod, so that any of them parses DDL
the same way.
- data_type_to_py(cls.py(), &dt)
+ Ok(cls
+ .py()
+ .import("pyspark.sql.types")?
+ .call_method1("_parse_datatype_string", (ddl,))?
+ .unbind())
}
#[pyo3(name = "typeName")]
fn __dt_type_name(&self) -> String {
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index fa8cace..a5f9d88 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -287,6 +287,16 @@ __all__ = [
]
+def _parse_datatype_string(s):
+ """The DataType of the DDL string `s`, as the server of the active session
parses it, as
+ PySpark Connect's `_parse_datatype_string`: a schema of fields, as "a INT,
b STRING", or a
+ type, as "decimal(10, 2)". It raises NO_ACTIVE_OR_DEFAULT_SESSION without
a session, and
+ the errors of Spark's parser."""
+ from pyspark.sql import SparkSession
+
+ return SparkSession.active()._parse_ddl(s)
+
+
def _parse_datatype_json_string(json_string):
"""Reconstruct a DataType from its Spark JSON (client-side; UDF workers
use the
official pyspark function of the same name). Handles the full recursive
grammar:
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index e588adc..6042f4d 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1508,6 +1508,30 @@ def
test_create_dataframe_takes_the_ddl_of_spark_as_pyspark(session):
assert mapped.collect() == [Row(id=0), Row(id=1)]
+
+def test_ddl_types_are_parsed_by_the_server_as_pyspark(session):
+ # As PySpark Connect's DataType.fromDDL and _parse_datatype_string, which
ask the server
+ # of the active session to parse a DDL string, as does a UDF of a DDL
return type.
+ from pyspark.errors import ParseException
+ from pyspark.sql import types as T
+ from pyspark.sql.types import _parse_datatype_string
+
+ field = T.DataType.fromDDL("a INT NOT NULL COMMENT 'c'").fields[0]
+ assert (field.nullable, field.metadata) == (False, {"comment": "c"})
+ assert T.StructType.fromDDL("st struct<a:int,b:double>").simpleString() ==
(
+ "struct<st:struct<a:int,b:double>>")
+ assert T.DataType.fromDDL("s STRING COLLATE UTF8_LCASE").simpleString() ==
(
+ "struct<s:string collate UTF8_LCASE>")
+ assert _parse_datatype_string("decimal(10, 2)") == T.DecimalType(10, 2)
+ for ddl, condition in [("a INT,", "PARSE_SYNTAX_ERROR"), ("a BOGUS",
"UNSUPPORTED_DATATYPE")]:
+ with pytest.raises(ParseException) as error:
+ T.DataType.fromDDL(ddl)
+ assert error.value.getCondition() == condition
+ pair = F.udf(lambda x: (x, 1.5), "st struct<a:bigint,b:double>")
+ assert pair.returnType.simpleString() ==
"struct<st:struct<a:bigint,b:double>>"
+ result = session.range(1).select(F.udf(lambda x: (x, 1.5),
"struct<a:bigint,b:double>")("id"))
+ assert [tuple(r[0]) for r in result.collect()] == [(0, 1.5)]
+
def test_functions_send_their_optional_arguments_as_pyspark(session):
# The values of official PySpark 4.3.0; each optional argument used to be
dropped.
df = session.sql(
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index b0bcb74..40a3e62 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -289,7 +289,7 @@ def test_datatype_eq_and_hash_match_pyspark():
assert type(T.LongType()).__eq__ is not object.__eq__
-def test_datatype_eq_across_construction_paths():
+def test_datatype_eq_across_construction_paths(mock_connect):
import json
st = T.StructType([T.StructField("a", T.IntegerType())])
@@ -1142,7 +1142,7 @@ def test_time_values_are_sent_as_pyspark(mock_connect):
assert parse_message(parse_message(expression[1][0])[26][0]) == {1:
[nanos], 2: [6]}
-def test_time_types_parse_as_pyspark():
+def test_time_types_parse_as_pyspark(mock_connect):
# As PySpark's: a bare time is TimeType(), of the default precision 6, in
a DDL string and
# in the JSON of a type.
assert T.DataType.fromDDL("t time") == T.StructType([T.StructField("t",
T.TimeType(6))])
@@ -1446,7 +1446,7 @@ def test_user_defined_type_repr_is_its_class_name():
"StructType([StructField('b', StringType(), True), StructField('a',
IntegerType(), True)])"),
("int", T.IntegerType, "IntegerType()"),
])
-def test_datatype_from_ddl_returns_the_concrete_type(ddl, cls, expected):
+def test_datatype_from_ddl_returns_the_concrete_type(ddl, cls, expected,
mock_connect):
parsed = T.DataType.fromDDL(ddl)
assert repr(parsed) == expected
assert isinstance(parsed, cls)
@@ -1454,7 +1454,7 @@ def test_datatype_from_ddl_returns_the_concrete_type(ddl,
cls, expected):
assert repr(T.LongType.fromDDL(ddl)) == expected
-def test_datatype_object_model():
+def test_datatype_object_model(mock_connect):
# json / jsonValue / typeName / simpleString / needConversion match
official pyspark.
assert T.IntegerType().typeName() == "integer"
assert T.IntegerType().simpleString() == "int"
@@ -4562,7 +4562,7 @@ def test_udf_string_arg_is_column_name_not_literal():
assert repr(c) == repr(u(col("id")))
-def test_udf_and_pandas_udf_decorator_forms():
+def test_udf_and_pandas_udf_decorator_forms(mock_connect):
# @udf(returnType) / @pandas_udf(returnType) return a decorator that wraps
a function.
from pyspark.sql.functions import udf, pandas_udf
from pyspark.sql.types import IntegerType
@@ -4580,7 +4580,7 @@ def test_udf_and_pandas_udf_decorator_forms():
assert isinstance(p._unwrapped, UserDefinedFunction) and p.__name__ == "p"
-def test_udf_return_type_normalized_to_concrete_datatype():
+def test_udf_return_type_normalized_to_concrete_datatype(mock_connect):
# A DDL-string returnType must become a concrete, picklable DataType so the
# cloudpickled command carries a real type (the pandas/Arrow worker needs
it).
import pickle
@@ -4595,7 +4595,7 @@ def
test_udf_return_type_normalized_to_concrete_datatype():
pickle.dumps(f.returnType) # concrete types pickle
-def test_pandas_udf_eval_type_inferred_from_type_hints():
+def test_pandas_udf_eval_type_inferred_from_type_hints(mock_connect):
# @pandas_udf("type") infers eval type from the function's pandas type
hints:
# Series->Series = scalar(200), Iterator->Iterator = scalar_iter(204),
# Series->scalar = grouped_agg(202).
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]