This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new e66293e [SPARK-61094][CONNECT][PYTHON] Add UnparsedDataType of
pyspark.sql.connect.types, the returnType of a UDTF of a DDL string, in the
Python client as PySpark does
e66293e is described below
commit e66293e3b3f2197c76684f4f7cfbb8466d22bd4f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 16:56:33 2026 +0900
[SPARK-61094][CONNECT][PYTHON] Add UnparsedDataType of
pyspark.sql.connect.types, the returnType of a UDTF of a DDL string, in the
Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61094](https://issues.apache.org/jira/browse/SPARK-61094):
`pyspark.sql.connect.types.UnparsedDataType` is added, and the `returnType` of
a UDTF given as a DDL string is one, as in PySpark Connect.
As PySpark Connect's:
- `UnparsedDataType` is a DDL string of a type that the server parses:
- its `simpleString` is `unparsed('<ddl>')` and its repr
`UnparsedDataType('<ddl>')`
- its `jsonValue`, `json`, `needConversion`, `toInternal` and
`fromInternal` raise `INVALID_CALL_ON_UNRESOLVED_OBJECT`
- the `returnType` of a UDTF given as a DDL string is an `UnparsedDataType`
- `spark.dataSource.register` of a class without a `name` raises the
`AttributeError` of `name()`
`pyspark.sql.connect.types` is a shim module with PySpark's class. Because
the drop-in's `DataType` pickles through its JSON, which an unparsed type
refuses, this class pickles from its DDL string instead. The client converts it
to and from the core's unparsed type. The `json()` of a pure-Python subclass of
`DataType` is now the `json.dumps` of its `jsonValue`, as PySpark's
`DataType.json` is.
### Why are the changes needed?
The client:
- lacked `pyspark.sql.connect.types`, so `from pyspark.sql.connect.types
import UnparsedDataType` raised an `ImportError`
- kept a DDL string as the `returnType` of a UDTF
- turned an unparsed type into a bare `DataType`
- raised a `TypeError` ("DataSource class must have a classmethod name()")
from `register`
### Does this PR introduce _any_ user-facing change?
Yes. `UnparsedDataType` and the `returnType` of a UDTF are those of PySpark
4.3.0.
### How was this patch tested?
- A new live test checks the `UnparsedDataType` `returnType` of a UDTF of a
DDL string, and the rows that the server returns for it. It passes with
official PySpark 4.3.0 and fails without this change.
- A new offline test checks the repr, the `simpleString`, the errors and
the pickling of `UnparsedDataType`, the `returnType` of a UDTF, and the error
of `register`. Two older offline tests now expect an `UnparsedDataType` where
they expected the DDL string or a bare `DataType`.
- 17 cases of `UnparsedDataType`, a UDTF and a data source give the same
results as official PySpark 4.3.0, except `typeName`, which SPARK-61080 (#281)
makes the class's (`unparseddata`).
- The lib tests of both crates (595 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2047 passed, 12 skipped), and the
live drop-in and Arrow suites (151 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #298 from HyukjinKwon/SPARK-61094-unparsed-data-type.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/datasource.rs | 13 ++---
crates/pyspark-rs/src/types.rs | 31 ++++++++++-
python/pyspark/sql/connect/types.py | 101 ++++++++++++++++++++++++++++++++++++
python/pyspark/sql/udtf.py | 5 ++
python/tests/test_dropin_e2e.py | 19 +++++++
python/tests/test_dropin_offline.py | 52 +++++++++++++++++--
6 files changed, 206 insertions(+), 15 deletions(-)
diff --git a/crates/pyspark-rs/src/datasource.rs
b/crates/pyspark-rs/src/datasource.rs
index c6abc2a..d6b8104 100644
--- a/crates/pyspark-rs/src/datasource.rs
+++ b/crates/pyspark-rs/src/datasource.rs
@@ -30,16 +30,9 @@ impl PyDataSourceRegistration {
/// Mirrors `pyspark.sql.SparkSession.dataSource.register`.
/// The data source class is cloudpickled and sent to the server.
fn register(&self, py: Python<'_>, data_source_class: &Bound<'_, PyAny>)
-> PyResult<()> {
- // Get the name via the classmethod `name()`
- let name_result = data_source_class.call_method0("name");
- let name: String = match name_result {
- Ok(name_obj) => name_obj.extract()?,
- Err(_) => {
- return Err(pyo3::exceptions::PyTypeError::new_err(
- "DataSource class must have a classmethod name()",
- ))
- }
- };
+ // The name of the classmethod `name()`, as PySpark Connect's: the
AttributeError of a
+ // class without one.
+ let name: String = data_source_class.call_method0("name")?.extract()?;
// Cloudpickle the class
let command = py_cloudpickle(py, data_source_class)?;
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 010125e..c0c198f 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -263,9 +263,18 @@ impl PyDataType {
}
// --- DataType object-model methods (v4.2.0 parity) ---
+ /// As PySpark's `DataType.json`, of the `jsonValue` of the type, also of
a pure-Python
+ /// subclass, such as an `UnparsedDataType`, whose `jsonValue` raises.
#[pyo3(name = "json")]
fn __obj_json(slf: &Bound<'_, Self>) -> PyResult<String> {
- Ok(py_to_data_type(slf.as_any())?.json())
+ let py = slf.py();
+ let kwargs = pyo3::types::PyDict::new(py);
+ kwargs.set_item("separators", (",", ":"))?;
+ kwargs.set_item("sort_keys", true)?;
+ py.import("json")?
+ .getattr("dumps")?
+ .call((slf.call_method0("jsonValue")?,), Some(&kwargs))?
+ .extract()
}
#[pyo3(name = "jsonValue")]
fn __obj_json_value<'py>(
@@ -3276,7 +3285,12 @@ pub(crate) fn data_type_to_py(py: Python<'_>, dt:
&DataType) -> PyResult<Py<PyAn
}
// An unparsed DDL string has no concrete Python class; keep it as a
bare `DataType`.
// (The Python parser delegates JSON strings back here, so routing it
there would loop.)
- DataType::Unparsed { .. } => Py::new(py, PyDataType { inner:
dt.clone() })?.into_any(),
+ // As PySpark Connect's, a DDL string that the server parses.
+ DataType::Unparsed { data_type_string } => py
+ .import("pyspark.sql.connect.types")?
+ .getattr("UnparsedDataType")?
+ .call1((data_type_string,))?
+ .unbind(),
DataType::Udt { .. } => {
// A UDT reconstructs by re-importing its Python class; delegate
to the Python
// parser.
@@ -3445,6 +3459,19 @@ pub(crate) fn py_to_data_type(obj: &Bound<'_, PyAny>) ->
PyResult<DataType> {
});
}
}
+ // `UnparsedDataType` of `pyspark.sql.connect.types`, a pure-Python
subclass of the base, is
+ // a DDL string that the server parses.
+ if let Ok(unparsed_cls) = obj
+ .py()
+ .import("pyspark.sql.connect.types")
+ .and_then(|m| m.getattr("UnparsedDataType"))
+ {
+ if obj.is_instance(&unparsed_cls)? {
+ return Ok(DataType::Unparsed {
+ data_type_string: obj.getattr("data_type_string")?.extract()?,
+ });
+ }
+ }
// The bare base `DataType`, which only `DataType()` and pure-Python
subclasses are, is
// matched LAST: concrete subclasses (which all share the PyDataType base)
are handled
// above via their own fields, so a mutated StructType reflects its live
fields rather
diff --git a/python/pyspark/sql/connect/types.py
b/python/pyspark/sql/connect/types.py
new file mode 100644
index 0000000..21f6431
--- /dev/null
+++ b/python/pyspark/sql/connect/types.py
@@ -0,0 +1,101 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""The types of ``pyspark.sql.connect.types`` that are not those of
``pyspark.sql.types``, as
+PySpark Connect's: ``UnparsedDataType``, a DDL string of a type that the
server parses. The
+conversions between the types and their protobuf messages of PySpark Connect's
module are those
+of the Rust transport."""
+
+from typing import Any, Dict
+
+from pyspark.errors import PySparkAssertionError
+from pyspark.sql.types import DataType
+
+
+class UnparsedDataType(DataType):
+ """
+ Unparsed data type.
+
+ The data type string will be parsed later.
+
+ Parameters
+ ----------
+ data_type_string : str
+ The data type string format equals :class:`DataType.simpleString`,
+ except that the top level struct type can omit the ``struct<>``.
+ This also supports a schema in a DDL-formatted string and
case-insensitive strings.
+
+ Examples
+ --------
+ >>> from pyspark.sql.connect.types import UnparsedDataType
+
+ >>> UnparsedDataType("int ")
+ UnparsedDataType('int ')
+ >>> UnparsedDataType("INT ")
+ UnparsedDataType('INT ')
+ >>> UnparsedDataType("a: byte, b: decimal( 16 , 8 ) ")
+ UnparsedDataType('a: byte, b: decimal( 16 , 8 ) ')
+ >>> UnparsedDataType("a DOUBLE, b STRING")
+ UnparsedDataType('a DOUBLE, b STRING')
+ >>> UnparsedDataType("a DOUBLE, b CHAR( 50 )")
+ UnparsedDataType('a DOUBLE, b CHAR( 50 )')
+ >>> UnparsedDataType("a DOUBLE, b VARCHAR( 50 )")
+ UnparsedDataType('a DOUBLE, b VARCHAR( 50 )')
+ >>> UnparsedDataType("a: array< short>")
+ UnparsedDataType('a: array< short>')
+ >>> UnparsedDataType(" map<string , string > ")
+ UnparsedDataType(' map<string , string > ')
+ """
+
+ def __init__(self, data_type_string: str):
+ self.data_type_string = data_type_string
+
+ def simpleString(self) -> str:
+ return "unparsed(%s)" % repr(self.data_type_string)
+
+ def __repr__(self) -> str:
+ return "UnparsedDataType(%s)" % repr(self.data_type_string)
+
+ def jsonValue(self) -> Dict[str, Any]:
+ raise PySparkAssertionError(
+ errorClass="INVALID_CALL_ON_UNRESOLVED_OBJECT",
+ messageParameters={"func_name": "jsonValue"},
+ )
+
+ def needConversion(self) -> bool:
+ raise PySparkAssertionError(
+ errorClass="INVALID_CALL_ON_UNRESOLVED_OBJECT",
+ messageParameters={"func_name": "needConversion"},
+ )
+
+ def toInternal(self, obj: Any) -> Any:
+ raise PySparkAssertionError(
+ errorClass="INVALID_CALL_ON_UNRESOLVED_OBJECT",
+ messageParameters={"func_name": "toInternal"},
+ )
+
+ def fromInternal(self, obj: Any) -> Any:
+ raise PySparkAssertionError(
+ errorClass="INVALID_CALL_ON_UNRESOLVED_OBJECT",
+ messageParameters={"func_name": "fromInternal"},
+ )
+
+ def __reduce__(self) -> Any:
+ # The DataType of the drop-in pickles through its JSON, which an
unparsed type has not.
+ return (UnparsedDataType, (self.data_type_string,))
+
+
+__all__ = ["UnparsedDataType"]
diff --git a/python/pyspark/sql/udtf.py b/python/pyspark/sql/udtf.py
index 79ea32b..4c71345 100644
--- a/python/pyspark/sql/udtf.py
+++ b/python/pyspark/sql/udtf.py
@@ -103,6 +103,11 @@ class UserDefinedTableFunction:
# As PySpark's UserDefinedTableFunction, for udtf(), arrow_udtf() and
register().
_validate_udtf_handler(func, returnType)
self.func = func
+ # As PySpark Connect's: a DDL string is a type that the server parses.
+ if isinstance(returnType, str):
+ from pyspark.sql.connect.types import UnparsedDataType
+
+ returnType = UnparsedDataType(returnType)
self.returnType = returnType
self.evalType = evalType
self.name = name or (
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index fac8fc9..c01428f 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -67,6 +67,25 @@ def
test_to_local_iterator_raises_the_error_of_the_execution_as_pyspark(session,
assert list(rows) == []
+def test_udtfs_of_ddl_return_types_are_unparsed_as_pyspark(session):
+ # As PySpark Connect's: the returnType of a UDTF of a DDL string is an
UnparsedDataType,
+ # which the server parses.
+ from pyspark.sql.connect.types import UnparsedDataType
+ from pyspark.sql.functions import lit, udtf
+
+ @udtf(returnType="x int, y string")
+ class Pairs:
+ def eval(self, n: int):
+ for i in range(n):
+ yield i, str(i)
+
+ assert isinstance(Pairs.returnType, UnparsedDataType)
+ assert Pairs.returnType.data_type_string == "x int, y string"
+ assert [tuple(row) for row in Pairs(lit(2)).collect()] == [(0, "0"), (1,
"1")]
+
+
+
+
def test_udf_and_udtf_calls_take_named_arguments_as_pyspark(session):
# As PySpark's: the keyword arguments of a UDF or UDTF call are its named
arguments, and
# a bare string is a column name.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 98ec6aa..4cb9ed5 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -207,10 +207,14 @@ def test_spatial_and_collated_json_forms_parse():
def test_unparsed_datatype_pickle_roundtrip():
+ # The core's JSON form of an unparsed type is the UnparsedDataType of
+ # pyspark.sql.connect.types, which pickles of its DDL string, as its JSON
raises.
+ from pyspark.sql.connect.types import UnparsedDataType
+
t = T._parse_datatype_json_value("unparsed(some_ddl)")
- assert t.json() == '"unparsed(some_ddl)"'
+ assert isinstance(t, UnparsedDataType) and t.data_type_string == "some_ddl"
r = pickle.loads(pickle.dumps(t))
- assert r == t and r.json() == t.json()
+ assert r == t and repr(r) == "UnparsedDataType('some_ddl')"
def test_user_defined_type_pickle_roundtrip():
@@ -2166,6 +2170,44 @@ def test_catalog_result_classes():
assert Table("t", None, ["a", "b"], None, "MANAGED", False).database is
None
+def test_unparsed_data_types_are_those_of_pyspark(mock_connect):
+ # As PySpark Connect's: UnparsedDataType of pyspark.sql.connect.types is a
DDL string of a
+ # type that the server parses, whose JSON and conversions raise
+ # INVALID_CALL_ON_UNRESOLVED_OBJECT; the returnType of a UDTF of a DDL
string is one; and
+ # the registration of a data source without a name raises the
AttributeError of name().
+ import pickle
+
+ from pyspark.errors import PySparkAssertionError
+ from pyspark.sql.connect.types import UnparsedDataType
+ from pyspark.sql.functions import udtf
+
+ spark, _ = mock_connect
+ unparsed = UnparsedDataType("a int, b string")
+ assert isinstance(unparsed, T.DataType)
+ assert repr(unparsed) == "UnparsedDataType('a int, b string')"
+ assert unparsed.simpleString() == "unparsed('a int, b string')"
+ for name, call in (("jsonValue", unparsed.jsonValue), ("jsonValue",
unparsed.json),
+ ("needConversion", unparsed.needConversion),
+ ("toInternal", lambda: unparsed.toInternal(1)),
+ ("fromInternal", lambda: unparsed.fromInternal(1))):
+ with pytest.raises(PySparkAssertionError) as e:
+ call()
+ assert e.value.getCondition() == "INVALID_CALL_ON_UNRESOLVED_OBJECT"
+ assert e.value.getMessageParameters() == {"func_name": name}
+ assert repr(pickle.loads(pickle.dumps(unparsed))) == repr(unparsed)
+
+ @udtf(returnType="x int")
+ class Tables:
+ def eval(self, n: int):
+ yield (n,)
+
+ assert repr(Tables.returnType) == "UnparsedDataType('x int')"
+ with pytest.raises(AttributeError):
+ spark.dataSource.register(1)
+
+
+
+
def test_udtf_analyze_result_family():
from pyspark.sql.udtf import (
AnalyzeArgument, PartitioningColumn, OrderingColumn, SelectedColumn,
@@ -2503,6 +2545,7 @@ def
test_udtf_return_type_connect_validation_and_command():
subprocess.run([sys.executable, "-c", """
from pyspark.sql import SparkSession, DataFrame, functions as F, types as T
+from pyspark.sql.connect.types import UnparsedDataType
from pyspark.serializers import CloudPickleSerializer
# No server listens: with useArrow, udtf() does not read the Arrow config of
the session.
@@ -2521,7 +2564,10 @@ for return_type in ["value BIGINT", "STRUCT<value:
BIGINT>",
handler = CloudPickleSerializer().loads(fn.command)
assert isinstance(handler, type)
assert list(handler().eval(7)) == [(7,)]
- assert fn.returnType == return_type
+ # As PySpark Connect's, a DDL string is an UnparsedDataType.
+ expected = (UnparsedDataType(return_type) if isinstance(return_type,
str)
+ else return_type)
+ assert fn.returnType == expected
# Dynamic schemas omit return_type and leave analysis to the Python handler.
class Dynamic(Echo):
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]