This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 7b9bdd6  [SPARK-61073][CONNECT][PYTHON] Raise KEY_NOT_EXISTS for a 
missing field of a StructType, and parse the JSON type of StructType.add, in 
the Python client as PySpark does
7b9bdd6 is described below

commit 7b9bdd6ceb94c40340086aa06f1a01c8bb02c804
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 13:22:23 2026 +0900

    [SPARK-61073][CONNECT][PYTHON] Raise KEY_NOT_EXISTS for a missing field of 
a StructType, and parse the JSON type of StructType.add, in the Python client 
as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61073](https://issues.apache.org/jira/browse/SPARK-61073): 
`StructType[key]` and `StructType.add` now take their arguments and raise their 
errors as PySpark's do.
    
    - `StructType[name]` raises the `PySparkKeyError` `KEY_NOT_EXISTS` for a 
name that no field has.
    - `StructType[index]` raises the `PySparkIndexError` `INDEX_OUT_OF_RANGE` 
for an index outside the fields. It raises `NOT_EXPECTED_TYPE` for a key that 
is not an int, a slice or a str.
    - `StructType.add(name, data_type)` parses a str `data_type` as the JSON 
value of a data type, such as `"integer"` or `"decimal(10,2)"`. For a str that 
is not one, such as `"int"`, it raises `CANNOT_PARSE_DATATYPE`. For a name 
without a data type, it raises `ARGUMENT_REQUIRED`.
    - `_parse_datatype_json_value` raises PySpark's `CANNOT_PARSE_DATATYPE` and 
`UNSUPPORTED_DATA_TYPE`.
    
    ### Why are the changes needed?
    
    The client raised these errors instead:
    - a plain `KeyError: 'No StructField named zz'` for a missing field
    - a plain `IndexError` and `TypeError` for the other keys
    
    `add` also differed:
    - it parsed a str data type as DDL, so `add("a", "int")` was accepted
    - it took a `StringType` for a name without a data type
    - it raised a plain `ValueError` for a type that does not parse
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `StructType[key]` and `StructType.add` take the arguments, and raise 
the errors, of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test builds a schema with `add` and the JSON values of types, 
creates a DataFrame of it, and checks the error for a missing field of its 
schema. It passes with official PySpark 4.3.0 and fails without this change.
    - A new offline test checks the errors of `StructType[key]` for names, 
indexes and other keys. It also checks the types that `add` parses, and its 
errors.
    - The lib tests of both crates (583 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1930 passed, 8 skipped), and the 
live drop-in and Arrow suites (127 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #271 from HyukjinKwon/SPARK-61073-struct-access.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/types.rs      | 84 ++++++++++++++++++++++++-------------
 python/pyspark/sql/types.py         | 13 +++++-
 python/tests/test_dropin_e2e.py     | 14 +++++++
 python/tests/test_dropin_offline.py | 42 ++++++++++++++++++-
 4 files changed, 122 insertions(+), 31 deletions(-)

diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 6e45a9f..a333443 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -1656,7 +1656,7 @@ impl PyStructType {
     }
 
     /// Append a field and return `self` (chainable, mirroring 
`StructType.add`):
-    /// `StructType().add("a", "int").add("b", StringType())`.
+    /// `StructType().add("a", "integer").add("b", StringType())`.
     #[pyo3(signature = (field, data_type=None, nullable=true, metadata=None))]
     fn add<'py>(
         slf: Bound<'py, Self>,
@@ -1668,12 +1668,28 @@ impl PyStructType {
         let new_field = if let Ok(sf) = 
field.extract::<PyRef<PyStructField>>() {
             sf.field.clone()
         } else {
+            // As PySpark's: a name takes a data type, of which a str is the 
JSON value of one.
+            let Some(data_type) = data_type else {
+                return Err(crate::column::raise_pyspark(
+                    field.py(),
+                    "PySparkValueError",
+                    "ARGUMENT_REQUIRED",
+                    &[
+                        ("arg_name", "data_type"),
+                        ("condition", "passing name of struct_field to 
create"),
+                    ],
+                ));
+            };
             let name: String = field.extract()?;
-            let dt = match data_type {
-                Some(d) => py_to_data_type(d)?,
-                None => DataType::String {
-                    collation: "UTF8_BINARY".to_string(),
-                },
+            let dt = if data_type.is_instance_of::<pyo3::types::PyString>() {
+                let parsed = field
+                    .py()
+                    .import("pyspark.sql.types")?
+                    .getattr("_parse_datatype_json_value")?
+                    .call1((data_type,))?;
+                py_to_data_type(&parsed)?
+            } else {
+                py_to_data_type(data_type)?
             };
             let md = match metadata {
                 Some(d) => py_metadata_to_map(d.py(), d)?,
@@ -1717,17 +1733,24 @@ impl PyStructType {
     }
 
     /// Index by position (`st[0]`) or by name (`st["a"]`), returning a 
`StructField`
-    /// (mirrors `StructType.__getitem__`; a slice yields a new `StructType`).
+    /// (mirrors `StructType.__getitem__`; a slice yields a new `StructType`), 
with the errors of
+    /// PySpark's: `KEY_NOT_EXISTS` of a name, `INDEX_OUT_OF_RANGE` of an 
index, and
+    /// `NOT_EXPECTED_TYPE` of another key.
     fn __getitem__(&self, py: Python<'_>, key: &Bound<'_, PyAny>) -> 
PyResult<Py<PyAny>> {
-        if let Ok(name) = key.extract::<String>() {
+        use crate::column::raise_pyspark;
+        if let Ok(name) = key.cast::<pyo3::types::PyString>() {
+            let name = name.to_cow()?;
             for f in &self.fields {
                 if f.name == name {
                     return Ok(Py::new(py, PyStructField { field: f.clone() 
})?.into_any());
                 }
             }
-            return Err(PyErr::new::<pyo3::exceptions::PyKeyError, _>(format!(
-                "No StructField named {name}"
-            )));
+            return Err(raise_pyspark(
+                py,
+                "PySparkKeyError",
+                "KEY_NOT_EXISTS",
+                &[("key", &name)],
+            ));
         }
         if let Ok(slice) = key.cast::<pyo3::types::PySlice>() {
             let idx = slice.indices(self.fields.len() as isize)?;
@@ -1739,25 +1762,30 @@ impl PyStructType {
             }
             return Ok(py_new_struct(py, out)?.into_any());
         }
-        let i: isize = key.extract().map_err(|_| {
-            PyErr::new::<pyo3::exceptions::PyTypeError, _>(
-                "StructType indices must be integers, slices, or field names",
-            )
-        })?;
-        let n = self.fields.len() as isize;
-        let idx = if i < 0 { i + n } else { i };
-        if idx < 0 || idx >= n {
-            return Err(PyErr::new::<pyo3::exceptions::PyIndexError, _>(
-                "StructType index out of range",
+        if !key.is_instance_of::<pyo3::types::PyInt>() {
+            return Err(crate::stat_arguments::not_expected_type(
+                key,
+                "key",
+                "int, slice or str",
             ));
         }
-        Ok(Py::new(
-            py,
-            PyStructField {
-                field: self.fields[idx as usize].clone(),
-            },
-        )?
-        .into_any())
+        let n = self.fields.len() as isize;
+        let index = key.extract::<isize>().ok();
+        match index.map(|i| if i < 0 { i + n } else { i }) {
+            Some(i) if (0..n).contains(&i) => Ok(Py::new(
+                py,
+                PyStructField {
+                    field: self.fields[i as usize].clone(),
+                },
+            )?
+            .into_any()),
+            _ => Err(raise_pyspark(
+                py,
+                "PySparkIndexError",
+                "INDEX_OUT_OF_RANGE",
+                &[("arg_name", "StructType"), ("index", 
&key.str()?.to_cow()?)],
+            )),
+        }
     }
 
     fn __iter__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index a5f9d88..062dfc6 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -322,7 +322,8 @@ def _parse_datatype_json_value(v):
         try:
             return parse_datatype_json(_json.dumps(v))
         except ValueError:
-            raise ValueError(f"cannot parse datatype json: {v!r}") from None
+            # As PySpark's, of a str that is not the JSON value of a data type.
+            raise _pyspark_value_error("CANNOT_PARSE_DATATYPE", {"msg": 
str(v)}) from None
     elif isinstance(v, dict):
         t = v.get("type")
         if t == "struct":
@@ -349,7 +350,15 @@ def _parse_datatype_json_value(v):
             )
         elif t == "udt":
             return UserDefinedType.fromJson(v)
-    raise ValueError(f"cannot parse datatype json: {v!r}")
+        # As PySpark's, of an object of a type that is not one of a data type.
+        raise _pyspark_value_error("UNSUPPORTED_DATA_TYPE", {"data_type": 
str(t)})
+    raise _pyspark_value_error("CANNOT_PARSE_DATATYPE", {"msg": str(v)})
+
+
+def _pyspark_value_error(error_class, parameters):
+    from pyspark.errors import PySparkValueError
+
+    return PySparkValueError(errorClass=error_class, 
messageParameters=parameters)
 
 
 def _drop_metadata(d):
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 4b51965..54dad46 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1601,6 +1601,20 @@ def test_sql_runs_statements_eagerly(session):
     assert [tuple(r) for r in query.collect()] == [(1,)]
 
 
+def test_struct_types_take_their_keys_and_fields_as_pyspark(session):
+    # As PySpark's StructType: add takes the JSON value of a data type, and a 
missing field is
+    # KEY_NOT_EXISTS.
+    from pyspark.errors import PySparkKeyError
+    from pyspark.sql import types as T
+
+    schema = T.StructType().add("a", "long").add("b", "string")
+    df = session.createDataFrame([(1, "x")], schema)
+    assert df.schema.simpleString() == "struct<a:bigint,b:string>"
+    with pytest.raises(PySparkKeyError) as e:
+        df.schema["zz"]
+    assert e.value.getCondition() == "KEY_NOT_EXISTS"
+
+
 def test_lists_and_numpy_arrays_are_arrays_as_pyspark(session):
     np = pytest.importorskip("numpy")
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index ad7aa3f..afbc757 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2499,13 +2499,53 @@ def test_struct_type_introspection():
     with pytest.raises(KeyError):
         _ = st["nope"]
     # chainable add
-    st2 = T.StructType().add("x", "int").add("y", T.StringType(), False).add(
+    st2 = T.StructType().add("x", "integer").add("y", T.StringType(), 
False).add(
         T.StructField("z", T.IntegerType())
     )
     assert st2.fieldNames() == ["x", "y", "z"]
     assert isinstance(st2, T.StructType)
 
 
+def test_struct_types_take_their_keys_and_fields_as_pyspark():
+    # As PySpark's StructType: __getitem__ raises KEY_NOT_EXISTS of a name,
+    # INDEX_OUT_OF_RANGE of an index and NOT_EXPECTED_TYPE of another key, and 
add takes the
+    # JSON value of a data type as a str, and requires a data type of a name.
+    from pyspark.errors import (
+        PySparkIndexError,
+        PySparkKeyError,
+        PySparkTypeError,
+        PySparkValueError,
+    )
+    from pyspark.sql import types as T
+
+    st = T.StructType([T.StructField("a", T.IntegerType())])
+    with pytest.raises(PySparkKeyError) as e:
+        st["zz"]
+    assert e.value.getCondition() == "KEY_NOT_EXISTS"
+    assert e.value.getMessageParameters() == {"key": "zz"}
+    for index in (1, -2):
+        with pytest.raises(PySparkIndexError) as e:
+            st[index]
+        assert e.value.getMessageParameters() == {"arg_name": "StructType", 
"index": str(index)}
+    assert st[-1].name == "a"
+    with pytest.raises(PySparkTypeError) as e:
+        st[1.5]
+    assert e.value.getMessageParameters() == {
+        "expected_type": "int, slice or str",
+        "arg_name": "key",
+        "arg_type": "float",
+    }
+    added = T.StructType().add("a", "integer").add("b", "decimal(10,2)", False)
+    assert added.simpleString() == "struct<a:int,b:decimal(10,2)>"
+    assert added["b"].nullable is False
+    with pytest.raises(PySparkValueError) as e:
+        T.StructType().add("a", "int")
+    assert e.value.getCondition() == "CANNOT_PARSE_DATATYPE"
+    with pytest.raises(PySparkValueError) as e:
+        T.StructType().add("a")
+    assert e.value.getCondition() == "ARGUMENT_REQUIRED"
+
+
 def test_struct_field_attributes():
     """StructField exposes name / dataType / nullable / metadata and a 
faithful repr/eq."""
     f = T.StructField("m", T.IntegerType(), True, {"k": "v"})


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to