This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 7b9bdd6 [SPARK-61073][CONNECT][PYTHON] Raise KEY_NOT_EXISTS for a
missing field of a StructType, and parse the JSON type of StructType.add, in
the Python client as PySpark does
7b9bdd6 is described below
commit 7b9bdd6ceb94c40340086aa06f1a01c8bb02c804
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 13:22:23 2026 +0900
[SPARK-61073][CONNECT][PYTHON] Raise KEY_NOT_EXISTS for a missing field of
a StructType, and parse the JSON type of StructType.add, in the Python client
as PySpark does
### What changes were proposed in this pull request?
[SPARK-61073](https://issues.apache.org/jira/browse/SPARK-61073):
`StructType[key]` and `StructType.add` now take their arguments and raise their
errors as PySpark's do.
- `StructType[name]` raises the `PySparkKeyError` `KEY_NOT_EXISTS` for a
name that no field has.
- `StructType[index]` raises the `PySparkIndexError` `INDEX_OUT_OF_RANGE`
for an index outside the fields. It raises `NOT_EXPECTED_TYPE` for a key that
is not an int, a slice or a str.
- `StructType.add(name, data_type)` parses a str `data_type` as the JSON
value of a data type, such as `"integer"` or `"decimal(10,2)"`. For a str that
is not one, such as `"int"`, it raises `CANNOT_PARSE_DATATYPE`. For a name
without a data type, it raises `ARGUMENT_REQUIRED`.
- `_parse_datatype_json_value` raises PySpark's `CANNOT_PARSE_DATATYPE` and
`UNSUPPORTED_DATA_TYPE`.
### Why are the changes needed?
The client raised these errors instead:
- a plain `KeyError: 'No StructField named zz'` for a missing field
- a plain `IndexError` and `TypeError` for the other keys
`add` also differed:
- it parsed a str data type as DDL, so `add("a", "int")` was accepted
- it took a `StringType` for a name without a data type
- it raised a plain `ValueError` for a type that does not parse
### Does this PR introduce _any_ user-facing change?
Yes. `StructType[key]` and `StructType.add` take the arguments, and raise
the errors, of PySpark 4.3.0.
### How was this patch tested?
- A new live test builds a schema with `add` and the JSON values of types,
creates a DataFrame of it, and checks the error for a missing field of its
schema. It passes with official PySpark 4.3.0 and fails without this change.
- A new offline test checks the errors of `StructType[key]` for names,
indexes and other keys. It also checks the types that `add` parses, and its
errors.
- The lib tests of both crates (583 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1930 passed, 8 skipped), and the
live drop-in and Arrow suites (127 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #271 from HyukjinKwon/SPARK-61073-struct-access.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/types.rs | 84 ++++++++++++++++++++++++-------------
python/pyspark/sql/types.py | 13 +++++-
python/tests/test_dropin_e2e.py | 14 +++++++
python/tests/test_dropin_offline.py | 42 ++++++++++++++++++-
4 files changed, 122 insertions(+), 31 deletions(-)
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 6e45a9f..a333443 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -1656,7 +1656,7 @@ impl PyStructType {
}
/// Append a field and return `self` (chainable, mirroring
`StructType.add`):
- /// `StructType().add("a", "int").add("b", StringType())`.
+ /// `StructType().add("a", "integer").add("b", StringType())`.
#[pyo3(signature = (field, data_type=None, nullable=true, metadata=None))]
fn add<'py>(
slf: Bound<'py, Self>,
@@ -1668,12 +1668,28 @@ impl PyStructType {
let new_field = if let Ok(sf) =
field.extract::<PyRef<PyStructField>>() {
sf.field.clone()
} else {
+ // As PySpark's: a name takes a data type, of which a str is the
JSON value of one.
+ let Some(data_type) = data_type else {
+ return Err(crate::column::raise_pyspark(
+ field.py(),
+ "PySparkValueError",
+ "ARGUMENT_REQUIRED",
+ &[
+ ("arg_name", "data_type"),
+ ("condition", "passing name of struct_field to
create"),
+ ],
+ ));
+ };
let name: String = field.extract()?;
- let dt = match data_type {
- Some(d) => py_to_data_type(d)?,
- None => DataType::String {
- collation: "UTF8_BINARY".to_string(),
- },
+ let dt = if data_type.is_instance_of::<pyo3::types::PyString>() {
+ let parsed = field
+ .py()
+ .import("pyspark.sql.types")?
+ .getattr("_parse_datatype_json_value")?
+ .call1((data_type,))?;
+ py_to_data_type(&parsed)?
+ } else {
+ py_to_data_type(data_type)?
};
let md = match metadata {
Some(d) => py_metadata_to_map(d.py(), d)?,
@@ -1717,17 +1733,24 @@ impl PyStructType {
}
/// Index by position (`st[0]`) or by name (`st["a"]`), returning a
`StructField`
- /// (mirrors `StructType.__getitem__`; a slice yields a new `StructType`).
+ /// (mirrors `StructType.__getitem__`; a slice yields a new `StructType`),
with the errors of
+ /// PySpark's: `KEY_NOT_EXISTS` of a name, `INDEX_OUT_OF_RANGE` of an
index, and
+ /// `NOT_EXPECTED_TYPE` of another key.
fn __getitem__(&self, py: Python<'_>, key: &Bound<'_, PyAny>) ->
PyResult<Py<PyAny>> {
- if let Ok(name) = key.extract::<String>() {
+ use crate::column::raise_pyspark;
+ if let Ok(name) = key.cast::<pyo3::types::PyString>() {
+ let name = name.to_cow()?;
for f in &self.fields {
if f.name == name {
return Ok(Py::new(py, PyStructField { field: f.clone()
})?.into_any());
}
}
- return Err(PyErr::new::<pyo3::exceptions::PyKeyError, _>(format!(
- "No StructField named {name}"
- )));
+ return Err(raise_pyspark(
+ py,
+ "PySparkKeyError",
+ "KEY_NOT_EXISTS",
+ &[("key", &name)],
+ ));
}
if let Ok(slice) = key.cast::<pyo3::types::PySlice>() {
let idx = slice.indices(self.fields.len() as isize)?;
@@ -1739,25 +1762,30 @@ impl PyStructType {
}
return Ok(py_new_struct(py, out)?.into_any());
}
- let i: isize = key.extract().map_err(|_| {
- PyErr::new::<pyo3::exceptions::PyTypeError, _>(
- "StructType indices must be integers, slices, or field names",
- )
- })?;
- let n = self.fields.len() as isize;
- let idx = if i < 0 { i + n } else { i };
- if idx < 0 || idx >= n {
- return Err(PyErr::new::<pyo3::exceptions::PyIndexError, _>(
- "StructType index out of range",
+ if !key.is_instance_of::<pyo3::types::PyInt>() {
+ return Err(crate::stat_arguments::not_expected_type(
+ key,
+ "key",
+ "int, slice or str",
));
}
- Ok(Py::new(
- py,
- PyStructField {
- field: self.fields[idx as usize].clone(),
- },
- )?
- .into_any())
+ let n = self.fields.len() as isize;
+ let index = key.extract::<isize>().ok();
+ match index.map(|i| if i < 0 { i + n } else { i }) {
+ Some(i) if (0..n).contains(&i) => Ok(Py::new(
+ py,
+ PyStructField {
+ field: self.fields[i as usize].clone(),
+ },
+ )?
+ .into_any()),
+ _ => Err(raise_pyspark(
+ py,
+ "PySparkIndexError",
+ "INDEX_OUT_OF_RANGE",
+ &[("arg_name", "StructType"), ("index",
&key.str()?.to_cow()?)],
+ )),
+ }
}
fn __iter__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index a5f9d88..062dfc6 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -322,7 +322,8 @@ def _parse_datatype_json_value(v):
try:
return parse_datatype_json(_json.dumps(v))
except ValueError:
- raise ValueError(f"cannot parse datatype json: {v!r}") from None
+ # As PySpark's, of a str that is not the JSON value of a data type.
+ raise _pyspark_value_error("CANNOT_PARSE_DATATYPE", {"msg":
str(v)}) from None
elif isinstance(v, dict):
t = v.get("type")
if t == "struct":
@@ -349,7 +350,15 @@ def _parse_datatype_json_value(v):
)
elif t == "udt":
return UserDefinedType.fromJson(v)
- raise ValueError(f"cannot parse datatype json: {v!r}")
+ # As PySpark's, of an object of a type that is not one of a data type.
+ raise _pyspark_value_error("UNSUPPORTED_DATA_TYPE", {"data_type":
str(t)})
+ raise _pyspark_value_error("CANNOT_PARSE_DATATYPE", {"msg": str(v)})
+
+
+def _pyspark_value_error(error_class, parameters):
+ from pyspark.errors import PySparkValueError
+
+ return PySparkValueError(errorClass=error_class,
messageParameters=parameters)
def _drop_metadata(d):
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 4b51965..54dad46 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1601,6 +1601,20 @@ def test_sql_runs_statements_eagerly(session):
assert [tuple(r) for r in query.collect()] == [(1,)]
+def test_struct_types_take_their_keys_and_fields_as_pyspark(session):
+ # As PySpark's StructType: add takes the JSON value of a data type, and a
missing field is
+ # KEY_NOT_EXISTS.
+ from pyspark.errors import PySparkKeyError
+ from pyspark.sql import types as T
+
+ schema = T.StructType().add("a", "long").add("b", "string")
+ df = session.createDataFrame([(1, "x")], schema)
+ assert df.schema.simpleString() == "struct<a:bigint,b:string>"
+ with pytest.raises(PySparkKeyError) as e:
+ df.schema["zz"]
+ assert e.value.getCondition() == "KEY_NOT_EXISTS"
+
+
def test_lists_and_numpy_arrays_are_arrays_as_pyspark(session):
np = pytest.importorskip("numpy")
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index ad7aa3f..afbc757 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2499,13 +2499,53 @@ def test_struct_type_introspection():
with pytest.raises(KeyError):
_ = st["nope"]
# chainable add
- st2 = T.StructType().add("x", "int").add("y", T.StringType(), False).add(
+ st2 = T.StructType().add("x", "integer").add("y", T.StringType(),
False).add(
T.StructField("z", T.IntegerType())
)
assert st2.fieldNames() == ["x", "y", "z"]
assert isinstance(st2, T.StructType)
+def test_struct_types_take_their_keys_and_fields_as_pyspark():
+ # As PySpark's StructType: __getitem__ raises KEY_NOT_EXISTS of a name,
+ # INDEX_OUT_OF_RANGE of an index and NOT_EXPECTED_TYPE of another key, and
add takes the
+ # JSON value of a data type as a str, and requires a data type of a name.
+ from pyspark.errors import (
+ PySparkIndexError,
+ PySparkKeyError,
+ PySparkTypeError,
+ PySparkValueError,
+ )
+ from pyspark.sql import types as T
+
+ st = T.StructType([T.StructField("a", T.IntegerType())])
+ with pytest.raises(PySparkKeyError) as e:
+ st["zz"]
+ assert e.value.getCondition() == "KEY_NOT_EXISTS"
+ assert e.value.getMessageParameters() == {"key": "zz"}
+ for index in (1, -2):
+ with pytest.raises(PySparkIndexError) as e:
+ st[index]
+ assert e.value.getMessageParameters() == {"arg_name": "StructType",
"index": str(index)}
+ assert st[-1].name == "a"
+ with pytest.raises(PySparkTypeError) as e:
+ st[1.5]
+ assert e.value.getMessageParameters() == {
+ "expected_type": "int, slice or str",
+ "arg_name": "key",
+ "arg_type": "float",
+ }
+ added = T.StructType().add("a", "integer").add("b", "decimal(10,2)", False)
+ assert added.simpleString() == "struct<a:int,b:decimal(10,2)>"
+ assert added["b"].nullable is False
+ with pytest.raises(PySparkValueError) as e:
+ T.StructType().add("a", "int")
+ assert e.value.getCondition() == "CANNOT_PARSE_DATATYPE"
+ with pytest.raises(PySparkValueError) as e:
+ T.StructType().add("a")
+ assert e.value.getCondition() == "ARGUMENT_REQUIRED"
+
+
def test_struct_field_attributes():
"""StructField exposes name / dataType / nullable / metadata and a
faithful repr/eq."""
f = T.StructField("m", T.IntegerType(), True, {"k": "v"})
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]