This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 50fb840c [SPARK-61137][CONNECT][PYTHON] Make StructType give back the
same StructField objects in the Python client as PySpark does
50fb840c is described below
commit 50fb840c4fa040c09c2d45c08aea56ee39fe4881
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 12:43:45 2026 +0900
[SPARK-61137][CONNECT][PYTHON] Make StructType give back the same
StructField objects in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61137](https://issues.apache.org/jira/browse/SPARK-61137): PySpark's
`StructType` holds its `StructField` objects. So `struct["f1"]`, `struct[0]`,
the iteration, the slices and `struct.fields` give back the same objects:
`struct["f1"] is struct.fields[0]`.
The client's `StructType` now does the same:
- it keeps the `StructField` objects that it is made of (`StructType([f1,
...])`) or that are added (`add(field)`)
- it makes the objects of its other fields once, such as the fields added
by name or those of a schema of the server
- each access gives back the same object, and a slice is a struct of the
same objects, as with PySpark
A `StructField` of the client cannot change, so its object is always that
of its field in the Rust core.
### Why are the changes needed?
The client's `StructType` made a new `StructField` object at each access,
so `struct["f1"] is struct.fields[0]` was `False`. Apache Spark's
`test_struct_type` failed on this client (`assertIs`).
### Does this PR introduce _any_ user-facing change?
Yes. The fields of a `StructType` are the same objects at each access, as
with PySpark 4.3.0.
### How was this patch tested?
- A new offline test checks the identity of the fields across:
- `fields`, the items of names and indexes, the iteration and the slices
- a struct made of `StructField` objects, one of fields added by name,
one of an added `StructField`, and one parsed from JSON
It passes with official PySpark 4.3.0, and fails without this change.
- Apache Spark's `test_struct_type`, of `test_parity_types.py`, passes on
this client against a local server.
- The lib tests of both crates (590 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2202 passed, 8 skipped), and the
live drop-in and Arrow suites (184 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #331 from HyukjinKwon/SPARK-61137-struct-field-objects.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/types.rs | 109 ++++++++++++++++++++++++------------
python/tests/test_dropin_offline.py | 20 +++++++
2 files changed, 94 insertions(+), 35 deletions(-)
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index f75050b6..88d01c45 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -181,13 +181,27 @@ macro_rules! init_chain {
/// Construct a `StructType` Python object (with its full base chain) from
fields.
fn py_new_struct(py: Python<'_>, fields: Vec<StructField>) ->
PyResult<Py<PyStructType>> {
+ let objects = std::iter::repeat_with(|| None).take(fields.len()).collect();
+ py_new_struct_of(py, fields, objects)
+}
+
+/// Construct a `StructType` Python object of `fields`, of the `StructField`
objects of
+/// `objects`, where they are known, as a slice of a struct is of those of the
struct.
+fn py_new_struct_of(
+ py: Python<'_>,
+ fields: Vec<StructField>,
+ objects: Vec<Option<Py<PyStructField>>>,
+) -> PyResult<Py<PyStructType>> {
Py::new(
py,
init_chain!(
DataType::Struct {
fields: fields.clone(),
},
- PyStructType { fields },
+ PyStructType {
+ fields,
+ objects: std::sync::Mutex::new(objects),
+ },
[],
value
),
@@ -2079,6 +2093,39 @@ impl PyStructField {
#[pyclass(name = "StructType", module = "pyspark.sql.types", extends =
PyDataType)]
pub struct PyStructType {
pub(crate) fields: Vec<StructField>,
+ /// The `StructField` objects of `fields`, by their positions: those that
the struct was
+ /// made of, and the others once they are made. So a field is the same
object each time,
+ /// as PySpark's `StructType`, which holds its fields, gives them. A
`StructField` cannot
+ /// change, so its object is always that of its field.
+ objects: std::sync::Mutex<Vec<Option<Py<PyStructField>>>>,
+}
+
+impl PyStructType {
+ /// The `StructField` object of the field at `index`, which is in range.
+ fn field_object(&self, py: Python<'_>, index: usize) ->
PyResult<Py<PyStructField>> {
+ let mut objects = self.objects.lock().unwrap_or_else(|e|
e.into_inner());
+ if objects.len() < self.fields.len() {
+ objects.resize_with(self.fields.len(), || None);
+ }
+ if let Some(object) = &objects[index] {
+ return Ok(object.clone_ref(py));
+ }
+ let object = Py::new(
+ py,
+ PyStructField {
+ field: self.fields[index].clone(),
+ },
+ )?;
+ objects[index] = Some(object.clone_ref(py));
+ Ok(object)
+ }
+
+ /// The `StructField` objects of the fields, in order.
+ fn field_objects(&self, py: Python<'_>) ->
PyResult<Vec<Py<PyStructField>>> {
+ (0..self.fields.len())
+ .map(|index| self.field_object(py, index))
+ .collect()
+ }
}
#[pymethods]
@@ -2121,12 +2168,7 @@ impl PyStructType {
fields: this.fields.clone(),
};
reduce_with_udts(slf.get_type(), &dt, || {
- let fields = this
- .fields
- .iter()
- .map(|f| Py::new(py, PyStructField { field: f.clone() }))
- .collect::<PyResult<Vec<_>>>()?;
- (fields,).into_pyobject(py)
+ (this.field_objects(py)?,).into_pyobject(py)
})
}
/// `StructType(fields=None)` where each field is a `StructField` (matching
@@ -2135,10 +2177,12 @@ impl PyStructType {
#[pyo3(signature = (fields=None))]
fn new(fields: Option<Vec<Bound<'_, PyAny>>>) ->
PyResult<pyo3::PyClassInitializer<Self>> {
let mut out = Vec::new();
+ let mut objects = Vec::new();
if let Some(fs) = fields {
for f in fs {
- if let Ok(sf) = f.extract::<PyRef<PyStructField>>() {
- out.push(sf.field.clone());
+ if let Ok(sf) = f.cast::<PyStructField>() {
+ out.push(sf.borrow().field.clone());
+ objects.push(Some(sf.clone().unbind()));
} else if let Ok(s) = f.extract::<String>() {
// "name type" DDL fragment.
let dt =
DataType::from_ddl(&format!("struct<{s}>")).map_err(|e| {
@@ -2147,6 +2191,7 @@ impl PyStructType {
))
})?;
if let DataType::Struct { fields } = dt {
+ objects.extend(fields.iter().map(|_| None));
out.extend(fields);
}
} else {
@@ -2160,7 +2205,10 @@ impl PyStructType {
DataType::Struct {
fields: out.clone(),
},
- PyStructType { fields: out },
+ PyStructType {
+ fields: out,
+ objects: std::sync::Mutex::new(objects),
+ },
[],
value
))
@@ -2176,8 +2224,10 @@ impl PyStructType {
nullable: bool,
metadata: Option<&Bound<'_, pyo3::types::PyDict>>,
) -> PyResult<Bound<'py, Self>> {
- let new_field = if let Ok(sf) =
field.extract::<PyRef<PyStructField>>() {
- sf.field.clone()
+ let mut object = None;
+ let new_field = if let Ok(sf) = field.cast::<PyStructField>() {
+ object = Some(sf.clone().unbind());
+ sf.borrow().field.clone()
} else {
// As PySpark's: a name takes a data type, of which a str is the
JSON value of one.
let Some(data_type) = data_type else {
@@ -2215,6 +2265,10 @@ impl PyStructType {
};
{
let mut me = slf.borrow_mut();
+ let length = me.fields.len();
+ let objects = me.objects.get_mut().unwrap_or_else(|e|
e.into_inner());
+ objects.resize_with(length, || None);
+ objects.push(object);
me.fields.push(new_field);
// The base `DataType` is what `str()` and the schemas of
`mapInArrow` and
// friends read, so it must follow the fields.
@@ -2227,10 +2281,7 @@ impl PyStructType {
/// The fields as `StructField` objects (mirrors `StructType.fields`).
#[getter]
fn fields(&self, py: Python<'_>) -> PyResult<Vec<Py<PyStructField>>> {
- self.fields
- .iter()
- .map(|f| Py::new(py, PyStructField { field: f.clone() }))
- .collect()
+ self.field_objects(py)
}
/// The field names in order (mirrors `StructType.names`).
@@ -2251,10 +2302,8 @@ impl PyStructType {
use crate::column::raise_pyspark;
if let Ok(name) = key.cast::<pyo3::types::PyString>() {
let name = name.to_cow()?;
- for f in &self.fields {
- if f.name == name {
- return Ok(Py::new(py, PyStructField { field: f.clone()
})?.into_any());
- }
+ if let Some(index) = self.fields.iter().position(|f| f.name ==
name) {
+ return Ok(self.field_object(py, index)?.into_any());
}
return Err(raise_pyspark(
py,
@@ -2265,13 +2314,14 @@ impl PyStructType {
}
if let Ok(slice) = key.cast::<pyo3::types::PySlice>() {
let idx = slice.indices(self.fields.len() as isize)?;
- let mut out = Vec::new();
+ let (mut out, mut objects) = (Vec::new(), Vec::new());
let (mut i, stop, step) = (idx.start, idx.stop, idx.step);
while (step > 0 && i < stop) || (step < 0 && i > stop) {
out.push(self.fields[i as usize].clone());
+ objects.push(Some(self.field_object(py, i as usize)?));
i += step;
}
- return Ok(py_new_struct(py, out)?.into_any());
+ return Ok(py_new_struct_of(py, out, objects)?.into_any());
}
if !key.is_instance_of::<pyo3::types::PyInt>() {
return Err(crate::stat_arguments::not_expected_type(
@@ -2283,13 +2333,7 @@ impl PyStructType {
let n = self.fields.len() as isize;
let index = key.extract::<isize>().ok();
match index.map(|i| if i < 0 { i + n } else { i }) {
- Some(i) if (0..n).contains(&i) => Ok(Py::new(
- py,
- PyStructField {
- field: self.fields[i as usize].clone(),
- },
- )?
- .into_any()),
+ Some(i) if (0..n).contains(&i) => Ok(self.field_object(py, i as
usize)?.into_any()),
_ => Err(raise_pyspark(
py,
"PySparkIndexError",
@@ -2300,12 +2344,7 @@ impl PyStructType {
}
fn __iter__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
- let items: Vec<Py<PyStructField>> = self
- .fields
- .iter()
- .map(|f| Py::new(py, PyStructField { field: f.clone() }))
- .collect::<PyResult<_>>()?;
- let list = pyo3::types::PyList::new(py, items)?;
+ let list = pyo3::types::PyList::new(py, self.field_objects(py)?)?;
Ok(list.as_any().call_method0("__iter__")?.unbind())
}
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 599a8daa..91098843 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3400,6 +3400,26 @@ def test_util_print_exec_and_skip_env(monkeypatch):
assert util.is_remote_only() is True
+def test_struct_types_give_back_their_struct_field_objects_as_pyspark():
+ # As PySpark's StructType, which holds its fields: the fields, the items
of a name or an
+ # index, the iteration and a slice give back the StructField objects that
the struct was
+ # made of or added, and the same objects of the others each time.
+ f1 = T.StructField("f1", T.StringType(), True)
+ struct = T.StructType([f1, T.StructField("f2", T.IntegerType())])
+ assert struct.fields[0] is f1 and struct["f1"] is f1 and struct[0] is f1
+ assert list(struct)[0] is f1 and struct[0:1].fields[0] is f1
+ assert struct[1] is struct["f2"] is struct.fields[1] and struct[-1] is
struct.fields[1]
+ added = T.StructType().add("f1", T.StringType(), True).add("f2",
T.StringType(), True, None)
+ assert added["f1"] is added.fields[0] and added[0] is added.fields[0]
+ assert added[0:1] == T.StructType(added.fields[0:1])
+ g = T.StructField("g", T.LongType())
+ assert added.add(g).fields[2] is g and added["g"] is g and added.names ==
["f1", "f2", "g"]
+ parsed = T._parse_datatype_json_string(struct.json())
+ assert parsed["f1"] is parsed.fields[0] and parsed == struct
+
+
+
+
def test_struct_type_introspection():
"""StructType.fields / names / __getitem__ / __iter__ / __len__ +
chainable add."""
st = T.StructType(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]