This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 50fb840c [SPARK-61137][CONNECT][PYTHON] Make StructType give back the 
same StructField objects in the Python client as PySpark does
50fb840c is described below

commit 50fb840c4fa040c09c2d45c08aea56ee39fe4881
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 12:43:45 2026 +0900

    [SPARK-61137][CONNECT][PYTHON] Make StructType give back the same 
StructField objects in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61137](https://issues.apache.org/jira/browse/SPARK-61137): PySpark's 
`StructType` holds its `StructField` objects. So `struct["f1"]`, `struct[0]`, 
the iteration, the slices and `struct.fields` give back the same objects: 
`struct["f1"] is struct.fields[0]`.
    
    The client's `StructType` now does the same:
    - it keeps the `StructField` objects that it is made of (`StructType([f1, 
...])`) or that are added (`add(field)`)
    - it makes the objects of its other fields once, such as the fields added 
by name or those of a schema of the server
    - each access gives back the same object, and a slice is a struct of the 
same objects, as with PySpark
    
    A `StructField` of the client cannot change, so its object is always that 
of its field in the Rust core.
    
    ### Why are the changes needed?
    
    The client's `StructType` made a new `StructField` object at each access, 
so `struct["f1"] is struct.fields[0]` was `False`. Apache Spark's 
`test_struct_type` failed on this client (`assertIs`).
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. The fields of a `StructType` are the same objects at each access, as 
with PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new offline test checks the identity of the fields across:
      - `fields`, the items of names and indexes, the iteration and the slices
      - a struct made of `StructField` objects, one of fields added by name, 
one of an added `StructField`, and one parsed from JSON
    
      It passes with official PySpark 4.3.0, and fails without this change.
    - Apache Spark's `test_struct_type`, of `test_parity_types.py`, passes on 
this client against a local server.
    - The lib tests of both crates (590 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2202 passed, 8 skipped), and the 
live drop-in and Arrow suites (184 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #331 from HyukjinKwon/SPARK-61137-struct-field-objects.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/types.rs      | 109 ++++++++++++++++++++++++------------
 python/tests/test_dropin_offline.py |  20 +++++++
 2 files changed, 94 insertions(+), 35 deletions(-)

diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index f75050b6..88d01c45 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -181,13 +181,27 @@ macro_rules! init_chain {
 
 /// Construct a `StructType` Python object (with its full base chain) from 
fields.
 fn py_new_struct(py: Python<'_>, fields: Vec<StructField>) -> 
PyResult<Py<PyStructType>> {
+    let objects = std::iter::repeat_with(|| None).take(fields.len()).collect();
+    py_new_struct_of(py, fields, objects)
+}
+
+/// Construct a `StructType` Python object of `fields`, of the `StructField` 
objects of
+/// `objects`, where they are known, as a slice of a struct is of those of the 
struct.
+fn py_new_struct_of(
+    py: Python<'_>,
+    fields: Vec<StructField>,
+    objects: Vec<Option<Py<PyStructField>>>,
+) -> PyResult<Py<PyStructType>> {
     Py::new(
         py,
         init_chain!(
             DataType::Struct {
                 fields: fields.clone(),
             },
-            PyStructType { fields },
+            PyStructType {
+                fields,
+                objects: std::sync::Mutex::new(objects),
+            },
             [],
             value
         ),
@@ -2079,6 +2093,39 @@ impl PyStructField {
 #[pyclass(name = "StructType", module = "pyspark.sql.types", extends = 
PyDataType)]
 pub struct PyStructType {
     pub(crate) fields: Vec<StructField>,
+    /// The `StructField` objects of `fields`, by their positions: those that 
the struct was
+    /// made of, and the others once they are made. So a field is the same 
object each time,
+    /// as PySpark's `StructType`, which holds its fields, gives them. A 
`StructField` cannot
+    /// change, so its object is always that of its field.
+    objects: std::sync::Mutex<Vec<Option<Py<PyStructField>>>>,
+}
+
+impl PyStructType {
+    /// The `StructField` object of the field at `index`, which is in range.
+    fn field_object(&self, py: Python<'_>, index: usize) -> 
PyResult<Py<PyStructField>> {
+        let mut objects = self.objects.lock().unwrap_or_else(|e| 
e.into_inner());
+        if objects.len() < self.fields.len() {
+            objects.resize_with(self.fields.len(), || None);
+        }
+        if let Some(object) = &objects[index] {
+            return Ok(object.clone_ref(py));
+        }
+        let object = Py::new(
+            py,
+            PyStructField {
+                field: self.fields[index].clone(),
+            },
+        )?;
+        objects[index] = Some(object.clone_ref(py));
+        Ok(object)
+    }
+
+    /// The `StructField` objects of the fields, in order.
+    fn field_objects(&self, py: Python<'_>) -> 
PyResult<Vec<Py<PyStructField>>> {
+        (0..self.fields.len())
+            .map(|index| self.field_object(py, index))
+            .collect()
+    }
 }
 
 #[pymethods]
@@ -2121,12 +2168,7 @@ impl PyStructType {
             fields: this.fields.clone(),
         };
         reduce_with_udts(slf.get_type(), &dt, || {
-            let fields = this
-                .fields
-                .iter()
-                .map(|f| Py::new(py, PyStructField { field: f.clone() }))
-                .collect::<PyResult<Vec<_>>>()?;
-            (fields,).into_pyobject(py)
+            (this.field_objects(py)?,).into_pyobject(py)
         })
     }
     /// `StructType(fields=None)` where each field is a `StructField` (matching
@@ -2135,10 +2177,12 @@ impl PyStructType {
     #[pyo3(signature = (fields=None))]
     fn new(fields: Option<Vec<Bound<'_, PyAny>>>) -> 
PyResult<pyo3::PyClassInitializer<Self>> {
         let mut out = Vec::new();
+        let mut objects = Vec::new();
         if let Some(fs) = fields {
             for f in fs {
-                if let Ok(sf) = f.extract::<PyRef<PyStructField>>() {
-                    out.push(sf.field.clone());
+                if let Ok(sf) = f.cast::<PyStructField>() {
+                    out.push(sf.borrow().field.clone());
+                    objects.push(Some(sf.clone().unbind()));
                 } else if let Ok(s) = f.extract::<String>() {
                     // "name type" DDL fragment.
                     let dt = 
DataType::from_ddl(&format!("struct<{s}>")).map_err(|e| {
@@ -2147,6 +2191,7 @@ impl PyStructType {
                         ))
                     })?;
                     if let DataType::Struct { fields } = dt {
+                        objects.extend(fields.iter().map(|_| None));
                         out.extend(fields);
                     }
                 } else {
@@ -2160,7 +2205,10 @@ impl PyStructType {
             DataType::Struct {
                 fields: out.clone(),
             },
-            PyStructType { fields: out },
+            PyStructType {
+                fields: out,
+                objects: std::sync::Mutex::new(objects),
+            },
             [],
             value
         ))
@@ -2176,8 +2224,10 @@ impl PyStructType {
         nullable: bool,
         metadata: Option<&Bound<'_, pyo3::types::PyDict>>,
     ) -> PyResult<Bound<'py, Self>> {
-        let new_field = if let Ok(sf) = 
field.extract::<PyRef<PyStructField>>() {
-            sf.field.clone()
+        let mut object = None;
+        let new_field = if let Ok(sf) = field.cast::<PyStructField>() {
+            object = Some(sf.clone().unbind());
+            sf.borrow().field.clone()
         } else {
             // As PySpark's: a name takes a data type, of which a str is the 
JSON value of one.
             let Some(data_type) = data_type else {
@@ -2215,6 +2265,10 @@ impl PyStructType {
         };
         {
             let mut me = slf.borrow_mut();
+            let length = me.fields.len();
+            let objects = me.objects.get_mut().unwrap_or_else(|e| 
e.into_inner());
+            objects.resize_with(length, || None);
+            objects.push(object);
             me.fields.push(new_field);
             // The base `DataType` is what `str()` and the schemas of 
`mapInArrow` and
             // friends read, so it must follow the fields.
@@ -2227,10 +2281,7 @@ impl PyStructType {
     /// The fields as `StructField` objects (mirrors `StructType.fields`).
     #[getter]
     fn fields(&self, py: Python<'_>) -> PyResult<Vec<Py<PyStructField>>> {
-        self.fields
-            .iter()
-            .map(|f| Py::new(py, PyStructField { field: f.clone() }))
-            .collect()
+        self.field_objects(py)
     }
 
     /// The field names in order (mirrors `StructType.names`).
@@ -2251,10 +2302,8 @@ impl PyStructType {
         use crate::column::raise_pyspark;
         if let Ok(name) = key.cast::<pyo3::types::PyString>() {
             let name = name.to_cow()?;
-            for f in &self.fields {
-                if f.name == name {
-                    return Ok(Py::new(py, PyStructField { field: f.clone() 
})?.into_any());
-                }
+            if let Some(index) = self.fields.iter().position(|f| f.name == 
name) {
+                return Ok(self.field_object(py, index)?.into_any());
             }
             return Err(raise_pyspark(
                 py,
@@ -2265,13 +2314,14 @@ impl PyStructType {
         }
         if let Ok(slice) = key.cast::<pyo3::types::PySlice>() {
             let idx = slice.indices(self.fields.len() as isize)?;
-            let mut out = Vec::new();
+            let (mut out, mut objects) = (Vec::new(), Vec::new());
             let (mut i, stop, step) = (idx.start, idx.stop, idx.step);
             while (step > 0 && i < stop) || (step < 0 && i > stop) {
                 out.push(self.fields[i as usize].clone());
+                objects.push(Some(self.field_object(py, i as usize)?));
                 i += step;
             }
-            return Ok(py_new_struct(py, out)?.into_any());
+            return Ok(py_new_struct_of(py, out, objects)?.into_any());
         }
         if !key.is_instance_of::<pyo3::types::PyInt>() {
             return Err(crate::stat_arguments::not_expected_type(
@@ -2283,13 +2333,7 @@ impl PyStructType {
         let n = self.fields.len() as isize;
         let index = key.extract::<isize>().ok();
         match index.map(|i| if i < 0 { i + n } else { i }) {
-            Some(i) if (0..n).contains(&i) => Ok(Py::new(
-                py,
-                PyStructField {
-                    field: self.fields[i as usize].clone(),
-                },
-            )?
-            .into_any()),
+            Some(i) if (0..n).contains(&i) => Ok(self.field_object(py, i as 
usize)?.into_any()),
             _ => Err(raise_pyspark(
                 py,
                 "PySparkIndexError",
@@ -2300,12 +2344,7 @@ impl PyStructType {
     }
 
     fn __iter__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
-        let items: Vec<Py<PyStructField>> = self
-            .fields
-            .iter()
-            .map(|f| Py::new(py, PyStructField { field: f.clone() }))
-            .collect::<PyResult<_>>()?;
-        let list = pyo3::types::PyList::new(py, items)?;
+        let list = pyo3::types::PyList::new(py, self.field_objects(py)?)?;
         Ok(list.as_any().call_method0("__iter__")?.unbind())
     }
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 599a8daa..91098843 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3400,6 +3400,26 @@ def test_util_print_exec_and_skip_env(monkeypatch):
     assert util.is_remote_only() is True
 
 
+def test_struct_types_give_back_their_struct_field_objects_as_pyspark():
+    # As PySpark's StructType, which holds its fields: the fields, the items 
of a name or an
+    # index, the iteration and a slice give back the StructField objects that 
the struct was
+    # made of or added, and the same objects of the others each time.
+    f1 = T.StructField("f1", T.StringType(), True)
+    struct = T.StructType([f1, T.StructField("f2", T.IntegerType())])
+    assert struct.fields[0] is f1 and struct["f1"] is f1 and struct[0] is f1
+    assert list(struct)[0] is f1 and struct[0:1].fields[0] is f1
+    assert struct[1] is struct["f2"] is struct.fields[1] and struct[-1] is 
struct.fields[1]
+    added = T.StructType().add("f1", T.StringType(), True).add("f2", 
T.StringType(), True, None)
+    assert added["f1"] is added.fields[0] and added[0] is added.fields[0]
+    assert added[0:1] == T.StructType(added.fields[0:1])
+    g = T.StructField("g", T.LongType())
+    assert added.add(g).fields[2] is g and added["g"] is g and added.names == 
["f1", "f2", "g"]
+    parsed = T._parse_datatype_json_string(struct.json())
+    assert parsed["f1"] is parsed.fields[0] and parsed == struct
+
+
+
+
 def test_struct_type_introspection():
     """StructType.fields / names / __getitem__ / __iter__ / __len__ + 
chainable add."""
     st = T.StructType(


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to