This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new b922b16  [SPARK-60959][CONNECT][PYTHON] Keep the values of a collected 
Row, and pickle VariantVal, as PySpark does
b922b16 is described below

commit b922b1624e68421b5e22ef33a79f2e81cf2c7d91
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 14:16:29 2026 +0900

    [SPARK-60959][CONNECT][PYTHON] Keep the values of a collected Row, and 
pickle VariantVal, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60959](https://issues.apache.org/jira/browse/SPARK-60959): rows and 
`VariantVal` behave as in PySpark 4.3.0.
    
    - **Collected rows keep their values.** A row that the server returned now 
makes the Python objects of its values when they are first read and keeps them, 
as a PySpark row is a tuple of them. So:
      - `row[0] is row[0]`, and `row.x is row[0]`;
      - a row equals itself, as a tuple compares its own elements by identity 
first, even with a NaN or a VARIANT value;
      - its hash stays the same.
    
      The values are made all at once on first read, as PySpark's row holds 
them all.
    - **`VariantVal` pickles** as `VariantVal(value, metadata)`, which 
PySpark's own class also takes. So a collected row of VARIANT values pickles, 
and unpickles on an official worker too.
    
    `VariantVal` has no `__eq__`, as in PySpark, whose class compares by 
identity.
    
    ### Why are the changes needed?
    
    A row made new Python objects of its values each time they were read, so:
    
    - `r == r` was `False` for `r = spark.sql("SELECT double('NaN') 
x").first()`, and for a row of a VARIANT value;
    - `hash(r)` could change between calls.
    
    `pickle.dumps(spark.sql("SELECT parse_json('1') v").collect())` also raised 
`TypeError: cannot pickle 'pyspark.sql.types.VariantVal' object`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, these behave as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - Against a local Spark 4.3.0 server, a probe of these rows gives the same 
results as official PySpark 4.3.0:
      - equality and hashes with a NaN and a VARIANT value;
      - value identity;
      - nested rows;
      - pickling of rows and of `VariantVal`.
    - A new offline test pickles a `VariantVal`, and a new live test checks the 
collected rows. Both also pass with official PySpark 4.3.0, and fail without 
this change.
    - The lib tests and every integration test, e2e included, against the local 
server (645 passed), the build of all tests with `wasm-udf`, the offline Python 
suites (648 passed, 8 live cases skipped), and the live drop-in and Arrow 
suites (64 passed) and live cases (10 passed). Clippy reports no diagnostics in 
the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #190 from HyukjinKwon/SPARK-60959-row-values.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/row.rs        | 40 ++++++++++++++++++++++++-------------
 crates/pyspark-rs/src/values.rs     | 16 +++++++++++++++
 python/tests/test_dropin_e2e.py     | 17 ++++++++++++++++
 python/tests/test_dropin_offline.py | 11 ++++++++++
 4 files changed, 70 insertions(+), 14 deletions(-)

diff --git a/crates/pyspark-rs/src/row.rs b/crates/pyspark-rs/src/row.rs
index 53d8b21..2dc524e 100644
--- a/crates/pyspark-rs/src/row.rs
+++ b/crates/pyspark-rs/src/row.rs
@@ -3,6 +3,7 @@
 use pyo3::basic::CompareOp;
 use pyo3::exceptions::PyIndexError;
 use pyo3::prelude::*;
+use pyo3::sync::PyOnceLock;
 use pyo3::types::{PyBytes, PyDict, PyList, PySlice, PyString, PyTuple};
 use spark_connect::row::{Row, Value};
 
@@ -20,8 +21,13 @@ pub struct PyRow {
 }
 
 enum RowData {
-    /// A row that the server returned, whose Spark values become Python 
values on access.
-    Spark(Row),
+    /// A row that the server returned, whose Spark values become Python 
values when first
+    /// read, and stay those objects, as a PySpark row is a tuple of them: so 
the row equals
+    /// itself, with a NaN or a VARIANT value, and keeps its hash.
+    Spark {
+        row: Row,
+        values: PyOnceLock<Py<PyTuple>>,
+    },
     /// A row made in Python, which holds its Python values as they are, as 
PySpark's does.
     Python {
         fields: Vec<String>,
@@ -32,7 +38,10 @@ enum RowData {
 impl PyRow {
     pub fn new(row: Row) -> Self {
         PyRow {
-            data: RowData::Spark(row),
+            data: RowData::Spark {
+                row,
+                values: PyOnceLock::new(),
+            },
             named: true,
         }
     }
@@ -50,7 +59,7 @@ impl PyRow {
 
     fn fields(&self) -> &[String] {
         match &self.data {
-            RowData::Spark(row) => row.fields(),
+            RowData::Spark { row, .. } => row.fields(),
             RowData::Python { fields, .. } => fields,
         }
     }
@@ -58,13 +67,16 @@ impl PyRow {
     /// The values, as the tuple that a PySpark row is.
     fn values<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyTuple>> {
         match &self.data {
-            RowData::Spark(row) => {
-                let values = row
-                    .values()
-                    .iter()
-                    .map(|v| value_to_py(py, v))
-                    .collect::<PyResult<Vec<_>>>()?;
-                PyTuple::new(py, values)
+            RowData::Spark { row, values } => {
+                let values = values.get_or_try_init(py, || -> 
PyResult<Py<PyTuple>> {
+                    let values = row
+                        .values()
+                        .iter()
+                        .map(|v| value_to_py(py, v))
+                        .collect::<PyResult<Vec<_>>>()?;
+                    Ok(PyTuple::new(py, values)?.unbind())
+                })?;
+                Ok(values.bind(py).clone())
             }
             RowData::Python { values, .. } => Ok(values.bind(py).clone()),
         }
@@ -73,7 +85,7 @@ impl PyRow {
     /// The value of the field at `index`, which is in range.
     fn value<'py>(&self, py: Python<'py>, index: usize) -> PyResult<Bound<'py, 
PyAny>> {
         match &self.data {
-            RowData::Spark(row) => value_to_py(py, row.get_unchecked(index)),
+            RowData::Spark { .. } => self.values(py)?.get_item(index),
             RowData::Python { values, .. } => values.bind(py).get_item(index),
         }
     }
@@ -103,7 +115,7 @@ impl PyRow {
     /// row made in Python as it converts those of a tuple.
     pub(crate) fn to_row(&self, py: Python<'_>) -> PyResult<Row> {
         match &self.data {
-            RowData::Spark(row) => Ok(row.clone()),
+            RowData::Spark { row, .. } => Ok(row.clone()),
             RowData::Python { fields, values } => {
                 let values = values
                     .bind(py)
@@ -346,7 +358,7 @@ impl PyRow {
         names.truncate(fields.len());
         names.extend(fields[names.len()..].iter().cloned());
         match &mut self.data {
-            RowData::Spark(row) => *row = Row::new(names, 
row.values().to_vec()),
+            RowData::Spark { row, .. } => *row = Row::new(names, 
row.values().to_vec()),
             RowData::Python { fields, .. } => *fields = names,
         }
         self.named = true;
diff --git a/crates/pyspark-rs/src/values.rs b/crates/pyspark-rs/src/values.rs
index 0cea8f8..e3bcce0 100644
--- a/crates/pyspark-rs/src/values.rs
+++ b/crates/pyspark-rs/src/values.rs
@@ -25,6 +25,22 @@ impl PyVariantVal {
     fn new(value: Vec<u8>, metadata: Vec<u8>) -> Self {
         PyVariantVal { value, metadata }
     }
+    /// Pickled as `VariantVal(value, metadata)`, which PySpark's own class 
also takes.
+    fn __reduce__<'py>(slf: &Bound<'py, Self>) -> PyResult<Bound<'py, 
pyo3::types::PyTuple>> {
+        let py = slf.py();
+        let this = slf.borrow();
+        let args = (
+            PyBytes::new(py, &this.value),
+            PyBytes::new(py, &this.metadata),
+        );
+        pyo3::types::PyTuple::new(
+            py,
+            [
+                slf.get_type().into_any(),
+                args.into_pyobject(py)?.into_any(),
+            ],
+        )
+    }
     #[getter]
     fn value<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
         PyBytes::new(py, &self.value)
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 4de0fb8..30971b9 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -557,6 +557,23 @@ def 
test_column_attributes_select_struct_fields_and_slices_substrings(session):
     assert tuple(out.first()) == (1, "q", 1, "bcd", "ab")
 
 
+def test_collected_rows_keep_their_values_as_pyspark(session):
+    import pickle
+
+    # A row holds the same values for as long as it lives, as a tuple does, so 
it equals
+    # itself with a NaN or a VARIANT value, and keeps its hash.
+    row = session.sql("SELECT double('NaN') x, 1 y").first()
+    assert row == row and hash(row) == hash(row)
+    assert row[0] is row[0] and row.x is row[0]
+    variant = session.sql("""SELECT parse_json('{"a": 1}') v""").first()
+    assert variant == variant and hash(variant) == hash(variant)
+    nested = session.sql("SELECT named_struct('a', double('NaN')) s").first()
+    assert nested == nested and nested.s is nested.s
+    # Rows of VARIANT values pickle, as in PySpark.
+    rows = pickle.loads(pickle.dumps(session.sql("SELECT parse_json('1') v, 2 
n").collect()))
+    assert (rows[0].v.toJson(), rows[0].n) == ("1", 2)
+
+
 def test_sql_runs_statements_eagerly(session):
     # sql() sends the statement at once: a command takes effect without an 
action, and the
     # DataFrame of its result is the local relation the server returned.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index b38873a..7393ff4 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2287,6 +2287,17 @@ def test_spatial_types_construction():
     assert "geography" in gg.simpleString()
 
 
+def test_variant_val_pickles_as_pyspark():
+    import pickle
+
+    from pyspark.sql.types import VariantVal
+
+    # As PySpark's VariantVal(value, metadata), which pickles as a plain class 
does.
+    value = pickle.loads(pickle.dumps(VariantVal(b"\x0c\x01", 
b"\x01\x00\x00")))
+    assert type(value) is VariantVal
+    assert (value.value, value.metadata) == (b"\x0c\x01", b"\x01\x00\x00")
+
+
 def test_variant_values():
     """Test VariantVal and other variant types."""
     from pyspark.sql.types import VariantVal, Geometry, Geography


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to