This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 90434a5  [SPARK-61138][CONNECT][PYTHON] Make str of a VariantVal its 
JSON, and createDataFrame reject rows that are VariantVals, in the Python 
client as PySpark does
90434a5 is described below

commit 90434a534a80ebd755c0b99ed53136051480d95e
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 09:07:19 2026 +0900

    [SPARK-61138][CONNECT][PYTHON] Make str of a VariantVal its JSON, and 
createDataFrame reject rows that are VariantVals, in the Python client as 
PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61138](https://issues.apache.org/jira/browse/SPARK-61138): Two 
behaviors of `VariantVal` now match PySpark 4.3.0:
    - **`str` of a `VariantVal` is its JSON**, as PySpark's 
`VariantVal.__str__` (`VariantUtils.to_json`, in UTC, as `toJson()`). Its 
`repr` is still that of its bytes.
    - **`createDataFrame` rejects rows that are `VariantVal`s** with 
`PySparkValueError` "Rows cannot be of type VariantVal". This holds with a 
struct schema, without a schema, and with a schema that is not a struct, such 
as `"variant"`.
    
      PySpark's `VariantVal` has a `__dict__`, so a first item that is one is 
taken as a row, and `to_row` of `LocalDataToArrowConversion` rejects it. The 
client's `VariantVal`, a class of the Rust extension, has no `__dict__`, so the 
client took it as a single value. It is now taken as a row, as PySpark takes 
it. As with PySpark, rows whose first item is `None` are still rows of single 
values, so a `VariantVal` after that is a value.
    
    ### Why are the changes needed?
    
    - `str(v)` gave `VariantVal(b'...', b'...')`, not the JSON.
    - `createDataFrame([VariantVal.parseJson("2")], "v variant")` gave a 
DataFrame of one row, where PySpark raises.
    
    Apache Spark's `test_variant_type` failed on this client for both.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes:
    - `str` of a `VariantVal` is its JSON.
    - `createDataFrame` of rows that are `VariantVal`s raises 
`PySparkValueError`, as with PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new offline test checks:
      - `str` and `repr` of `VariantVal`s
      - the `PySparkValueError` of rows that are `VariantVal`s, of each kind of 
schema
      - the rows of a first `None`
    
      It passes with official PySpark 4.3.0, and fails without this change.
    - Apache Spark's variant tests of `test_parity_types.py`, with #326 
(SPARK-61133), which lets the file be collected, run on this client against a 
local server, and all 3 pass, including `test_variant_type`.
    - The lib tests of both crates (590 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2197 passed, 8 skipped), and the 
live drop-in and Arrow suites (182 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #327 from HyukjinKwon/SPARK-61138-variant-str.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/infer.rs      | 11 ++++++-----
 crates/pyspark-rs/src/session.rs    |  6 ++++++
 crates/pyspark-rs/src/values.rs     |  5 +++++
 python/tests/test_dropin_offline.py | 24 ++++++++++++++++++++++++
 4 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/crates/pyspark-rs/src/infer.rs b/crates/pyspark-rs/src/infer.rs
index 4825c72..7b0bc7a 100644
--- a/crates/pyspark-rs/src/infer.rs
+++ b/crates/pyspark-rs/src/infer.rs
@@ -1568,6 +1568,11 @@ pub(crate) fn value_field(data_type: DataType) -> 
StructField {
     nullable_field("value".to_string(), data_type)
 }
 
+/// What PySpark's `to_row` raises for a row that is a VariantVal.
+pub(crate) fn variant_row(py: Python<'_>) -> PyErr {
+    pyspark_message(py, "PySparkValueError", "Rows cannot be of type 
VariantVal")
+}
+
 /// The values of a row in the order of `fields`, as PySpark's `to_row` takes 
them: nulls for
 /// None, a tuple or a Row by position, a dict by name, another object by its 
`__dict__`, and
 /// another value, such as a list, by position.
@@ -1590,11 +1595,7 @@ pub(crate) fn row_values<'py>(
         return values_by_name(item, fields);
     }
     if item.is_instance_of::<PyVariantVal>() {
-        return Err(pyspark_message(
-            py,
-            "PySparkValueError",
-            "Rows cannot be of type VariantVal",
-        ));
+        return Err(variant_row(py));
     }
     if item.hasattr(intern!(py, "__dict__"))? {
         return values_by_name(&item.getattr(intern!(py, "__dict__"))?, fields);
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 89b3d27..1d82245 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -616,9 +616,12 @@ impl PySparkSession {
             }
             false
         } else {
+            // A VariantVal of PySpark has a `__dict__`, so that it is taken 
as a row, which
+            // PySpark's `to_row` rejects, as `row_values` does.
             !(first.is_instance_of::<crate::row::PyRow>()
                 || first.is_instance_of::<pyo3::types::PyTuple>()
                 || first.is_instance_of::<PyList>()
+                || first.is_instance_of::<crate::values::PyVariantVal>()
                 || first.hasattr("__dict__")?)
         };
 
@@ -633,6 +636,9 @@ impl PySparkSession {
                     let row_type = if single {
                         let items = infer::positional(py, vec![item.clone()], 
cols.as_mut());
                         infer::infer_fields(py, items, options)?
+                    } else if 
item.is_instance_of::<crate::values::PyVariantVal>() {
+                        // PySpark infers the type of its `__dict__`, then 
rejects the row.
+                        return Err(infer::variant_row(py));
                     } else {
                         infer::infer_schema(item, cols.as_mut(), options)?
                     };
diff --git a/crates/pyspark-rs/src/values.rs b/crates/pyspark-rs/src/values.rs
index 53d0fb7..92ab34e 100644
--- a/crates/pyspark-rs/src/values.rs
+++ b/crates/pyspark-rs/src/values.rs
@@ -83,6 +83,11 @@ impl PyVariantVal {
     fn __repr__(&self, py: Python<'_>) -> String {
         format!("VariantVal({:?}, {:?})", self.value(py), self.metadata(py))
     }
+    /// The JSON of the value, as PySpark's `VariantVal.__str__`: 
`VariantUtils.to_json` of it,
+    /// in UTC, as `toJson()`.
+    fn __str__(&self, py: Python<'_>) -> PyResult<String> {
+        self.toJson(py, "UTC")
+    }
 }
 
 /// A Geography value (WKB bytes + SRID). Mirrors 
`pyspark.sql.types.Geography`.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 553048a..5417a02 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -4675,6 +4675,30 @@ def test_variant_val_pickles_as_pyspark():
     assert (value.value, value.metadata) == (b"\x0c\x01", b"\x01\x00\x00")
 
 
+def test_str_of_variant_values_is_their_json_and_they_are_no_rows_as_pyspark():
+    # As PySpark's VariantVal: str is its JSON, in UTC, as toJson(); repr is 
of its bytes. And
+    # as PySpark's createDataFrame, whose VariantVal has a __dict__, so that 
it is a row, which
+    # to_row rejects: a VariantVal is no row, also without a schema or of one 
that is not a
+    # struct; but rows of a first None are rows of single values, of the 
VariantVals too.
+    from pyspark.errors import PySparkValueError
+    from pyspark.sql.types import VariantVal
+
+    v = VariantVal.parseJson('{"b": [1, "2", true, null], "a": 1.5}')
+    assert str(v) == v.toJson() == '{"a":1.5,"b":[1,"2",true,null]}'
+    assert repr(v) == "VariantVal(%r, %r)" % (v.value, v.metadata)
+    assert str(VariantVal.parseJson('"x"')) == '"x"'
+    assert "%s" % VariantVal.parseJson("1") == "1"
+
+    spark = _offline_session()
+    struct = T.StructType([T.StructField("v", T.VariantType())])
+    for data, schema in [([v], struct), ([v, None], struct), ([v], None), 
([v], T.VariantType())]:
+        with pytest.raises(PySparkValueError, match="Rows cannot be of type 
VariantVal"):
+            spark.createDataFrame(data, schema)
+    assert type(spark.createDataFrame([None, v], struct)).__name__ == 
"DataFrame"
+
+
+
+
 def test_variant_values():
     """Test VariantVal and other variant types."""
     from pyspark.sql.types import VariantVal, Geometry, Geography


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to