This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 90434a5 [SPARK-61138][CONNECT][PYTHON] Make str of a VariantVal its
JSON, and createDataFrame reject rows that are VariantVals, in the Python
client as PySpark does
90434a5 is described below
commit 90434a534a80ebd755c0b99ed53136051480d95e
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 09:07:19 2026 +0900
[SPARK-61138][CONNECT][PYTHON] Make str of a VariantVal its JSON, and
createDataFrame reject rows that are VariantVals, in the Python client as
PySpark does
### What changes were proposed in this pull request?
[SPARK-61138](https://issues.apache.org/jira/browse/SPARK-61138): Two
behaviors of `VariantVal` now match PySpark 4.3.0:
- **`str` of a `VariantVal` is its JSON**, as PySpark's
`VariantVal.__str__` (`VariantUtils.to_json`, in UTC, as `toJson()`). Its
`repr` is still that of its bytes.
- **`createDataFrame` rejects rows that are `VariantVal`s** with
`PySparkValueError` "Rows cannot be of type VariantVal". This holds with a
struct schema, without a schema, and with a schema that is not a struct, such
as `"variant"`.
PySpark's `VariantVal` has a `__dict__`, so a first item that is one is
taken as a row, and `to_row` of `LocalDataToArrowConversion` rejects it. The
client's `VariantVal`, a class of the Rust extension, has no `__dict__`, so the
client took it as a single value. It is now taken as a row, as PySpark takes
it. As with PySpark, rows whose first item is `None` are still rows of single
values, so a `VariantVal` after that is a value.
### Why are the changes needed?
- `str(v)` gave `VariantVal(b'...', b'...')`, not the JSON.
- `createDataFrame([VariantVal.parseJson("2")], "v variant")` gave a
DataFrame of one row, where PySpark raises.
Apache Spark's `test_variant_type` failed on this client for both.
### Does this PR introduce _any_ user-facing change?
Yes:
- `str` of a `VariantVal` is its JSON.
- `createDataFrame` of rows that are `VariantVal`s raises
`PySparkValueError`, as with PySpark 4.3.0.
### How was this patch tested?
- A new offline test checks:
- `str` and `repr` of `VariantVal`s
- the `PySparkValueError` of rows that are `VariantVal`s, of each kind of
schema
- the rows of a first `None`
It passes with official PySpark 4.3.0, and fails without this change.
- Apache Spark's variant tests of `test_parity_types.py`, with #326
(SPARK-61133), which lets the file be collected, run on this client against a
local server, and all 3 pass, including `test_variant_type`.
- The lib tests of both crates (590 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2197 passed, 8 skipped), and the
live drop-in and Arrow suites (182 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #327 from HyukjinKwon/SPARK-61138-variant-str.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/infer.rs | 11 ++++++-----
crates/pyspark-rs/src/session.rs | 6 ++++++
crates/pyspark-rs/src/values.rs | 5 +++++
python/tests/test_dropin_offline.py | 24 ++++++++++++++++++++++++
4 files changed, 41 insertions(+), 5 deletions(-)
diff --git a/crates/pyspark-rs/src/infer.rs b/crates/pyspark-rs/src/infer.rs
index 4825c72..7b0bc7a 100644
--- a/crates/pyspark-rs/src/infer.rs
+++ b/crates/pyspark-rs/src/infer.rs
@@ -1568,6 +1568,11 @@ pub(crate) fn value_field(data_type: DataType) ->
StructField {
nullable_field("value".to_string(), data_type)
}
+/// What PySpark's `to_row` raises for a row that is a VariantVal.
+pub(crate) fn variant_row(py: Python<'_>) -> PyErr {
+ pyspark_message(py, "PySparkValueError", "Rows cannot be of type
VariantVal")
+}
+
/// The values of a row in the order of `fields`, as PySpark's `to_row` takes
them: nulls for
/// None, a tuple or a Row by position, a dict by name, another object by its
`__dict__`, and
/// another value, such as a list, by position.
@@ -1590,11 +1595,7 @@ pub(crate) fn row_values<'py>(
return values_by_name(item, fields);
}
if item.is_instance_of::<PyVariantVal>() {
- return Err(pyspark_message(
- py,
- "PySparkValueError",
- "Rows cannot be of type VariantVal",
- ));
+ return Err(variant_row(py));
}
if item.hasattr(intern!(py, "__dict__"))? {
return values_by_name(&item.getattr(intern!(py, "__dict__"))?, fields);
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 89b3d27..1d82245 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -616,9 +616,12 @@ impl PySparkSession {
}
false
} else {
+ // A VariantVal of PySpark has a `__dict__`, so that it is taken
as a row, which
+ // PySpark's `to_row` rejects, as `row_values` does.
!(first.is_instance_of::<crate::row::PyRow>()
|| first.is_instance_of::<pyo3::types::PyTuple>()
|| first.is_instance_of::<PyList>()
+ || first.is_instance_of::<crate::values::PyVariantVal>()
|| first.hasattr("__dict__")?)
};
@@ -633,6 +636,9 @@ impl PySparkSession {
let row_type = if single {
let items = infer::positional(py, vec![item.clone()],
cols.as_mut());
infer::infer_fields(py, items, options)?
+ } else if
item.is_instance_of::<crate::values::PyVariantVal>() {
+ // PySpark infers the type of its `__dict__`, then
rejects the row.
+ return Err(infer::variant_row(py));
} else {
infer::infer_schema(item, cols.as_mut(), options)?
};
diff --git a/crates/pyspark-rs/src/values.rs b/crates/pyspark-rs/src/values.rs
index 53d0fb7..92ab34e 100644
--- a/crates/pyspark-rs/src/values.rs
+++ b/crates/pyspark-rs/src/values.rs
@@ -83,6 +83,11 @@ impl PyVariantVal {
fn __repr__(&self, py: Python<'_>) -> String {
format!("VariantVal({:?}, {:?})", self.value(py), self.metadata(py))
}
+ /// The JSON of the value, as PySpark's `VariantVal.__str__`:
`VariantUtils.to_json` of it,
+ /// in UTC, as `toJson()`.
+ fn __str__(&self, py: Python<'_>) -> PyResult<String> {
+ self.toJson(py, "UTC")
+ }
}
/// A Geography value (WKB bytes + SRID). Mirrors
`pyspark.sql.types.Geography`.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 553048a..5417a02 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -4675,6 +4675,30 @@ def test_variant_val_pickles_as_pyspark():
assert (value.value, value.metadata) == (b"\x0c\x01", b"\x01\x00\x00")
+def test_str_of_variant_values_is_their_json_and_they_are_no_rows_as_pyspark():
+ # As PySpark's VariantVal: str is its JSON, in UTC, as toJson(); repr is
of its bytes. And
+ # as PySpark's createDataFrame, whose VariantVal has a __dict__, so that
it is a row, which
+ # to_row rejects: a VariantVal is no row, also without a schema or of one
that is not a
+ # struct; but rows of a first None are rows of single values, of the
VariantVals too.
+ from pyspark.errors import PySparkValueError
+ from pyspark.sql.types import VariantVal
+
+ v = VariantVal.parseJson('{"b": [1, "2", true, null], "a": 1.5}')
+ assert str(v) == v.toJson() == '{"a":1.5,"b":[1,"2",true,null]}'
+ assert repr(v) == "VariantVal(%r, %r)" % (v.value, v.metadata)
+ assert str(VariantVal.parseJson('"x"')) == '"x"'
+ assert "%s" % VariantVal.parseJson("1") == "1"
+
+ spark = _offline_session()
+ struct = T.StructType([T.StructField("v", T.VariantType())])
+ for data, schema in [([v], struct), ([v, None], struct), ([v], None),
([v], T.VariantType())]:
+ with pytest.raises(PySparkValueError, match="Rows cannot be of type
VariantVal"):
+ spark.createDataFrame(data, schema)
+ assert type(spark.createDataFrame([None, v], struct)).__name__ ==
"DataFrame"
+
+
+
+
def test_variant_values():
"""Test VariantVal and other variant types."""
from pyspark.sql.types import VariantVal, Geometry, Geography
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]