This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new e87abf8 [SPARK-61093][CONNECT][PYTHON] Raise the errors of PySpark of
Column.try_cast, and of Column.isin of a tuple, in the Python client
e87abf8 is described below
commit e87abf839a70a96f86b74722d3a4d62236623edc
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 16:11:45 2026 +0900
[SPARK-61093][CONNECT][PYTHON] Raise the errors of PySpark of
Column.try_cast, and of Column.isin of a tuple, in the Python client
### What changes were proposed in this pull request?
[SPARK-61093](https://issues.apache.org/jira/browse/SPARK-61093):
`Column.try_cast` and `Column.isin` now take their arguments as PySpark
Connect's do.
As PySpark Connect's:
- `try_cast` raises `NOT_EXPECTED_TYPE` (`expected_type` "DataType or str")
for a type that is neither a `DataType` nor a str, as `cast` already does.
- `isin` unpacks a single list or set argument, but not a tuple. A tuple is
a value, so `col.isin((1, 2))` raises the `UNSUPPORTED_DATA_TYPE` of tuple, as
`lit((1, 2))` does.
### Why are the changes needed?
The client raised a plain `TypeError` for `try_cast(1)`, and took
`col.isin((1, 2))` as `col.isin(1, 2)`.
### Does this PR introduce _any_ user-facing change?
Yes. `try_cast` and `isin` raise the errors of PySpark 4.3.0.
### How was this patch tested?
- A new live test checks the errors of `try_cast(1)` and of `isin` of a
tuple, and the rows of `isin` of a list. It passes with official PySpark 4.3.0
and fails without this change.
- A new offline test checks these errors, and that `isin` of a list, a set
and of separate values is the same column.
- On a local server, 52 cases of Column methods with bad and good arguments
give the same results as official PySpark 4.3.0. The only exception is the
CPython message of a `TypeError` for the arguments of `desc`.
- The lib tests of both crates (594 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2045 passed, 12 skipped), and the
live drop-in and Arrow suites (148 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #297 from HyukjinKwon/SPARK-61093-try-cast-isin.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/column.rs | 18 ++++++++++++++----
python/tests/test_dropin_e2e.py | 17 +++++++++++++++++
python/tests/test_dropin_offline.py | 22 ++++++++++++++++++++++
3 files changed, 53 insertions(+), 4 deletions(-)
diff --git a/crates/pyspark-rs/src/column.rs b/crates/pyspark-rs/src/column.rs
index 6e504d5..6883aaa 100644
--- a/crates/pyspark-rs/src/column.rs
+++ b/crates/pyspark-rs/src/column.rs
@@ -210,6 +210,16 @@ impl PyColumn {
#[pyo3(name = "try_cast")]
fn try_cast(&self, dataType: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
let _origin = crate::origin::with_origin("try_cast");
+ // As PySpark's: a DataType or a str, else NOT_EXPECTED_TYPE.
+ if !dataType.is_instance_of::<crate::types::PyDataType>()
+ && !dataType.is_instance_of::<pyo3::types::PyString>()
+ {
+ return Err(crate::stat_arguments::not_expected_type(
+ dataType,
+ "dataType",
+ "DataType or str",
+ ));
+ }
if let Ok(s) = dataType.extract::<String>() {
Ok(PyColumn::new(self.column.clone().try_cast_str(&s)))
} else {
@@ -762,14 +772,14 @@ impl PyColumn {
));
}
}
- // pyspark's Column.isin unpacks a single list/tuple/set argument:
- // col.isin([1, 2]) == col.isin(1, 2). pandas-on-Spark's Series.isin
relies on
- // this (it passes a single list of lit(..) columns).
+ // pyspark's Column.isin unpacks a single list or set argument, but
not a tuple, which
+ // is a value (UNSUPPORTED_DATA_TYPE of lit): col.isin([1, 2]) ==
col.isin(1, 2).
+ // pandas-on-Spark's Series.isin relies on this (it passes a single
list of lit(..)
+ // columns).
let mut items: Vec<Py<PyAny>> = cols;
if items.len() == 1 {
let only = items[0].bind(py);
if only.is_instance_of::<pyo3::types::PyList>()
- || only.is_instance_of::<pyo3::types::PyTuple>()
|| only.is_instance_of::<pyo3::types::PySet>()
{
items = only
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 868dfc4..fac8fc9 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -2255,6 +2255,23 @@ def
test_observations_receive_the_observed_metrics_as_pyspark(session):
assert error.value.getCondition() == "UNSUPPORTED_OPERATION"
+def test_try_cast_and_isin_raise_the_errors_of_pyspark(session):
+ # As PySpark Connect's: try_cast of an int, and isin of a tuple, raise
PySpark's errors.
+ from pyspark.errors import PySparkTypeError
+ from pyspark.sql import functions as F
+
+ df = session.range(3)
+ with pytest.raises(PySparkTypeError) as e:
+ df.select(F.col("id").try_cast(1))
+ assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+ with pytest.raises(PySparkTypeError) as e:
+ df.where(F.col("id").isin((1, 2)))
+ assert e.value.getCondition() == "UNSUPPORTED_DATA_TYPE"
+ assert df.where(F.col("id").isin([1, 2])).count() == 2
+
+
+
+
def test_ml_vector_functions_as_pyspark(session):
from pyspark.ml.functions import array_to_vector, vector_to_array
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 378d796..98ec6aa 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1258,6 +1258,28 @@ def
test_map_functions_send_the_names_of_their_functions_as_pyspark(mock_connect
assert sent_name(stateful, 34, 3) == "with_state"
+def test_try_cast_and_isin_take_their_arguments_as_pyspark():
+ # As PySpark Connect's: try_cast raises NOT_EXPECTED_TYPE for a type that
is neither a
+ # DataType nor a str, and isin unpacks a single list or set, but not a
tuple, which is a
+ # value of lit (UNSUPPORTED_DATA_TYPE).
+ from pyspark.errors import PySparkTypeError
+ from pyspark.sql import functions as F
+
+ column = F.col("a")
+ with pytest.raises(PySparkTypeError) as e:
+ column.try_cast(1)
+ assert e.value.getMessageParameters() == {
+ "expected_type": "DataType or str", "arg_name": "dataType",
"arg_type": "int"}
+ assert "TRY_CAST" in repr(column.try_cast("string")).upper()
+ column.try_cast(T.StringType())
+ assert repr(column.isin([1, 2])) == repr(column.isin({1, 2})) ==
repr(column.isin(1, 2))
+ with pytest.raises(PySparkTypeError) as e:
+ column.isin((1, 2))
+ assert e.value.getMessageParameters() == {"data_type": "tuple"}
+
+
+
+
def test_map_functions_send_their_resource_profile_as_pyspark(mock_connect):
# As PySpark's, mapInPandas and mapInArrow send the id of their resource
profile, which
# `profile.id` creates on the server, so that their stage runs with its
resources.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]