This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new e87abf8  [SPARK-61093][CONNECT][PYTHON] Raise the errors of PySpark of 
Column.try_cast, and of Column.isin of a tuple, in the Python client
e87abf8 is described below

commit e87abf839a70a96f86b74722d3a4d62236623edc
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 16:11:45 2026 +0900

    [SPARK-61093][CONNECT][PYTHON] Raise the errors of PySpark of 
Column.try_cast, and of Column.isin of a tuple, in the Python client
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61093](https://issues.apache.org/jira/browse/SPARK-61093): 
`Column.try_cast` and `Column.isin` now take their arguments as PySpark 
Connect's do.
    
    As PySpark Connect's:
    - `try_cast` raises `NOT_EXPECTED_TYPE` (`expected_type` "DataType or str") 
for a type that is neither a `DataType` nor a str, as `cast` already does.
    - `isin` unpacks a single list or set argument, but not a tuple. A tuple is 
a value, so `col.isin((1, 2))` raises the `UNSUPPORTED_DATA_TYPE` of tuple, as 
`lit((1, 2))` does.
    
    ### Why are the changes needed?
    
    The client raised a plain `TypeError` for `try_cast(1)`, and took 
`col.isin((1, 2))` as `col.isin(1, 2)`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `try_cast` and `isin` raise the errors of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks the errors of `try_cast(1)` and of `isin` of a 
tuple, and the rows of `isin` of a list. It passes with official PySpark 4.3.0 
and fails without this change.
    - A new offline test checks these errors, and that `isin` of a list, a set 
and of separate values is the same column.
    - On a local server, 52 cases of Column methods with bad and good arguments 
give the same results as official PySpark 4.3.0. The only exception is the 
CPython message of a `TypeError` for the arguments of `desc`.
    - The lib tests of both crates (594 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2045 passed, 12 skipped), and the 
live drop-in and Arrow suites (148 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #297 from HyukjinKwon/SPARK-61093-try-cast-isin.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/column.rs     | 18 ++++++++++++++----
 python/tests/test_dropin_e2e.py     | 17 +++++++++++++++++
 python/tests/test_dropin_offline.py | 22 ++++++++++++++++++++++
 3 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/crates/pyspark-rs/src/column.rs b/crates/pyspark-rs/src/column.rs
index 6e504d5..6883aaa 100644
--- a/crates/pyspark-rs/src/column.rs
+++ b/crates/pyspark-rs/src/column.rs
@@ -210,6 +210,16 @@ impl PyColumn {
     #[pyo3(name = "try_cast")]
     fn try_cast(&self, dataType: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
         let _origin = crate::origin::with_origin("try_cast");
+        // As PySpark's: a DataType or a str, else NOT_EXPECTED_TYPE.
+        if !dataType.is_instance_of::<crate::types::PyDataType>()
+            && !dataType.is_instance_of::<pyo3::types::PyString>()
+        {
+            return Err(crate::stat_arguments::not_expected_type(
+                dataType,
+                "dataType",
+                "DataType or str",
+            ));
+        }
         if let Ok(s) = dataType.extract::<String>() {
             Ok(PyColumn::new(self.column.clone().try_cast_str(&s)))
         } else {
@@ -762,14 +772,14 @@ impl PyColumn {
                 ));
             }
         }
-        // pyspark's Column.isin unpacks a single list/tuple/set argument:
-        // col.isin([1, 2]) == col.isin(1, 2). pandas-on-Spark's Series.isin 
relies on
-        // this (it passes a single list of lit(..) columns).
+        // pyspark's Column.isin unpacks a single list or set argument, but 
not a tuple, which
+        // is a value (UNSUPPORTED_DATA_TYPE of lit): col.isin([1, 2]) == 
col.isin(1, 2).
+        // pandas-on-Spark's Series.isin relies on this (it passes a single 
list of lit(..)
+        // columns).
         let mut items: Vec<Py<PyAny>> = cols;
         if items.len() == 1 {
             let only = items[0].bind(py);
             if only.is_instance_of::<pyo3::types::PyList>()
-                || only.is_instance_of::<pyo3::types::PyTuple>()
                 || only.is_instance_of::<pyo3::types::PySet>()
             {
                 items = only
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 868dfc4..fac8fc9 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -2255,6 +2255,23 @@ def 
test_observations_receive_the_observed_metrics_as_pyspark(session):
     assert error.value.getCondition() == "UNSUPPORTED_OPERATION"
 
 
+def test_try_cast_and_isin_raise_the_errors_of_pyspark(session):
+    # As PySpark Connect's: try_cast of an int, and isin of a tuple, raise 
PySpark's errors.
+    from pyspark.errors import PySparkTypeError
+    from pyspark.sql import functions as F
+
+    df = session.range(3)
+    with pytest.raises(PySparkTypeError) as e:
+        df.select(F.col("id").try_cast(1))
+    assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+    with pytest.raises(PySparkTypeError) as e:
+        df.where(F.col("id").isin((1, 2)))
+    assert e.value.getCondition() == "UNSUPPORTED_DATA_TYPE"
+    assert df.where(F.col("id").isin([1, 2])).count() == 2
+
+
+
+
 def test_ml_vector_functions_as_pyspark(session):
     from pyspark.ml.functions import array_to_vector, vector_to_array
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 378d796..98ec6aa 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1258,6 +1258,28 @@ def 
test_map_functions_send_the_names_of_their_functions_as_pyspark(mock_connect
     assert sent_name(stateful, 34, 3) == "with_state"
 
 
+def test_try_cast_and_isin_take_their_arguments_as_pyspark():
+    # As PySpark Connect's: try_cast raises NOT_EXPECTED_TYPE for a type that 
is neither a
+    # DataType nor a str, and isin unpacks a single list or set, but not a 
tuple, which is a
+    # value of lit (UNSUPPORTED_DATA_TYPE).
+    from pyspark.errors import PySparkTypeError
+    from pyspark.sql import functions as F
+
+    column = F.col("a")
+    with pytest.raises(PySparkTypeError) as e:
+        column.try_cast(1)
+    assert e.value.getMessageParameters() == {
+        "expected_type": "DataType or str", "arg_name": "dataType", 
"arg_type": "int"}
+    assert "TRY_CAST" in repr(column.try_cast("string")).upper()
+    column.try_cast(T.StringType())
+    assert repr(column.isin([1, 2])) == repr(column.isin({1, 2})) == 
repr(column.isin(1, 2))
+    with pytest.raises(PySparkTypeError) as e:
+        column.isin((1, 2))
+    assert e.value.getMessageParameters() == {"data_type": "tuple"}
+
+
+
+
 def test_map_functions_send_their_resource_profile_as_pyspark(mock_connect):
     # As PySpark's, mapInPandas and mapInArrow send the id of their resource 
profile, which
     # `profile.id` creates on the server, so that their stage runs with its 
resources.


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to