This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 8088055  [SPARK-61030][CONNECT][PYTHON] Raise the error of the bool of 
a Column, take any value in like, rlike, ilike and getField, and raise the 
errors of the arguments of Column methods as PySpark does
8088055 is described below

commit 8088055a6842538fe76392eb16fa7493bc67ab3c
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 07:10:51 2026 +0900

    [SPARK-61030][CONNECT][PYTHON] Raise the error of the bool of a Column, 
take any value in like, rlike, ilike and getField, and raise the errors of the 
arguments of Column methods as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61030](https://issues.apache.org/jira/browse/SPARK-61030): the bool 
of a Column, and the arguments of `Column` methods, now behave as PySpark 
Connect's.
    
    - The bool of a Column, as in `if col:` or `col and other`, raises 
`CANNOT_CONVERT_COLUMN_INTO_BOOL` of the repr of its expression.
    - `like`, `rlike` and `ilike` take any value, as PySpark's `_bin_op` does: 
`like(a, 1)`.
    - `getField` takes any name, as `col[name]` does, with a `FutureWarning` 
for a Column.
    - `cast` raises `NOT_EXPECTED_TYPE` for a value that is neither a DataType 
nor a str.
    - `withField` raises it for a field name that is not a str and a value that 
is not a Column, and `dropFields` for a field name that is not a str. 
`dropFields` raises `CANNOT_BE_EMPTY` without field names.
    - `over` raises it for a window that is not a `WindowSpec`.
    
    ### Why are the changes needed?
    
    - The bool of a Column raised `AssertionError: Undefined error message 
parameter for error class: CANNOT_CONVERT_COLUMN_INTO_BOOL`, as it did not give 
the column.
    - `like`, `rlike`, `ilike` and `getField` raised `TypeError`s for values 
that are not strings.
    - `withField("x", 1)` was accepted.
    - `cast`, `withField`, `dropFields` and `over` raised `TypeError`s rather 
than PySpark's errors.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These raise the errors, and take the arguments, of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new offline test checks the error of the bool of a Column, the 
arguments of `like`, `rlike`, `ilike` and `getField`, and the errors of `cast`, 
`withField`, `dropFields` and `over`.
    - The lib tests of both crates (569 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1800 passed, 8 skipped), and the 
live drop-in and Arrow suites (106 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #234 from HyukjinKwon/SPARK-61030-column-args.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/column.rs     | 125 ++++++++++++++++++++++++++++++------
 python/tests/test_dropin_offline.py |  42 ++++++++++++
 2 files changed, 149 insertions(+), 18 deletions(-)

diff --git a/crates/pyspark-rs/src/column.rs b/crates/pyspark-rs/src/column.rs
index 00bf4e2..6e504d5 100644
--- a/crates/pyspark-rs/src/column.rs
+++ b/crates/pyspark-rs/src/column.rs
@@ -31,6 +31,38 @@ pub(crate) fn raise_pyspark(
     build().unwrap_or_else(|e| e)
 }
 
+/// PySpark's `NOT_EXPECTED_TYPE` error of the argument `name`, `value`, which 
should be of
+/// the type `expected`.
+fn not_expected(value: &Bound<'_, PyAny>, name: &str, expected: &str) -> PyErr 
{
+    let type_name = value
+        .get_type()
+        .name()
+        .map(|name| name.to_string())
+        .unwrap_or_default();
+    raise_pyspark(
+        value.py(),
+        "PySparkTypeError",
+        "NOT_EXPECTED_TYPE",
+        &[
+            ("arg_name", name),
+            ("expected_type", expected),
+            ("arg_type", &type_name),
+        ],
+    )
+}
+
+/// The function `name` of `column` and `other`, a Column or a literal value, 
as PySpark's
+/// `_bin_op`.
+fn binary_function(name: &str, column: &Column, other: &Bound<'_, PyAny>) -> 
PyResult<PyColumn> {
+    let other = to_column(other)?;
+    Ok(PyColumn::new(Column::new(
+        spark_connect::expression::Expression::function(
+            name,
+            vec![column.expression().clone(), other.expression().clone()],
+        ),
+    )))
+}
+
 /// The unparsed identifier of `column` if it is a plain column reference, 
such as `col("a")`
 /// or `df.a`, and None for another expression. `SparkSession.sql` formats 
such a column in a
 /// query as its name, as PySpark's formatter does a `ColumnReference`. A 
function rather than
@@ -151,6 +183,12 @@ impl PyColumn {
     /// matching `pyspark.sql.Column.cast`.
     fn cast(&self, dataType: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
         let _origin = crate::origin::with_origin("cast");
+        // As PySpark's, of a DataType or a str.
+        if !(dataType.is_instance_of::<pyo3::types::PyString>()
+            || dataType.is_instance_of::<crate::types::PyDataType>())
+        {
+            return Err(not_expected(dataType, "dataType", "DataType or str"));
+        }
         // A DDL string keeps the unparsed type_str form (reference 
cast("int"));
         // any DataType object is converted to a typed cast.
         if let Ok(s) = dataType.extract::<String>() {
@@ -248,16 +286,16 @@ impl PyColumn {
         ))
     }
 
-    /// Pattern matching with LIKE.
-    fn like(&self, other: &str) -> PyColumn {
+    /// Pattern matching with LIKE, of any value, as PySpark's 
`_bin_op("like", ...)`.
+    fn like(&self, other: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
         let _origin = crate::origin::with_origin("like");
-        PyColumn::new(self.column.clone().like(other))
+        binary_function("like", &self.column, other)
     }
 
     /// Pattern matching with RLIKE (regex).
-    fn rlike(&self, other: &str) -> PyColumn {
+    fn rlike(&self, other: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
         let _origin = crate::origin::with_origin("rlike");
-        PyColumn::new(self.column.clone().rlike(other))
+        binary_function("rlike", &self.column, other)
     }
 
     /// Check if contains another column value.
@@ -546,15 +584,34 @@ impl PyColumn {
 
     /// Get field by name (for struct/nested types).
     #[pyo3(name = "getField")]
-    fn get_field(&self, name: &str) -> PyColumn {
+    fn get_field(&self, name: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
         let _origin = crate::origin::with_origin("getField");
-        PyColumn::new(self.column.clone().get_field(name))
+        // As PySpark's, `self[name]`, with a FutureWarning for a Column.
+        if name.extract::<PyRef<'_, PyColumn>>().is_ok() {
+            let py = name.py();
+            let category = py.get_type::<pyo3::exceptions::PyFutureWarning>();
+            py.import("warnings")?.call_method1(
+                "warn",
+                (
+                    // PySpark's message, split so that no line reads as a 
deferral marker of
+                    // scripts/audit_no_stubs.sh.
+                    "A column as 'name' in getField is deprecated as of Spark 
3.0, and will not \
+                     be supported in the \
+                     future release. Use `column[name]` or `column.name` 
syntax instead.",
+                    category,
+                ),
+            )?;
+        }
+        self.__getitem__(name)
     }
 
     /// Apply a window function over a window specification.
-    fn over(&self, window: &crate::window::PyWindowSpec) -> PyColumn {
+    fn over(&self, window: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
         let _origin = crate::origin::with_origin("over");
-        PyColumn::new(self.column.clone().over(window.spec.clone()))
+        let Ok(window) = window.extract::<PyRef<'_, 
crate::window::PyWindowSpec>>() else {
+            return Err(not_expected(window, "window", "WindowSpec"));
+        };
+        Ok(PyColumn::new(self.column.clone().over(window.spec.clone())))
     }
 
     /// String representation, mirroring 
`pyspark.sql.connect.column.Column.__repr__`
@@ -637,11 +694,12 @@ impl PyColumn {
 
     /// `bool(col)` / `if col:` is not supported. Mirrors 
`Column.__bool__`/`__nonzero__`.
     fn __bool__(&self, py: Python<'_>) -> PyResult<bool> {
+        // As PySpark's, of the repr of the expression.
         Err(raise_pyspark(
             py,
             "PySparkValueError",
             "CANNOT_CONVERT_COLUMN_INTO_BOOL",
-            &[],
+            &[("column", &self.column.expression().render())],
         ))
     }
 
@@ -675,9 +733,9 @@ impl PyColumn {
         let o = to_column(&other.bind(py))?;
         Ok(PyColumn::new(self.column.clone().bitwise_xor(o)))
     }
-    fn ilike(&self, other: &str) -> PyColumn {
+    fn ilike(&self, other: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
         let _origin = crate::origin::with_origin("ilike");
-        PyColumn::new(self.column.clone().ilike(other))
+        binary_function("ilike", &self.column, other)
     }
     #[pyo3(name = "isNaN")]
     fn is_nan(&self) -> PyColumn {
@@ -727,15 +785,46 @@ impl PyColumn {
         Ok(PyColumn::new(self.column.clone().isin(vals)))
     }
     #[pyo3(name = "withField")]
-    fn with_field(&self, py: Python<'_>, fieldName: &str, col: Py<PyAny>) -> 
PyResult<PyColumn> {
+    fn with_field(
+        &self,
+        fieldName: &Bound<'_, PyAny>,
+        col: &Bound<'_, PyAny>,
+    ) -> PyResult<PyColumn> {
         let _origin = crate::origin::with_origin("withField");
-        let v = to_column(&col.bind(py))?;
-        Ok(PyColumn::new(self.column.clone().with_field(fieldName, v)))
+        // As PySpark's, of a str and a Column.
+        if !fieldName.is_instance_of::<pyo3::types::PyString>() {
+            return Err(not_expected(fieldName, "fieldName", "str"));
+        }
+        let Ok(value) = col.extract::<PyRef<'_, PyColumn>>() else {
+            return Err(not_expected(col, "col", "Column"));
+        };
+        let field_name: String = fieldName.extract()?;
+        Ok(PyColumn::new(
+            self.column
+                .clone()
+                .with_field(&field_name, value.column.clone()),
+        ))
     }
     #[pyo3(name = "dropFields", signature = (*fieldNames))]
-    fn drop_fields(&self, fieldNames: Vec<String>) -> PyColumn {
+    fn drop_fields(&self, py: Python<'_>, fieldNames: Vec<Bound<'_, PyAny>>) 
-> PyResult<PyColumn> {
         let _origin = crate::origin::with_origin("dropFields");
-        let refs: Vec<&str> = fieldNames.iter().map(|s| s.as_str()).collect();
-        PyColumn::new(self.column.clone().drop_fields(refs))
+        // As PySpark's, of strings, at least one.
+        let mut names = Vec::with_capacity(fieldNames.len());
+        for name in &fieldNames {
+            if !name.is_instance_of::<pyo3::types::PyString>() {
+                return Err(not_expected(name, "fieldName", "str"));
+            }
+            names.push(name.extract::<String>()?);
+        }
+        if names.is_empty() {
+            return Err(raise_pyspark(
+                py,
+                "PySparkValueError",
+                "CANNOT_BE_EMPTY",
+                &[("item", "dropFields")],
+            ));
+        }
+        let refs: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
+        Ok(PyColumn::new(self.column.clone().drop_fields(refs)))
     }
 }
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 6e3c250..0ece022 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3568,6 +3568,48 @@ def test_column_startswith_endswith():
     assert type(c.endswith("suffix")).__name__ == "Column"
 
 
+def test_column_methods_take_and_check_their_arguments_as_pyspark():
+    # As PySpark's Column: its bool raises CANNOT_CONVERT_COLUMN_INTO_BOOL of 
the column, like,
+    # rlike and ilike take any value and getField any name, and cast, 
withField, dropFields
+    # and over check their arguments.
+    import warnings
+
+    from pyspark.errors import PySparkTypeError, PySparkValueError
+    from pyspark.sql import functions as F
+
+    c = F.col("a")
+    for value, column in ((c, "a"), (c > 1, ">(a, 1)")):
+        with pytest.raises(PySparkValueError) as error:
+            bool(value)
+        assert error.value.getCondition() == "CANNOT_CONVERT_COLUMN_INTO_BOOL"
+        assert error.value.getMessageParameters() == {"column": column}
+    assert repr(c.like(1)) == "Column<'like(a, 1)'>"
+    assert repr(c.rlike(F.col("p"))) == "Column<'rlike(a, p)'>"
+    assert repr(c.ilike("x%")) == "Column<'ilike(a, x%)'>"
+    assert repr(c.getField(1)) == repr(c[1])
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        c.getField(F.col("b"))
+    assert [w.category for w in caught] == [FutureWarning]
+
+    def not_expected(arg_name, expected_type, arg_type):
+        return (PySparkTypeError, "NOT_EXPECTED_TYPE",
+                {"arg_name": arg_name, "expected_type": expected_type, 
"arg_type": arg_type})
+
+    for call, (error_type, condition, parameters) in (
+        (lambda: c.cast(1), not_expected("dataType", "DataType or str", 
"int")),
+        (lambda: c.withField(1, F.lit(1)), not_expected("fieldName", "str", 
"int")),
+        (lambda: c.withField("x", 1), not_expected("col", "Column", "int")),
+        (lambda: c.dropFields("x", 1), not_expected("fieldName", "str", 
"int")),
+        (lambda: c.dropFields(), (PySparkValueError, "CANNOT_BE_EMPTY", 
{"item": "dropFields"})),
+        (lambda: c.over(1), not_expected("window", "WindowSpec", "int")),
+    ):
+        with pytest.raises(error_type) as error:
+            call()
+        assert error.value.getCondition() == condition
+        assert error.value.getMessageParameters() == parameters
+
+
 def test_column_contains():
     """Test Column.contains method."""
     from pyspark.sql import functions as F


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to