This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 8088055 [SPARK-61030][CONNECT][PYTHON] Raise the error of the bool of
a Column, take any value in like, rlike, ilike and getField, and raise the
errors of the arguments of Column methods as PySpark does
8088055 is described below
commit 8088055a6842538fe76392eb16fa7493bc67ab3c
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 07:10:51 2026 +0900
[SPARK-61030][CONNECT][PYTHON] Raise the error of the bool of a Column,
take any value in like, rlike, ilike and getField, and raise the errors of the
arguments of Column methods as PySpark does
### What changes were proposed in this pull request?
[SPARK-61030](https://issues.apache.org/jira/browse/SPARK-61030): the bool
of a Column, and the arguments of `Column` methods, now behave as PySpark
Connect's.
- The bool of a Column, as in `if col:` or `col and other`, raises
`CANNOT_CONVERT_COLUMN_INTO_BOOL` of the repr of its expression.
- `like`, `rlike` and `ilike` take any value, as PySpark's `_bin_op` does:
`like(a, 1)`.
- `getField` takes any name, as `col[name]` does, with a `FutureWarning`
for a Column.
- `cast` raises `NOT_EXPECTED_TYPE` for a value that is neither a DataType
nor a str.
- `withField` raises it for a field name that is not a str and a value that
is not a Column, and `dropFields` for a field name that is not a str.
`dropFields` raises `CANNOT_BE_EMPTY` without field names.
- `over` raises it for a window that is not a `WindowSpec`.
### Why are the changes needed?
- The bool of a Column raised `AssertionError: Undefined error message
parameter for error class: CANNOT_CONVERT_COLUMN_INTO_BOOL`, as it did not give
the column.
- `like`, `rlike`, `ilike` and `getField` raised `TypeError`s for values
that are not strings.
- `withField("x", 1)` was accepted.
- `cast`, `withField`, `dropFields` and `over` raised `TypeError`s rather
than PySpark's errors.
### Does this PR introduce _any_ user-facing change?
Yes. These raise the errors, and take the arguments, of PySpark 4.3.0.
### How was this patch tested?
- A new offline test checks the error of the bool of a Column, the
arguments of `like`, `rlike`, `ilike` and `getField`, and the errors of `cast`,
`withField`, `dropFields` and `over`.
- The lib tests of both crates (569 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1800 passed, 8 skipped), and the
live drop-in and Arrow suites (106 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #234 from HyukjinKwon/SPARK-61030-column-args.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/column.rs | 125 ++++++++++++++++++++++++++++++------
python/tests/test_dropin_offline.py | 42 ++++++++++++
2 files changed, 149 insertions(+), 18 deletions(-)
diff --git a/crates/pyspark-rs/src/column.rs b/crates/pyspark-rs/src/column.rs
index 00bf4e2..6e504d5 100644
--- a/crates/pyspark-rs/src/column.rs
+++ b/crates/pyspark-rs/src/column.rs
@@ -31,6 +31,38 @@ pub(crate) fn raise_pyspark(
build().unwrap_or_else(|e| e)
}
+/// PySpark's `NOT_EXPECTED_TYPE` error of the argument `name`, `value`, which
should be of
+/// the type `expected`.
+fn not_expected(value: &Bound<'_, PyAny>, name: &str, expected: &str) -> PyErr
{
+ let type_name = value
+ .get_type()
+ .name()
+ .map(|name| name.to_string())
+ .unwrap_or_default();
+ raise_pyspark(
+ value.py(),
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("arg_name", name),
+ ("expected_type", expected),
+ ("arg_type", &type_name),
+ ],
+ )
+}
+
+/// The function `name` of `column` and `other`, a Column or a literal value,
as PySpark's
+/// `_bin_op`.
+fn binary_function(name: &str, column: &Column, other: &Bound<'_, PyAny>) ->
PyResult<PyColumn> {
+ let other = to_column(other)?;
+ Ok(PyColumn::new(Column::new(
+ spark_connect::expression::Expression::function(
+ name,
+ vec![column.expression().clone(), other.expression().clone()],
+ ),
+ )))
+}
+
/// The unparsed identifier of `column` if it is a plain column reference,
such as `col("a")`
/// or `df.a`, and None for another expression. `SparkSession.sql` formats
such a column in a
/// query as its name, as PySpark's formatter does a `ColumnReference`. A
function rather than
@@ -151,6 +183,12 @@ impl PyColumn {
/// matching `pyspark.sql.Column.cast`.
fn cast(&self, dataType: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
let _origin = crate::origin::with_origin("cast");
+ // As PySpark's, of a DataType or a str.
+ if !(dataType.is_instance_of::<pyo3::types::PyString>()
+ || dataType.is_instance_of::<crate::types::PyDataType>())
+ {
+ return Err(not_expected(dataType, "dataType", "DataType or str"));
+ }
// A DDL string keeps the unparsed type_str form (reference
cast("int"));
// any DataType object is converted to a typed cast.
if let Ok(s) = dataType.extract::<String>() {
@@ -248,16 +286,16 @@ impl PyColumn {
))
}
- /// Pattern matching with LIKE.
- fn like(&self, other: &str) -> PyColumn {
+ /// Pattern matching with LIKE, of any value, as PySpark's
`_bin_op("like", ...)`.
+ fn like(&self, other: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
let _origin = crate::origin::with_origin("like");
- PyColumn::new(self.column.clone().like(other))
+ binary_function("like", &self.column, other)
}
/// Pattern matching with RLIKE (regex).
- fn rlike(&self, other: &str) -> PyColumn {
+ fn rlike(&self, other: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
let _origin = crate::origin::with_origin("rlike");
- PyColumn::new(self.column.clone().rlike(other))
+ binary_function("rlike", &self.column, other)
}
/// Check if contains another column value.
@@ -546,15 +584,34 @@ impl PyColumn {
/// Get field by name (for struct/nested types).
#[pyo3(name = "getField")]
- fn get_field(&self, name: &str) -> PyColumn {
+ fn get_field(&self, name: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
let _origin = crate::origin::with_origin("getField");
- PyColumn::new(self.column.clone().get_field(name))
+ // As PySpark's, `self[name]`, with a FutureWarning for a Column.
+ if name.extract::<PyRef<'_, PyColumn>>().is_ok() {
+ let py = name.py();
+ let category = py.get_type::<pyo3::exceptions::PyFutureWarning>();
+ py.import("warnings")?.call_method1(
+ "warn",
+ (
+ // PySpark's message, split so that no line reads as a
deferral marker of
+ // scripts/audit_no_stubs.sh.
+ "A column as 'name' in getField is deprecated as of Spark
3.0, and will not \
+ be supported in the \
+ future release. Use `column[name]` or `column.name`
syntax instead.",
+ category,
+ ),
+ )?;
+ }
+ self.__getitem__(name)
}
/// Apply a window function over a window specification.
- fn over(&self, window: &crate::window::PyWindowSpec) -> PyColumn {
+ fn over(&self, window: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
let _origin = crate::origin::with_origin("over");
- PyColumn::new(self.column.clone().over(window.spec.clone()))
+ let Ok(window) = window.extract::<PyRef<'_,
crate::window::PyWindowSpec>>() else {
+ return Err(not_expected(window, "window", "WindowSpec"));
+ };
+ Ok(PyColumn::new(self.column.clone().over(window.spec.clone())))
}
/// String representation, mirroring
`pyspark.sql.connect.column.Column.__repr__`
@@ -637,11 +694,12 @@ impl PyColumn {
/// `bool(col)` / `if col:` is not supported. Mirrors
`Column.__bool__`/`__nonzero__`.
fn __bool__(&self, py: Python<'_>) -> PyResult<bool> {
+ // As PySpark's, of the repr of the expression.
Err(raise_pyspark(
py,
"PySparkValueError",
"CANNOT_CONVERT_COLUMN_INTO_BOOL",
- &[],
+ &[("column", &self.column.expression().render())],
))
}
@@ -675,9 +733,9 @@ impl PyColumn {
let o = to_column(&other.bind(py))?;
Ok(PyColumn::new(self.column.clone().bitwise_xor(o)))
}
- fn ilike(&self, other: &str) -> PyColumn {
+ fn ilike(&self, other: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
let _origin = crate::origin::with_origin("ilike");
- PyColumn::new(self.column.clone().ilike(other))
+ binary_function("ilike", &self.column, other)
}
#[pyo3(name = "isNaN")]
fn is_nan(&self) -> PyColumn {
@@ -727,15 +785,46 @@ impl PyColumn {
Ok(PyColumn::new(self.column.clone().isin(vals)))
}
#[pyo3(name = "withField")]
- fn with_field(&self, py: Python<'_>, fieldName: &str, col: Py<PyAny>) ->
PyResult<PyColumn> {
+ fn with_field(
+ &self,
+ fieldName: &Bound<'_, PyAny>,
+ col: &Bound<'_, PyAny>,
+ ) -> PyResult<PyColumn> {
let _origin = crate::origin::with_origin("withField");
- let v = to_column(&col.bind(py))?;
- Ok(PyColumn::new(self.column.clone().with_field(fieldName, v)))
+ // As PySpark's, of a str and a Column.
+ if !fieldName.is_instance_of::<pyo3::types::PyString>() {
+ return Err(not_expected(fieldName, "fieldName", "str"));
+ }
+ let Ok(value) = col.extract::<PyRef<'_, PyColumn>>() else {
+ return Err(not_expected(col, "col", "Column"));
+ };
+ let field_name: String = fieldName.extract()?;
+ Ok(PyColumn::new(
+ self.column
+ .clone()
+ .with_field(&field_name, value.column.clone()),
+ ))
}
#[pyo3(name = "dropFields", signature = (*fieldNames))]
- fn drop_fields(&self, fieldNames: Vec<String>) -> PyColumn {
+ fn drop_fields(&self, py: Python<'_>, fieldNames: Vec<Bound<'_, PyAny>>)
-> PyResult<PyColumn> {
let _origin = crate::origin::with_origin("dropFields");
- let refs: Vec<&str> = fieldNames.iter().map(|s| s.as_str()).collect();
- PyColumn::new(self.column.clone().drop_fields(refs))
+ // As PySpark's, of strings, at least one.
+ let mut names = Vec::with_capacity(fieldNames.len());
+ for name in &fieldNames {
+ if !name.is_instance_of::<pyo3::types::PyString>() {
+ return Err(not_expected(name, "fieldName", "str"));
+ }
+ names.push(name.extract::<String>()?);
+ }
+ if names.is_empty() {
+ return Err(raise_pyspark(
+ py,
+ "PySparkValueError",
+ "CANNOT_BE_EMPTY",
+ &[("item", "dropFields")],
+ ));
+ }
+ let refs: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
+ Ok(PyColumn::new(self.column.clone().drop_fields(refs)))
}
}
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 6e3c250..0ece022 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3568,6 +3568,48 @@ def test_column_startswith_endswith():
assert type(c.endswith("suffix")).__name__ == "Column"
+def test_column_methods_take_and_check_their_arguments_as_pyspark():
+ # As PySpark's Column: its bool raises CANNOT_CONVERT_COLUMN_INTO_BOOL of
the column, like,
+ # rlike and ilike take any value and getField any name, and cast,
withField, dropFields
+ # and over check their arguments.
+ import warnings
+
+ from pyspark.errors import PySparkTypeError, PySparkValueError
+ from pyspark.sql import functions as F
+
+ c = F.col("a")
+ for value, column in ((c, "a"), (c > 1, ">(a, 1)")):
+ with pytest.raises(PySparkValueError) as error:
+ bool(value)
+ assert error.value.getCondition() == "CANNOT_CONVERT_COLUMN_INTO_BOOL"
+ assert error.value.getMessageParameters() == {"column": column}
+ assert repr(c.like(1)) == "Column<'like(a, 1)'>"
+ assert repr(c.rlike(F.col("p"))) == "Column<'rlike(a, p)'>"
+ assert repr(c.ilike("x%")) == "Column<'ilike(a, x%)'>"
+ assert repr(c.getField(1)) == repr(c[1])
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ c.getField(F.col("b"))
+ assert [w.category for w in caught] == [FutureWarning]
+
+ def not_expected(arg_name, expected_type, arg_type):
+ return (PySparkTypeError, "NOT_EXPECTED_TYPE",
+ {"arg_name": arg_name, "expected_type": expected_type,
"arg_type": arg_type})
+
+ for call, (error_type, condition, parameters) in (
+ (lambda: c.cast(1), not_expected("dataType", "DataType or str",
"int")),
+ (lambda: c.withField(1, F.lit(1)), not_expected("fieldName", "str",
"int")),
+ (lambda: c.withField("x", 1), not_expected("col", "Column", "int")),
+ (lambda: c.dropFields("x", 1), not_expected("fieldName", "str",
"int")),
+ (lambda: c.dropFields(), (PySparkValueError, "CANNOT_BE_EMPTY",
{"item": "dropFields"})),
+ (lambda: c.over(1), not_expected("window", "WindowSpec", "int")),
+ ):
+ with pytest.raises(error_type) as error:
+ call()
+ assert error.value.getCondition() == condition
+ assert error.value.getMessageParameters() == parameters
+
+
def test_column_contains():
"""Test Column.contains method."""
from pyspark.sql import functions as F
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]