This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 73cde0c  [SPARK-61045][CONNECT][PYTHON] Take the arguments of 
colRegex, metadataColumn, where and transpose in the Python client as PySpark 
does
73cde0c is described below

commit 73cde0c137c2973bb6fdda38d5ee16c85fa6d4f7
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:27:04 2026 +0900

    [SPARK-61045][CONNECT][PYTHON] Take the arguments of colRegex, 
metadataColumn, where and transpose in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61045](https://issues.apache.org/jira/browse/SPARK-61045): 
`colRegex`, `metadataColumn`, `where` and `transpose` now take their arguments 
as PySpark Connect's do.
    
    - `df.colRegex(colName)` and `df.metadataColumn(colName)` raise 
`NOT_EXPECTED_TYPE` of `colName` for one that is not a str.
    - `df.where(condition)` raises `NOT_EXPECTED_TYPE` of `condition` for one 
that is neither a Column nor a str.
    - `df.transpose(indexColumn)` takes a Column or a column name, else 
`NOT_EXPECTED_TYPE` of `col`.
    
    ### Why are the changes needed?
    
    The client raised `TypeError`s for these. It also rejected a column name in 
`transpose`, as `df.transpose("id")`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These methods take the arguments of PySpark 4.3.0 and raise its errors.
    
    ### How was this patch tested?
    
    - A new live test checks the rows of `transpose` of a column name. It 
passes with official PySpark 4.3.0, and fails without this change.
    - A new offline test checks the errors of these methods for arguments of 
other types, and the `transpose` of a column name and of a Column.
    - The lib tests of both crates (571 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1811 passed, 8 skipped), and the 
live drop-in and Arrow suites (109 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #246 from HyukjinKwon/SPARK-61045-df-str-args.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  | 45 +++++++++++++++++++++++++++++++------
 python/tests/test_dropin_e2e.py     |  8 +++++++
 python/tests/test_dropin_offline.py | 30 +++++++++++++++++++++++++
 3 files changed, 76 insertions(+), 7 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 6c7061b..cfffa05 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -475,6 +475,12 @@ impl PyDataFrame {
     /// Alias for filter (reference pyspark name is `where`).
     #[pyo3(name = "where")]
     fn where_(&self, condition: &Bound<'_, PyAny>) -> PyResult<PyDataFrame> {
+        // As PySpark's: a Column or a str, else NOT_EXPECTED_TYPE.
+        if !(condition.is_instance_of::<PyColumn>()
+            || condition.is_instance_of::<pyo3::types::PyString>())
+        {
+            return Err(not_expected_type(condition, "condition", "Column or 
str"));
+        }
         Ok(PyDataFrame::new(
             self.dataframe.filter(cond_column(condition)?),
         ))
@@ -738,9 +744,21 @@ impl PyDataFrame {
     /// column. Mirrors `DataFrame.transpose(indexColumn=None)`.
     #[pyo3(signature = (indexColumn=None))]
     #[allow(non_snake_case)]
-    fn transpose(&self, indexColumn: Option<PyRef<PyColumn>>) -> 
PyResult<PyDataFrame> {
-        let df = match indexColumn {
-            Some(c) => self.dataframe.transpose_with_index(c.column.clone()),
+    fn transpose(&self, indexColumn: Option<&Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        // As PySpark's `F._to_col`: a Column or a column name, else 
NOT_EXPECTED_TYPE of `col`.
+        let index = match indexColumn {
+            None => None,
+            Some(c) => Some(if let Ok(column) = c.extract::<PyRef<'_, 
PyColumn>>() {
+                column.column.clone()
+            } else if c.is_instance_of::<pyo3::types::PyString>() {
+                let _origin = crate::origin::with_origin("col");
+                spark_connect::functions::col(&c.extract::<String>()?)
+            } else {
+                return Err(not_expected_type(c, "col", "Column or str"));
+            }),
+        };
+        let df = match index {
+            Some(column) => self.dataframe.transpose_with_index(column),
             None => self.dataframe.transpose(),
         }
         .to_pyerr()?;
@@ -1084,8 +1102,14 @@ impl PyDataFrame {
     /// The columns of this DataFrame whose names match a regex, as a Column. 
Mirrors
     /// `DataFrame.colRegex`.
     #[pyo3(name = "colRegex")]
-    fn col_regex(&self, colName: &str) -> PyColumn {
-        PyColumn::new(self.dataframe.col_regex(colName))
+    fn col_regex(&self, colName: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
+        // As PySpark's: a str, else NOT_EXPECTED_TYPE.
+        if !colName.is_instance_of::<pyo3::types::PyString>() {
+            return Err(not_expected_type(colName, "colName", "str"));
+        }
+        Ok(PyColumn::new(
+            self.dataframe.col_regex(&colName.extract::<String>()?),
+        ))
     }
 
     /// Basic statistics for the given columns (count/mean/stddev/min/max), of 
column names or
@@ -2223,8 +2247,15 @@ impl PyDataFrame {
 
     /// `DataFrame.metadataColumn`.
     #[pyo3(name = "metadataColumn")]
-    fn metadata_column(&self, colName: &str) -> PyColumn {
-        PyColumn::new(self.dataframe.metadata_column(colName))
+    fn metadata_column(&self, colName: &Bound<'_, PyAny>) -> 
PyResult<PyColumn> {
+        // As PySpark's: a str, else NOT_EXPECTED_TYPE.
+        if !colName.is_instance_of::<pyo3::types::PyString>() {
+            return Err(not_expected_type(colName, "colName", "str"));
+        }
+        Ok(PyColumn::new(
+            self.dataframe
+                .metadata_column(&colName.extract::<String>()?),
+        ))
     }
 }
 
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 85c6dd7..deb3ef3 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1184,6 +1184,14 @@ def 
test_collected_rows_keep_their_values_as_pyspark(session):
     assert (rows[0].v.toJson(), rows[0].n) == ("1", 2)
 
 
+def test_transpose_takes_a_column_name_as_pyspark(session):
+    # As PySpark's transpose, of a Column or a column name as the index column.
+    df = session.createDataFrame([("a", 1, 2), ("b", 3, 4)], "k string, x int, 
y int")
+    by_name = df.transpose("k")
+    assert by_name.columns == ["key", "a", "b"]
+    assert by_name.collect() == df.transpose(F.col("k")).collect()
+
+
 def test_getitem_indexes_filters_and_selects_as_pyspark(session):
     from pyspark.errors import PySparkTypeError
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 73cf3e8..7e531bd 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -7601,6 +7601,36 @@ def 
test_explain_checks_its_arguments_and_prints_with_print_as_pyspark(mock_conn
     assert modes == [4, 2]
 
 
+def 
test_col_regex_metadata_column_where_and_transpose_take_their_arguments_as_pyspark():
+    # As PySpark's: colRegex and metadataColumn take a str, where a Column or 
a str, and
+    # transpose a Column or a column name, else NOT_EXPECTED_TYPE.
+    from pyspark.errors import PySparkTypeError
+    from pyspark.sql import functions as F
+
+    df = _offline_session().range(1)
+    for value in (1, None, ["id"], F.col("id")):
+        for call, arg_name, expected in (
+            (lambda: df.colRegex(value), "colName", "str"),
+            (lambda: df.metadataColumn(value), "colName", "str"),
+        ):
+            with pytest.raises(PySparkTypeError) as error:
+                call()
+            assert error.value.getMessageParameters() == {
+                "arg_name": arg_name, "expected_type": expected,
+                "arg_type": type(value).__name__}
+    for value in (1, 1.5, ["id"]):
+        for call, arg_name in ((lambda: df.where(value), "condition"),
+                               (lambda: df.transpose(value), "col")):
+            with pytest.raises(PySparkTypeError) as error:
+                call()
+            assert error.value.getMessageParameters() == {
+                "arg_name": arg_name, "expected_type": "Column or str",
+                "arg_type": type(value).__name__}
+    # transpose takes a column name as the column of it.
+    assert type(df.transpose("id")).__name__ == "DataFrame"
+    assert type(df.transpose(F.col("id"))).__name__ == "DataFrame"
+
+
 def test_session_context_manager_stops_session():
     spark = _offline_session()
     with spark as s:


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to