This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 73cde0c [SPARK-61045][CONNECT][PYTHON] Take the arguments of
colRegex, metadataColumn, where and transpose in the Python client as PySpark
does
73cde0c is described below
commit 73cde0c137c2973bb6fdda38d5ee16c85fa6d4f7
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:27:04 2026 +0900
[SPARK-61045][CONNECT][PYTHON] Take the arguments of colRegex,
metadataColumn, where and transpose in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61045](https://issues.apache.org/jira/browse/SPARK-61045):
`colRegex`, `metadataColumn`, `where` and `transpose` now take their arguments
as PySpark Connect's do.
- `df.colRegex(colName)` and `df.metadataColumn(colName)` raise
`NOT_EXPECTED_TYPE` of `colName` for one that is not a str.
- `df.where(condition)` raises `NOT_EXPECTED_TYPE` of `condition` for one
that is neither a Column nor a str.
- `df.transpose(indexColumn)` takes a Column or a column name, else
`NOT_EXPECTED_TYPE` of `col`.
### Why are the changes needed?
The client raised `TypeError`s for these. It also rejected a column name in
`transpose`, as `df.transpose("id")`.
### Does this PR introduce _any_ user-facing change?
Yes. These methods take the arguments of PySpark 4.3.0 and raise its errors.
### How was this patch tested?
- A new live test checks the rows of `transpose` of a column name. It
passes with official PySpark 4.3.0, and fails without this change.
- A new offline test checks the errors of these methods for arguments of
other types, and the `transpose` of a column name and of a Column.
- The lib tests of both crates (571 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1811 passed, 8 skipped), and the
live drop-in and Arrow suites (109 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #246 from HyukjinKwon/SPARK-61045-df-str-args.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 45 +++++++++++++++++++++++++++++++------
python/tests/test_dropin_e2e.py | 8 +++++++
python/tests/test_dropin_offline.py | 30 +++++++++++++++++++++++++
3 files changed, 76 insertions(+), 7 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 6c7061b..cfffa05 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -475,6 +475,12 @@ impl PyDataFrame {
/// Alias for filter (reference pyspark name is `where`).
#[pyo3(name = "where")]
fn where_(&self, condition: &Bound<'_, PyAny>) -> PyResult<PyDataFrame> {
+ // As PySpark's: a Column or a str, else NOT_EXPECTED_TYPE.
+ if !(condition.is_instance_of::<PyColumn>()
+ || condition.is_instance_of::<pyo3::types::PyString>())
+ {
+ return Err(not_expected_type(condition, "condition", "Column or
str"));
+ }
Ok(PyDataFrame::new(
self.dataframe.filter(cond_column(condition)?),
))
@@ -738,9 +744,21 @@ impl PyDataFrame {
/// column. Mirrors `DataFrame.transpose(indexColumn=None)`.
#[pyo3(signature = (indexColumn=None))]
#[allow(non_snake_case)]
- fn transpose(&self, indexColumn: Option<PyRef<PyColumn>>) ->
PyResult<PyDataFrame> {
- let df = match indexColumn {
- Some(c) => self.dataframe.transpose_with_index(c.column.clone()),
+ fn transpose(&self, indexColumn: Option<&Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
+ // As PySpark's `F._to_col`: a Column or a column name, else
NOT_EXPECTED_TYPE of `col`.
+ let index = match indexColumn {
+ None => None,
+ Some(c) => Some(if let Ok(column) = c.extract::<PyRef<'_,
PyColumn>>() {
+ column.column.clone()
+ } else if c.is_instance_of::<pyo3::types::PyString>() {
+ let _origin = crate::origin::with_origin("col");
+ spark_connect::functions::col(&c.extract::<String>()?)
+ } else {
+ return Err(not_expected_type(c, "col", "Column or str"));
+ }),
+ };
+ let df = match index {
+ Some(column) => self.dataframe.transpose_with_index(column),
None => self.dataframe.transpose(),
}
.to_pyerr()?;
@@ -1084,8 +1102,14 @@ impl PyDataFrame {
/// The columns of this DataFrame whose names match a regex, as a Column.
Mirrors
/// `DataFrame.colRegex`.
#[pyo3(name = "colRegex")]
- fn col_regex(&self, colName: &str) -> PyColumn {
- PyColumn::new(self.dataframe.col_regex(colName))
+ fn col_regex(&self, colName: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
+ // As PySpark's: a str, else NOT_EXPECTED_TYPE.
+ if !colName.is_instance_of::<pyo3::types::PyString>() {
+ return Err(not_expected_type(colName, "colName", "str"));
+ }
+ Ok(PyColumn::new(
+ self.dataframe.col_regex(&colName.extract::<String>()?),
+ ))
}
/// Basic statistics for the given columns (count/mean/stddev/min/max), of
column names or
@@ -2223,8 +2247,15 @@ impl PyDataFrame {
/// `DataFrame.metadataColumn`.
#[pyo3(name = "metadataColumn")]
- fn metadata_column(&self, colName: &str) -> PyColumn {
- PyColumn::new(self.dataframe.metadata_column(colName))
+ fn metadata_column(&self, colName: &Bound<'_, PyAny>) ->
PyResult<PyColumn> {
+ // As PySpark's: a str, else NOT_EXPECTED_TYPE.
+ if !colName.is_instance_of::<pyo3::types::PyString>() {
+ return Err(not_expected_type(colName, "colName", "str"));
+ }
+ Ok(PyColumn::new(
+ self.dataframe
+ .metadata_column(&colName.extract::<String>()?),
+ ))
}
}
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 85c6dd7..deb3ef3 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1184,6 +1184,14 @@ def
test_collected_rows_keep_their_values_as_pyspark(session):
assert (rows[0].v.toJson(), rows[0].n) == ("1", 2)
+def test_transpose_takes_a_column_name_as_pyspark(session):
+ # As PySpark's transpose, of a Column or a column name as the index column.
+ df = session.createDataFrame([("a", 1, 2), ("b", 3, 4)], "k string, x int,
y int")
+ by_name = df.transpose("k")
+ assert by_name.columns == ["key", "a", "b"]
+ assert by_name.collect() == df.transpose(F.col("k")).collect()
+
+
def test_getitem_indexes_filters_and_selects_as_pyspark(session):
from pyspark.errors import PySparkTypeError
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 73cf3e8..7e531bd 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -7601,6 +7601,36 @@ def
test_explain_checks_its_arguments_and_prints_with_print_as_pyspark(mock_conn
assert modes == [4, 2]
+def
test_col_regex_metadata_column_where_and_transpose_take_their_arguments_as_pyspark():
+ # As PySpark's: colRegex and metadataColumn take a str, where a Column or
a str, and
+ # transpose a Column or a column name, else NOT_EXPECTED_TYPE.
+ from pyspark.errors import PySparkTypeError
+ from pyspark.sql import functions as F
+
+ df = _offline_session().range(1)
+ for value in (1, None, ["id"], F.col("id")):
+ for call, arg_name, expected in (
+ (lambda: df.colRegex(value), "colName", "str"),
+ (lambda: df.metadataColumn(value), "colName", "str"),
+ ):
+ with pytest.raises(PySparkTypeError) as error:
+ call()
+ assert error.value.getMessageParameters() == {
+ "arg_name": arg_name, "expected_type": expected,
+ "arg_type": type(value).__name__}
+ for value in (1, 1.5, ["id"]):
+ for call, arg_name in ((lambda: df.where(value), "condition"),
+ (lambda: df.transpose(value), "col")):
+ with pytest.raises(PySparkTypeError) as error:
+ call()
+ assert error.value.getMessageParameters() == {
+ "arg_name": arg_name, "expected_type": "Column or str",
+ "arg_type": type(value).__name__}
+ # transpose takes a column name as the column of it.
+ assert type(df.transpose("id")).__name__ == "DataFrame"
+ assert type(df.transpose(F.col("id"))).__name__ == "DataFrame"
+
+
def test_session_context_manager_stops_session():
spark = _offline_session()
with spark as s:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]