This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new a71d501  [SPARK-61026][CONNECT][PYTHON] Take the positional fraction 
and seed of sample, the ordinals of sort and groupBy, the Columns of drop and 
the list of selectExpr as PySpark does
a71d501 is described below

commit a71d50163974455519c07a0f13c6ee1238aa9c68
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 04:32:06 2026 +0900

    [SPARK-61026][CONNECT][PYTHON] Take the positional fraction and seed of 
sample, the ordinals of sort and groupBy, the Columns of drop and the list of 
selectExpr as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61026](https://issues.apache.org/jira/browse/SPARK-61026): `sample`, 
`sort`, `groupBy`, `drop` and `selectExpr` now take their arguments as 
PySpark's do.
    
    - `df.sample(withReplacement, fraction, seed)` takes a fraction and a seed 
positionally, as in `sample(0.5, 42)`. It also takes a bool and a float, or a 
fraction alone, and raises `NOT_EXPECTED_TYPE` for other arguments.
    - `df.sort`, `df.orderBy` and `df.sortWithinPartitions` take the 1-based 
ordinals of columns, descending for a negative one. `df.sort(2)` sorts by the 
second column, and `df.sort(-2)` by it descending. They raise `CANNOT_BE_EMPTY` 
without columns, `ZERO_INDEX` for 0, and `NOT_EXPECTED_TYPE` for another type. 
They apply `ascending` as PySpark does.
    - `df.groupBy`, `df.rollup`, `df.cube` and `df.groupingSets` take the 
1-based ordinals of columns, and raise `INDEX_NOT_POSITIVE` for one that is not 
positive.
    - `df.drop` takes Columns, which it sends as expressions, as well as column 
names. It raises `NOT_EXPECTED_TYPE` for another type.
    - `df.selectExpr` takes a list of expressions.
    - The columns that `sort` and `groupBy` make of an ordinal or a name, and 
`df[ordinal]`, are those of `col`, and carry its origin, as PySpark's `F.col`. 
So errors in them carry the `DataFrameQueryContext` of `col`.
    
    In the Rust API, `DataFrame::drop_columns` drops Columns and column names.
    
    ### Why are the changes needed?
    
    - The 42 of `df.sample(0.5, 42)` was taken as the fraction, which failed on 
the server.
    - The ordinals of `sort` and `groupBy` raised `TypeError`s.
    - The Columns of `drop` raised `TypeError`s.
    - The list of `selectExpr` raised a `TypeError`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These methods take the arguments of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test sorts and groups by ordinals, drops Columns, selects a 
list of expressions, and samples by a positional fraction and seed. It passes 
with official PySpark 4.3.0, and fails without this change.
    - A new offline test checks the relations sent to a mock Connect server for 
these calls, and their errors. It fails without this change.
    - A new offline test checks, in a new interpreter against a mock Connect 
server, the `col` origin of `df[0]` and of the columns of `sort` and `groupBy` 
of ordinals and names, as official PySpark 4.3.0 sends them.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #227 from HyukjinKwon/SPARK-61026-df-args.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs    | 342 +++++++++++++++++++++++++---------
 crates/spark-connect/src/dataframe.rs |  10 +-
 crates/spark-connect/src/plan.rs      |  11 +-
 python/tests/test_dropin_e2e.py       |  18 ++
 python/tests/test_dropin_offline.py   | 114 ++++++++++++
 5 files changed, 399 insertions(+), 96 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 3750db0..2b87429 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -159,6 +159,145 @@ impl PyDataFrame {
     pub fn new(dataframe: DataFrame) -> Self {
         PyDataFrame { dataframe }
     }
+
+    /// The column at the 0-based index `index`, as PySpark's `df[index]`: 
`col` of the name of
+    /// the column at that index of `columns`, of Python's list indexing, and 
of the origin of
+    /// `col`.
+    fn column_at(&self, py: Python<'_>, index: i64) -> 
PyResult<spark_connect::column::Column> {
+        let columns = self.columns(py)?;
+        let name: String = PyList::new(py, columns)?
+            .as_any()
+            .get_item(index)?
+            .extract()?;
+        let _origin = crate::origin::with_origin("col");
+        Ok(spark_connect::functions::col(&name))
+    }
+
+    /// The grouping columns of `groupBy`, `rollup`, `cube` and 
`groupingSets`, as PySpark's
+    /// `_to_cols` of ordinals: Columns, column names and the 1-based ordinals 
of columns, or
+    /// a list or tuple of them.
+    fn grouping_columns(
+        &self,
+        py: Python<'_>,
+        cols: Vec<Bound<'_, PyAny>>,
+        arg_name: &str,
+    ) -> PyResult<Vec<spark_connect::column::Column>> {
+        use pyo3::types::{PyBool, PyString};
+        unwrap_sequence(cols)?
+            .iter()
+            .map(|c| {
+                if let Ok(column) = c.extract::<PyRef<'_, PyColumn>>() {
+                    return Ok(column.column.clone());
+                }
+                if c.is_instance_of::<PyString>() {
+                    // As PySpark's `F._to_col`, by `col`, of its origin.
+                    let _origin = crate::origin::with_origin("col");
+                    return 
Ok(spark_connect::functions::col(&c.extract::<String>()?));
+                }
+                if c.is_instance_of::<PyInt>() && 
!c.is_instance_of::<PyBool>() {
+                    let ordinal: i64 = c.extract()?;
+                    if ordinal < 1 {
+                        return Err(crate::column::raise_pyspark(
+                            py,
+                            "PySparkIndexError",
+                            "INDEX_NOT_POSITIVE",
+                            &[("index", &ordinal.to_string())],
+                        ));
+                    }
+                    return self.column_at(py, ordinal - 1);
+                }
+                Err(not_expected_type(c, arg_name, "Column, int or str"))
+            })
+            .collect()
+    }
+
+    /// The sort orders of `sort`, `orderBy` and `sortWithinPartitions`, as 
PySpark's
+    /// `_preapare_cols_for_sort`: Columns, column names and the 1-based 
ordinals of columns,
+    /// descending for a negative one, or a list or tuple of them, each 
descending for a
+    /// false `ascending`, or for a false one of a list of them.
+    fn sort_columns(
+        &self,
+        py: Python<'_>,
+        cols: Vec<Bound<'_, PyAny>>,
+        kwargs: Option<&Bound<'_, PyDict>>,
+    ) -> PyResult<Vec<spark_connect::expression::Expression>> {
+        use pyo3::types::{PyBool, PyString};
+        if cols.is_empty() {
+            return Err(crate::column::raise_pyspark(
+                py,
+                "PySparkValueError",
+                "CANNOT_BE_EMPTY",
+                &[("item", "cols")],
+            ));
+        }
+        let mut columns = Vec::new();
+        for c in unwrap_sequence(cols)? {
+            let column = if c.is_instance_of::<PyInt>() && 
!c.is_instance_of::<PyBool>() {
+                let ordinal: i64 = c.extract()?;
+                match ordinal {
+                    0 => {
+                        return Err(crate::column::raise_pyspark(
+                            py,
+                            "PySparkIndexError",
+                            "ZERO_INDEX",
+                            &[],
+                        ))
+                    }
+                    ordinal if ordinal > 0 => self.column_at(py, ordinal - 1)?,
+                    ordinal => self.column_at(py, -ordinal - 1)?.desc(),
+                }
+            } else if let Ok(column) = c.extract::<PyRef<'_, PyColumn>>() {
+                column.column.clone()
+            } else if c.is_instance_of::<PyString>() {
+                // As PySpark's `F._to_col`, by `col`, of its origin.
+                let _origin = crate::origin::with_origin("col");
+                spark_connect::functions::col(&c.extract::<String>()?)
+            } else {
+                return Err(not_expected_type(&c, "col", "Column, int or str"));
+            };
+            columns.push(column);
+        }
+        let ascending = match kwargs {
+            Some(kwargs) => kwargs.get_item("ascending")?,
+            None => None,
+        };
+        if let Some(ascending) = ascending {
+            if ascending.is_instance_of::<PyInt>() {
+                if !ascending.is_truthy()? {
+                    columns = columns.into_iter().map(|c| c.desc()).collect();
+                }
+            } else if ascending.is_instance_of::<PyList>() {
+                // As PySpark's zip of them, which keeps a column for each 
flag.
+                let flags: Vec<bool> = ascending
+                    .try_iter()?
+                    .map(|flag| flag?.is_truthy())
+                    .collect::<PyResult<_>>()?;
+                columns = columns
+                    .into_iter()
+                    .zip(flags)
+                    .map(|(c, asc)| if asc { c } else { c.desc() })
+                    .collect();
+            } else {
+                return Err(not_expected_type(
+                    &ascending,
+                    "ascending",
+                    "Column, int or str",
+                ));
+            }
+        }
+        Ok(columns.iter().map(|c| c.expression().clone()).collect())
+    }
+}
+
+/// The arguments of a method of `*cols`, the elements of a single list or 
tuple of them, as
+/// PySpark takes them.
+fn unwrap_sequence(cols: Vec<Bound<'_, PyAny>>) -> PyResult<Vec<Bound<'_, 
PyAny>>> {
+    match cols.as_slice() {
+        [only] if only.is_instance_of::<PyList>() || 
only.is_instance_of::<PyTuple>() => {
+            only.try_iter()?.collect()
+        }
+        _ => Ok(cols),
+    }
 }
 
 /// PySpark's `NOT_EXPECTED_TYPE` error of the argument `name`, `value`, which 
should be of
@@ -254,48 +393,6 @@ pub(crate) fn to_column_list(
     Ok(cols)
 }
 
-/// Build sort-order expressions for sort/orderBy/sortWithinPartitions, 
honoring an
-/// optional `ascending` kwarg (a bool applied to all columns, or a per-column 
list of
-/// bools/ints). Mirrors reference pyspark, which wraps each column with 
asc()/desc()
-/// when `ascending` is given; without it, columns are used as-is.
-fn sort_order_exprs(
-    columns: Vec<spark_connect::column::Column>,
-    kwargs: Option<&Bound<'_, PyDict>>,
-) -> PyResult<Vec<spark_connect::expression::Expression>> {
-    let ascending = match kwargs {
-        Some(kw) => kw.get_item("ascending")?,
-        None => None,
-    };
-    let Some(ascending) = ascending else {
-        return Ok(columns.iter().map(|c| c.expression().clone()).collect());
-    };
-    let flags: Vec<bool> = if let Ok(b) = ascending.extract::<bool>() {
-        vec![b; columns.len()]
-    } else if let Ok(i) = ascending.extract::<i64>() {
-        vec![i != 0; columns.len()]
-    } else if let Ok(list) = ascending.extract::<Vec<bool>>() {
-        list
-    } else if let Ok(list) = ascending.extract::<Vec<i64>>() {
-        list.into_iter().map(|i| i != 0).collect()
-    } else {
-        return Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(
-            "ascending must be a bool, an int, or a list of bools/ints",
-        ));
-    };
-    if flags.len() != columns.len() {
-        return Err(PyErr::new::<pyo3::exceptions::PyValueError, _>(format!(
-            "the list length of ascending ({}) must equal the number of sort 
columns ({})",
-            flags.len(),
-            columns.len()
-        )));
-    }
-    Ok(columns
-        .into_iter()
-        .zip(flags)
-        .map(|(c, asc)| (if asc { c.asc() } else { c.desc() 
}).expression().clone())
-        .collect())
-}
-
 /// Map a pyspark `how` string to a JoinType (shared by join / lateralJoin).
 fn parse_join_type(how: &str) -> PyResult<JoinType> {
     Ok(match how.to_lowercase().as_str() {
@@ -333,14 +430,29 @@ impl PyDataFrame {
         Ok(PyDataFrame::new(self.dataframe.select(columns)))
     }
 
-    /// Select using SQL expressions.
+    /// Select using SQL expressions, as PySpark's `DataFrame.selectExpr`: 
strings, or lists of
+    /// them.
     #[pyo3(signature = (*expr))]
-    fn selectExpr(&self, _py: Python<'_>, expr: Vec<String>) -> PyDataFrame {
-        let columns: Vec<_> = expr
-            .iter()
-            .map(|e| spark_connect::functions::expr(e))
-            .collect();
-        PyDataFrame::new(self.dataframe.select(columns))
+    fn selectExpr(&self, expr: Vec<Bound<'_, PyAny>>) -> PyResult<PyDataFrame> 
{
+        let expr = match expr.as_slice() {
+            [only] if only.is_instance_of::<PyList>() => {
+                only.try_iter()?.collect::<PyResult<Vec<_>>>()?
+            }
+            _ => expr,
+        };
+        let mut columns = Vec::new();
+        for element in &expr {
+            if element.is_instance_of::<pyo3::types::PyString>() {
+                columns.push(spark_connect::functions::expr(
+                    &element.extract::<String>()?,
+                ));
+            } else {
+                for e in element.try_iter()? {
+                    
columns.push(spark_connect::functions::expr(&e?.extract::<String>()?));
+                }
+            }
+        }
+        Ok(PyDataFrame::new(self.dataframe.select(columns)))
     }
 
     /// Filter rows by a Column condition or a SQL-string condition.
@@ -462,11 +574,32 @@ impl PyDataFrame {
         ))
     }
 
-    /// Drop columns.
+    /// Drop columns, of Columns or column names, as PySpark's 
`DataFrame.drop`.
     #[pyo3(signature = (*cols))]
-    fn drop(&self, _py: Python<'_>, cols: Vec<String>) -> PyDataFrame {
-        let col_refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
-        PyDataFrame::new(self.dataframe.drop(col_refs))
+    fn drop(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        let (mut columns, mut names) = (Vec::new(), Vec::new());
+        for column in &cols {
+            if let Ok(column) = column.extract::<PyRef<'_, PyColumn>>() {
+                columns.push(column.column.clone());
+            } else if column.is_instance_of::<pyo3::types::PyString>() {
+                names.push(column.extract::<String>()?);
+            } else {
+                return Err(crate::column::raise_pyspark(
+                    py,
+                    "PySparkTypeError",
+                    "NOT_EXPECTED_TYPE",
+                    &[
+                        ("expected_type", "Column or str"),
+                        ("arg_name", "cols"),
+                        ("arg_type", "tuple"),
+                    ],
+                ));
+            }
+        }
+        let names: Vec<&str> = names.iter().map(String::as_str).collect();
+        Ok(PyDataFrame::new(
+            self.dataframe.drop_columns(columns, names),
+        ))
     }
 
     /// Limit the number of rows.
@@ -503,7 +636,7 @@ impl PyDataFrame {
         cols: Vec<Bound<'_, PyAny>>,
         kwargs: Option<Bound<'_, PyDict>>,
     ) -> PyResult<PyDataFrame> {
-        let exprs = sort_order_exprs(to_column_list(cols)?, kwargs.as_ref())?;
+        let exprs = self.sort_columns(_py, cols, kwargs.as_ref())?;
         Ok(PyDataFrame::new(self.dataframe.sort(exprs)))
     }
 
@@ -515,7 +648,7 @@ impl PyDataFrame {
         cols: Vec<Bound<'_, PyAny>>,
         kwargs: Option<Bound<'_, PyDict>>,
     ) -> PyResult<PyDataFrame> {
-        let exprs = sort_order_exprs(to_column_list(cols)?, kwargs.as_ref())?;
+        let exprs = self.sort_columns(_py, cols, kwargs.as_ref())?;
         Ok(PyDataFrame::new(self.dataframe.order_by(exprs)))
     }
 
@@ -806,24 +939,26 @@ impl PyDataFrame {
 
     /// Group by columns for aggregation.
     #[pyo3(signature = (*cols))]
-    fn groupBy(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyGroupedData> {
-        let columns = to_column_list(cols)?;
+    fn groupBy(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyGroupedData> {
+        let columns = self.grouping_columns(py, cols, "cols")?;
         Ok(PyGroupedData::new(self.dataframe.group_by(columns)))
     }
 
     /// Rollup multi-dimensional aggregation.
     #[pyo3(signature = (*cols))]
-    fn rollup(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyGroupedData> {
+    fn rollup(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyGroupedData> {
         Ok(PyGroupedData::new(
-            self.dataframe.rollup(to_column_list(cols)?),
+            self.dataframe
+                .rollup(self.grouping_columns(py, cols, "cols")?),
         ))
     }
 
     /// Cube multi-dimensional aggregation.
     #[pyo3(signature = (*cols))]
-    fn cube(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyGroupedData> {
+    fn cube(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyGroupedData> {
         Ok(PyGroupedData::new(
-            self.dataframe.cube(to_column_list(cols)?),
+            self.dataframe
+                .cube(self.grouping_columns(py, cols, "cols")?),
         ))
     }
 
@@ -836,13 +971,14 @@ impl PyDataFrame {
     #[allow(unused_variables)]
     fn grouping_sets(
         &self,
+        py: Python<'_>,
         groupingSets: Vec<Vec<Bound<'_, PyAny>>>,
         cols: Vec<Bound<'_, PyAny>>,
     ) -> PyResult<PyGroupedData> {
         let mut out: Vec<Vec<spark_connect::column::Column>> =
             Vec::with_capacity(groupingSets.len());
         for s in groupingSets {
-            out.push(to_column_list(s)?);
+            out.push(self.grouping_columns(py, s, "groupingSets")?);
         }
         Ok(PyGroupedData::new(self.dataframe.grouping_sets(out)))
     }
@@ -890,7 +1026,7 @@ impl PyDataFrame {
         cols: Vec<Bound<'_, PyAny>>,
         kwargs: Option<Bound<'_, PyDict>>,
     ) -> PyResult<PyDataFrame> {
-        let exprs = sort_order_exprs(to_column_list(cols)?, kwargs.as_ref())?;
+        let exprs = self.sort_columns(_py, cols, kwargs.as_ref())?;
         Ok(PyDataFrame::new(
             self.dataframe.sort_within_partitions(exprs),
         ))
@@ -1680,43 +1816,61 @@ impl PyDataFrame {
         rows_to_py(py, rows, schema.as_ref())
     }
 
-    /// Sample a fraction of rows. Mirrors 
`DataFrame.sample(withReplacement=None,
-    /// fraction=None, seed=None)` including the legacy positional form
-    /// `sample(fraction)` (where the first positional arg is the fraction).
+    /// Sample a fraction of rows, as PySpark's 
`DataFrame.sample(withReplacement=None,
+    /// fraction=None, seed=None)`, which takes `sample(fraction[, seed])` 
positionally as
+    /// well: a bool and a float, a float, or a float and the seed.
     #[pyo3(signature = (withReplacement=None, fraction=None, seed=None))]
     fn sample(
         &self,
+        py: Python<'_>,
         withReplacement: Option<&Bound<'_, PyAny>>,
-        fraction: Option<f64>,
-        seed: Option<i64>,
+        fraction: Option<&Bound<'_, PyAny>>,
+        seed: Option<&Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
-        // Resolve the (withReplacement, fraction) overloads exactly like 
pyspark:
-        // if the first positional is a float and no fraction was given, it IS 
the
-        // fraction (legacy `sample(0.5)`); otherwise it's the bool 
withReplacement.
-        let (replace, frac) = match (withReplacement, fraction) {
-            (Some(w), None) => {
-                if let Ok(f) = w.extract::<f64>() {
-                    (false, f) // legacy sample(fraction)
-                } else {
-                    let b = w.extract::<bool>().map_err(|_| {
-                        PyErr::new::<pyo3::exceptions::PyTypeError, _>(
-                            "withReplacement must be a bool, or pass fraction 
as a float",
-                        )
-                    })?;
-                    return Err(PyErr::new::<pyo3::exceptions::PyValueError, 
_>(format!(
-                        "sample requires a fraction (got withReplacement={b} 
with no fraction)",
-                    )));
-                }
+        use pyo3::types::{PyBool, PyFloat};
+        let int = |value: &Bound<'_, PyAny>| -> PyResult<i64> {
+            py.import("builtins")?
+                .getattr("int")?
+                .call1((value,))?
+                .extract()
+        };
+        let (replace, frac, seed) = match (withReplacement, fraction) {
+            (Some(w), Some(f)) if w.is_instance_of::<PyBool>() && 
f.is_instance_of::<PyFloat>() => {
+                (w.extract()?, f.extract()?, seed.map(int).transpose()?)
             }
-            (Some(w), Some(f)) => {
-                let b = w.extract::<bool>().unwrap_or(false);
-                (b, f)
+            (None, Some(f)) if f.is_instance_of::<PyFloat>() => {
+                (false, f.extract()?, seed.map(int).transpose()?)
             }
-            (None, Some(f)) => (false, f),
-            (None, None) => {
-                return Err(PyErr::new::<pyo3::exceptions::PyValueError, _>(
-                    "sample requires a fraction",
-                ))
+            // `sample(fraction[, seed])`: the fraction is the first argument, 
and the seed the
+            // second.
+            (Some(w), f) if w.is_instance_of::<PyFloat>() => {
+                (false, w.extract()?, f.map(int).transpose()?)
+            }
+            _ => {
+                let type_name = |value: Option<&Bound<'_, PyAny>>| -> 
PyResult<String> {
+                    Ok(match value {
+                        Some(value) => value.get_type().name()?.to_string(),
+                        None => "NoneType".to_string(),
+                    })
+                };
+                let arg_types = [withReplacement, fraction, seed]
+                    .into_iter()
+                    .map(type_name)
+                    .collect::<PyResult<Vec<_>>>()?
+                    .join(", ");
+                return Err(crate::column::raise_pyspark(
+                    py,
+                    "PySparkTypeError",
+                    "NOT_EXPECTED_TYPE",
+                    &[
+                        ("expected_type", "bool, float or int"),
+                        (
+                            "arg_name",
+                            "withReplacement (optional), fraction (required) 
and seed (optional)",
+                        ),
+                        ("arg_type", &arg_types),
+                    ],
+                ));
             }
         };
         Ok(PyDataFrame::new(
@@ -1853,6 +2007,8 @@ impl PyDataFrame {
                 ],
             ));
         };
+        // As PySpark's `F.col(self.columns[item])`, of the origin of `col`.
+        let _origin = crate::origin::with_origin("col");
         PyColumn::new(spark_connect::functions::col(&name)).into_py_any(py)
     }
 
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index f1e9120..b0fd446 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -391,10 +391,16 @@ impl DataFrame {
 
     /// Drop columns.
     pub fn drop(&self, columns: Vec<&str>) -> DataFrame {
-        let col_names = columns.iter().map(|s| s.to_string()).collect();
+        self.drop_columns(Vec::new(), columns)
+    }
+
+    /// Drop `columns` and the columns of `column_names`, mirroring 
`pyspark.sql.DataFrame.drop`
+    /// of Columns and column names: a Column is sent as its expression, and a 
name as it is.
+    pub fn drop_columns(&self, columns: Vec<Column>, column_names: Vec<&str>) 
-> DataFrame {
         let plan = LogicalPlan::Drop {
             input: Arc::clone(&self.plan),
-            columns: col_names,
+            columns: column_names.iter().map(|s| s.to_string()).collect(),
+            expressions: columns.iter().map(|c| 
c.expression().clone()).collect(),
         };
         DataFrame::new(self.session.clone(), plan)
     }
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index 36be8a3..f0a700e 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -162,6 +162,8 @@ pub enum LogicalPlan {
     Drop {
         input: Arc<LogicalPlan>,
         columns: Vec<String>,
+        /// The columns to drop as expressions, as PySpark's `Drop` of Columns 
sends them.
+        expressions: Vec<Expression>,
     },
     /// ToDF: `df.toDF(*column_names)`.
     ToDF {
@@ -989,9 +991,15 @@ impl LogicalPlan {
                     
Some(proto::relation::RelType::WithColumnsRenamed(Box::new(wcr)));
             }
 
-            LogicalPlan::Drop { input, columns } => {
+            LogicalPlan::Drop {
+                input,
+                columns,
+                expressions,
+            } => {
                 let mut drop = proto::Drop::default();
                 drop.input = Some(Box::new(relation_of(input, plan_ids)));
+                drop.columns
+                    .extend(expressions.iter().map(|e| e.to_proto()));
                 drop.column_names.extend(columns.clone());
                 relation.rel_type = 
Some(proto::relation::RelType::Drop(Box::new(drop)));
             }
@@ -1850,6 +1858,7 @@ pub fn drop(input: impl Into<Arc<LogicalPlan>>, columns: 
Vec<String>) -> Logical
     LogicalPlan::Drop {
         input: input.into(),
         columns,
+        expressions: Vec::new(),
     }
 }
 
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 143995b..43a9e42 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -992,6 +992,24 @@ def test_collected_rows_behave_as_pyspark_rows(session):
     assert repr(pickle.loads(pickle.dumps(duplicate))) == "Row(a=1, a=2)"
 
 
+def test_sample_sort_group_by_drop_and_select_expr_as_pyspark(session):
+    # As PySpark's: sample takes a fraction and a seed positionally, sort and 
groupBy take the
+    # 1-based ordinals of columns, drop takes Columns, and selectExpr a list 
of expressions.
+    df = session.createDataFrame(
+        [(1, "b", 2.0), (2, "a", 1.0), (3, "c", 3.0)], "x int, k string, v 
double"
+    )
+    assert [r.k for r in df.sort(2).collect()] == ["a", "b", "c"]
+    assert [r.k for r in df.orderBy(-2).collect()] == ["c", "b", "a"]
+    one_partition = df.coalesce(1).sortWithinPartitions(3, ascending=False)
+    assert [r.v for r in one_partition.collect()] == [3.0, 2.0, 1.0]
+    grouped = df.groupBy(2).agg(F.max("x").alias("m")).orderBy("k").collect()
+    assert grouped == [Row(k="a", m=2), Row(k="b", m=1), Row(k="c", m=3)]
+    assert df.drop(df.x, "k").columns == ["v"]
+    assert df.drop(F.col("v")).columns == ["x", "k"]
+    assert df.selectExpr(["x", "k AS name"]).columns == ["x", "name"]
+    assert df.sample(0.5, 42).collect() == df.sample(fraction=0.5, 
seed=42).collect()
+
+
 def test_column_attributes_select_struct_fields_and_slices_substrings(session):
     df = session.sql("SELECT named_struct('x', 1, 'y', named_struct('z', 'q')) 
AS s, 'abcdef' AS t")
     out = df.select(df.s.x, df.s.y.z, F.col("s").x.alias("x2"), df.t[2:3], 
F.col("t")[F.lit(1):F.lit(2)])
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index b7d675c..297ce84 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -4908,6 +4908,120 @@ def 
test_create_dataframe_raises_inference_errors_as_pyspark(data, condition, pa
     assert error.value.getMessageParameters() == parameters
 
 
+def 
test_sample_sort_group_by_and_drop_take_their_arguments_as_pyspark(mock_connect):
+    # As PySpark's: sample takes a fraction and a seed positionally, sort and 
groupBy take the
+    # 1-based ordinals of columns, a negative one of sort descending, and drop 
takes Columns,
+    # which it sends as expressions.
+    import struct
+
+    from pyspark.errors import PySparkIndexError, PySparkTypeError, 
PySparkValueError
+    from pyspark.sql import functions as F
+
+    spark, service = mock_connect
+    df = spark.range(1)
+
+    def relation(dataframe, field):
+        # ExecutePlanRequest.plan=3, Plan.root=1.
+        dataframe.collect()
+        plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+        return parse_message(parse_message(plan)[field][0])
+
+    # Relation.sample=12: Sample.upper_bound=3, a double, seed=5.
+    for sampled in (df.sample(0.5, 42), df.sample(fraction=0.5, seed=42),
+                    df.sample(False, 0.5, 42)):
+        sample = relation(sampled, 12)
+        assert struct.unpack("<d", sample[3][0]) == (0.5,) and sample[5] == 
[42]
+    # Relation.drop=21: Drop.columns=2, of expressions, column_names=3.
+    drop = relation(df.drop(F.col("a"), "b"), 21)
+    assert len(drop[2]) == 1 and drop[3] == [b"b"]
+
+    def column_name(expression):
+        # Expression.unresolved_attribute=2, 
UnresolvedAttribute.unparsed_identifier=1.
+        return parse_message(parse_message(expression)[2][0])[1][0]
+
+    # Relation.sort=7, Sort.order=2: SortOrder.child=1, direction=2 
(DESCENDING=2).
+    for sorted_df, direction in ((df.sort(1), 1), (df.orderBy(-1), 2),
+                                 (df.sort(1, ascending=False), 2)):
+        order = parse_message(relation(sorted_df, 7)[2][0])
+        assert column_name(order[1][0]) == b"id" and order[2] == [direction]
+    # Relation.aggregate=9, Aggregate.grouping_expressions=3.
+    grouping = relation(df.groupBy(1).count(), 9)[3]
+    assert [column_name(expression) for expression in grouping] == [b"id"]
+
+    for call, (error_type, condition, parameters) in (
+        (lambda: df.sort(), (PySparkValueError, "CANNOT_BE_EMPTY", {"item": 
"cols"})),
+        (lambda: df.sort(0), (PySparkIndexError, "ZERO_INDEX", {})),
+        (lambda: df.groupBy(0), (PySparkIndexError, "INDEX_NOT_POSITIVE", 
{"index": "0"})),
+        (lambda: df.sort(1.5), (PySparkTypeError, "NOT_EXPECTED_TYPE",
+                                {"expected_type": "Column, int or str", 
"arg_name": "col",
+                                 "arg_type": "float"})),
+        (lambda: df.groupBy(1.5), (PySparkTypeError, "NOT_EXPECTED_TYPE",
+                                   {"expected_type": "Column, int or str", 
"arg_name": "cols",
+                                    "arg_type": "float"})),
+        (lambda: df.drop(1), (PySparkTypeError, "NOT_EXPECTED_TYPE",
+                              {"expected_type": "Column or str", "arg_name": 
"cols",
+                               "arg_type": "tuple"})),
+        (lambda: df.sample(True, 1), (PySparkTypeError, "NOT_EXPECTED_TYPE",
+                                      {"expected_type": "bool, float or int",
+                                       "arg_name": "withReplacement 
(optional), fraction "
+                                                   "(required) and seed 
(optional)",
+                                       "arg_type": "bool, int, NoneType"})),
+    ):
+        with pytest.raises(error_type) as error:
+            call()
+        assert error.value.getCondition() == condition
+        assert error.value.getMessageParameters() == parameters
+
+
+def test_ordinals_and_names_of_columns_carry_the_origin_of_col_as_pyspark():
+    # As PySpark's, `df[ordinal]`, and the columns of sort and groupBy of an 
ordinal or a
+    # name, are of `F.col`, whose origin they carry. In a new interpreter, 
whose first column
+    # of a session reads spark.python.sql.dataFrameDebugging.enabled.
+    import subprocess
+    import sys
+    from pathlib import Path
+
+    pytest.importorskip("grpc")
+    code = """
+import sys
+sys.path.insert(0, sys.argv[1])
+from _mock_connect_server import mock_connect_session, parse_message
+
+def fragment(expression):
+    # Expression.common=18 -> ExpressionCommon.origin=1 -> 
Origin.python_origin=1 ->
+    # PythonOrigin.fragment=1.
+    expression = parse_message(expression)
+    if 18 not in expression:
+        return None
+    common = parse_message(expression[18][0])
+    return parse_message(parse_message(common[1][0])[1][0])[1][0].decode()
+
+if __name__ == "__main__":
+    with mock_connect_session() as (spark, service):
+        df = spark.range(1)
+
+        def relation(dataframe, field):
+            # ExecutePlanRequest.plan=3, Plan.root=1.
+            dataframe.collect()
+            plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+            return parse_message(parse_message(plan)[field][0])
+
+        # Relation.project=3: Project.expressions=3.
+        found = [fragment(relation(df.select(df[0]), 3)[3][0])]
+        # Relation.sort=7: Sort.order=2, SortOrder.child=1.
+        for sorted_df in (df.sort(1), df.sort(-1), df.sort("id")):
+            found.append(fragment(parse_message(relation(sorted_df, 
7)[2][0])[1][0]))
+        # Relation.aggregate=9: Aggregate.grouping_expressions=3.
+        for grouped in (df.groupBy(1), df.groupBy("id")):
+            found.append(fragment(relation(grouped.count(), 9)[3][0]))
+        print(found)
+"""
+    tests = str(Path(__file__).resolve().parent)
+    result = subprocess.run(
+        [sys.executable, "-c", code, tests], check=True, capture_output=True, 
text=True)
+    assert result.stdout.strip().splitlines()[-1] == str(["col"] * 6)
+
+
 def test_create_dataframe_checks_every_row_against_the_merged_schema():
     from pyspark.errors import PySparkValueError
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to