This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 2f17367  [SPARK-61024][CONNECT][PYTHON] Raise the errors of PySpark 
for the arguments of DataFrame methods
2f17367 is described below

commit 2f173672183bc17238af0b3141ef53f3982a4782
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 03:48:35 2026 +0900

    [SPARK-61024][CONNECT][PYTHON] Raise the errors of PySpark for the 
arguments of DataFrame methods
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61024](https://issues.apache.org/jira/browse/SPARK-61024): DataFrame 
methods now check their arguments on the client and raise PySpark's errors.
    
    - `select`, `repartition` and the other methods that take columns raise 
`NOT_EXPECTED_TYPE` for a value that is neither a Column nor a string, as 
PySpark's `_to_cols`. `toDF` does so for a value that is not a string.
    - `withColumn` raises `NOT_EXPECTED_TYPE` for a value that is not a Column, 
and `withColumns` and `withColumnsRenamed` for one that is not a dict. 
`withColumns` asserts that it is given one dict of Columns. 
`withColumnsRenamed` sends the renames in the order of the dict.
    - `hint` raises `NOT_EXPECTED_TYPE` for a name that is not a string, and 
`INVALID_ITEM_FOR_CONTAINER` for a parameter that is not a string, a float, an 
int, a Column or a list of them. It sends each parameter as `lit` of it, a 
literal of its type.
    - `dropDuplicates`, `drop_duplicates` and `dropDuplicatesWithinWatermark` 
raise `NOT_EXPECTED_TYPE` for a subset that is not a list or tuple of strings.
    - `repartition`, `repartitionByRange`, `repartitionById` and `coalesce` 
raise `VALUE_NOT_POSITIVE` for a number of partitions that is not positive, and 
`NOT_EXPECTED_TYPE` for one of another type. `repartitionByRange` raises 
`CANNOT_BE_EMPTY` without columns, and sorts by each column ascending unless it 
is already a sort order.
    
    In the Rust API, `DataFrame::hint_expressions` takes the parameters of a 
hint as expressions.
    
    ### Why are the changes needed?
    
    These methods raised `TypeError`s, such as `argument 'col2': 'int' object 
is not an instance of 'str'`, whose `getCondition()` is not 
`NOT_EXPECTED_TYPE`. They also:
    
    - sent the parameters of `hint` as strings, and accepted parameters of 
other types;
    - sent numbers of partitions that were not positive to the server, which 
failed;
    - repartitioned by range by no columns.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These methods raise the errors of PySpark 4.3.0 for arguments of other 
types.
    
    ### How was this patch tested?
    
    - A new offline test checks the errors of 24 calls. It passes with official 
PySpark 4.3.0, and fails without this change.
    - Probes of these calls match official PySpark 4.3.0.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #226 from HyukjinKwon/SPARK-61024-type-errors.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs    | 322 ++++++++++++++++++++++++++--------
 crates/spark-connect/src/dataframe.rs |  12 +-
 crates/spark-connect/src/plan.rs      |  44 ++---
 python/tests/test_dropin_offline.py   |  59 +++++++
 4 files changed, 346 insertions(+), 91 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index a1e0517..63bfc59 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -161,6 +161,65 @@ impl PyDataFrame {
     }
 }
 
+/// PySpark's `NOT_EXPECTED_TYPE` error of the argument `name`, `value`, which 
should be of
+/// the type `expected`.
+fn not_expected_type(value: &Bound<'_, PyAny>, name: &str, expected: &str) -> 
PyErr {
+    let type_name = value
+        .get_type()
+        .name()
+        .map(|name| name.to_string())
+        .unwrap_or_default();
+    crate::column::raise_pyspark(
+        value.py(),
+        "PySparkTypeError",
+        "NOT_EXPECTED_TYPE",
+        &[
+            ("expected_type", expected),
+            ("arg_name", name),
+            ("arg_type", &type_name),
+        ],
+    )
+}
+
+/// PySpark's `VALUE_NOT_POSITIVE` error of `numPartitions`, unless it is 
positive.
+fn positive_partitions(py: Python<'_>, num_partitions: i64) -> PyResult<i32> {
+    if num_partitions > 0 {
+        return Ok(i32::try_from(num_partitions)?);
+    }
+    Err(crate::column::raise_pyspark(
+        py,
+        "PySparkValueError",
+        "VALUE_NOT_POSITIVE",
+        &[
+            ("arg_name", "numPartitions"),
+            ("arg_value", &num_partitions.to_string()),
+        ],
+    ))
+}
+
+/// The `subset` of `dropDuplicates` and `dropDuplicatesWithinWatermark`, as 
PySpark's: None,
+/// or a list or tuple of column names.
+fn deduplicate_subset(subset: Option<&Bound<'_, PyAny>>) -> 
PyResult<Option<Vec<String>>> {
+    let Some(subset) = subset else {
+        return Ok(None);
+    };
+    if !(subset.is_instance_of::<PyList>() || 
subset.is_instance_of::<PyTuple>()) {
+        return Err(not_expected_type(subset, "subset", "list or tuple"));
+    }
+    subset
+        .try_iter()?
+        .map(|column| {
+            let column = column?;
+            if column.is_instance_of::<pyo3::types::PyString>() {
+                column.extract()
+            } else {
+                Err(not_expected_type(&column, "subset", "str"))
+            }
+        })
+        .collect::<PyResult<Vec<String>>>()
+        .map(Some)
+}
+
 /// Helper to convert arguments to a vector of Columns.
 pub(crate) fn to_column_list(
     mut _args: Vec<Bound<'_, PyAny>>,
@@ -187,9 +246,8 @@ pub(crate) fn to_column_list(
                 let _origin = crate::origin::with_origin("col");
                 cols.push(spark_connect::functions::col(&name));
             } else {
-                return Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(
-                    "Expected Column or column name (str)",
-                ));
+                // As PySpark's `_to_cols`.
+                return Err(not_expected_type(&arg, "cols", "Column or str"));
             }
         }
     }
@@ -300,9 +358,14 @@ impl PyDataFrame {
         ))
     }
 
-    /// Add or replace a column.
-    fn withColumn(&self, colName: &str, col: &PyColumn) -> PyDataFrame {
-        PyDataFrame::new(self.dataframe.with_column(colName, 
col.column.clone()))
+    /// Add or replace a column, of a Column, as PySpark's 
`DataFrame.withColumn`.
+    fn withColumn(&self, colName: &str, col: &Bound<'_, PyAny>) -> 
PyResult<PyDataFrame> {
+        let Ok(column) = col.extract::<PyRef<'_, PyColumn>>() else {
+            return Err(not_expected_type(col, "col", "Column"));
+        };
+        Ok(PyDataFrame::new(
+            self.dataframe.with_column(colName, column.column.clone()),
+        ))
     }
 
     /// Rename a column.
@@ -310,16 +373,22 @@ impl PyDataFrame {
         PyDataFrame::new(self.dataframe.with_column_renamed(existing, new))
     }
 
-    /// Add or replace multiple columns from a {name: Column} mapping.
+    /// Add or replace multiple columns from a {name: Column} mapping, as 
PySpark's
+    /// `DataFrame.withColumns`, which asserts that it is given one, of 
Columns.
     #[pyo3(name = "withColumns", signature = (*colsMap))]
-    fn with_columns(&self, colsMap: Vec<Bound<'_, PyDict>>) -> 
PyResult<PyDataFrame> {
-        let mut pairs = Vec::new();
-        for d in &colsMap {
-            for (k, v) in d.iter() {
-                let name: String = k.extract()?;
-                let pycol = v.extract::<PyRef<PyColumn>>()?;
-                pairs.push((name, pycol.column.clone()));
-            }
+    fn with_columns(&self, colsMap: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        let [cols_map] = colsMap.as_slice() else {
+            return Err(pyo3::exceptions::PyAssertionError::new_err(()));
+        };
+        let Ok(cols_map) = cols_map.cast::<PyDict>() else {
+            return Err(not_expected_type(cols_map, "colsMap", "dict"));
+        };
+        let mut pairs = Vec::with_capacity(cols_map.len());
+        for (name, column) in cols_map.iter() {
+            let Ok(column) = column.extract::<PyRef<'_, PyColumn>>() else {
+                return Err(pyo3::exceptions::PyAssertionError::new_err(()));
+            };
+            pairs.push((name.extract::<String>()?, column.column.clone()));
         }
         Ok(PyDataFrame::new(self.dataframe.with_columns(pairs)))
     }
@@ -329,12 +398,68 @@ impl PyDataFrame {
     /// accept any value and stringify it; the core parses a numeric string 
back into a
     /// long literal, matching reference semantics.
     #[pyo3(signature = (name, *parameters))]
-    fn hint(&self, name: &str, parameters: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
-        let params: Vec<String> = parameters
-            .iter()
-            .map(|p| Ok(p.str()?.to_string()))
-            .collect::<PyResult<_>>()?;
-        Ok(PyDataFrame::new(self.dataframe.hint(name, params)))
+    fn hint(
+        &self,
+        py: Python<'_>,
+        name: &Bound<'_, PyAny>,
+        parameters: Vec<Bound<'_, PyAny>>,
+    ) -> PyResult<PyDataFrame> {
+        use pyo3::types::{PyFloat, PyString};
+        // As PySpark's: a single list is the parameters, each a str, a float, 
an int, a
+        // Column or a list of str, float or int, sent as `lit` of it.
+        let parameters = match parameters.as_slice() {
+            [only] if only.is_instance_of::<PyList>() => {
+                only.try_iter()?.collect::<PyResult<Vec<_>>>()?
+            }
+            _ => parameters,
+        };
+        if !name.is_instance_of::<PyString>() {
+            return Err(not_expected_type(name, "name", "str"));
+        }
+        let primitive = |p: &Bound<'_, PyAny>| {
+            p.is_instance_of::<PyString>()
+                || p.is_instance_of::<PyFloat>()
+                || p.is_instance_of::<PyInt>()
+        };
+        let invalid = |item_type: &str| {
+            crate::column::raise_pyspark(
+                py,
+                "PySparkTypeError",
+                "INVALID_ITEM_FOR_CONTAINER",
+                &[
+                    ("arg_name", "parameters"),
+                    (
+                        "allowed_types",
+                        "str, float, int, Column, list[str], list[float], 
list[int]",
+                    ),
+                    ("item_type", item_type),
+                ],
+            )
+        };
+        let type_name =
+            |p: &Bound<'_, PyAny>| -> PyResult<String> { 
Ok(p.get_type().name()?.to_string()) };
+        let mut expressions = Vec::with_capacity(parameters.len());
+        for p in &parameters {
+            if p.is_instance_of::<PyList>() {
+                let items: Vec<Bound<'_, PyAny>> = 
p.try_iter()?.collect::<PyResult<_>>()?;
+                if !items.iter().all(primitive) {
+                    // As PySpark's, which names the type of the first item.
+                    let first = items
+                        .first()
+                        .map(type_name)
+                        .transpose()?
+                        .unwrap_or_default();
+                    return Err(invalid(&format!("list[{first}]")));
+                }
+            } else if !(primitive(p) || p.extract::<PyRef<'_, 
PyColumn>>().is_ok()) {
+                return Err(invalid(&type_name(p)?));
+            }
+            
expressions.push(crate::functions::to_column(p)?.expression().clone());
+        }
+        let name: String = name.extract()?;
+        Ok(PyDataFrame::new(
+            self.dataframe.hint_expressions(&name, expressions),
+        ))
     }
 
     /// Drop columns.
@@ -361,11 +486,12 @@ impl PyDataFrame {
 
     /// Remove duplicate rows, optionally on specific columns.
     #[pyo3(signature = (subset=None))]
-    fn dropDuplicates(&self, _py: Python<'_>, subset: Option<Vec<String>>) -> 
PyDataFrame {
+    fn dropDuplicates(&self, subset: Option<&Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        let subset = deduplicate_subset(subset)?;
         let col_refs: Option<Vec<&str>> = subset
             .as_ref()
             .map(|names| names.iter().map(|s| s.as_str()).collect());
-        PyDataFrame::new(self.dataframe.drop_duplicates(col_refs))
+        Ok(PyDataFrame::new(self.dataframe.drop_duplicates(col_refs)))
     }
 
     /// Sort rows. `ascending` (a bool or list of bools, via kwargs) mirrors 
reference
@@ -484,13 +610,45 @@ impl PyDataFrame {
     #[allow(non_snake_case)]
     fn repartitionByRange(
         &self,
-        numPartitions: i32,
+        py: Python<'_>,
+        numPartitions: Bound<'_, PyAny>,
         cols: Vec<Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
-        let columns = to_column_list(cols)?;
-        let exprs: Vec<_> = columns.iter().map(|c| 
c.expression().clone()).collect();
+        // As PySpark's: a positive partition count and columns, or columns 
alone, each sorted
+        // ascending unless it is a sort order.
+        let (num_partitions, cols) = if 
numPartitions.is_instance_of::<PyInt>() {
+            let n = positive_partitions(py, numPartitions.extract()?)?;
+            if cols.is_empty() {
+                return Err(crate::column::raise_pyspark(
+                    py,
+                    "PySparkValueError",
+                    "CANNOT_BE_EMPTY",
+                    &[("item", "cols")],
+                ));
+            }
+            (n, cols)
+        } else if numPartitions.is_instance_of::<pyo3::types::PyString>()
+            || numPartitions.extract::<PyRef<'_, PyColumn>>().is_ok()
+        {
+            let mut all = vec![numPartitions];
+            all.extend(cols);
+            (0, all)
+        } else {
+            return Err(not_expected_type(
+                &numPartitions,
+                "numPartitions",
+                "Column, int or str",
+            ));
+        };
+        let exprs: Vec<_> = to_column_list(cols)?
+            .into_iter()
+            .map(|c| match c.expression().node() {
+                spark_connect::expression::ExpressionNode::SortOrder(_) => 
c.expression().clone(),
+                _ => c.asc().expression().clone(),
+            })
+            .collect();
         Ok(PyDataFrame::new(
-            self.dataframe.repartition_by_range(numPartitions, exprs),
+            self.dataframe.repartition_by_range(num_partitions, exprs),
         ))
     }
 
@@ -500,14 +658,17 @@ impl PyDataFrame {
     #[allow(non_snake_case)]
     fn repartitionById(
         &self,
-        numPartitions: i32,
+        py: Python<'_>,
+        numPartitions: &Bound<'_, PyAny>,
         partitionIdCol: &Bound<'_, PyAny>,
     ) -> PyResult<PyDataFrame> {
-        if numPartitions <= 0 {
-            return Err(pyo3::exceptions::PyValueError::new_err(format!(
-                "numPartitions must be positive, got {numPartitions}"
-            )));
+        // As PySpark's: an int that is not a bool, and positive.
+        if !numPartitions.is_instance_of::<PyInt>()
+            || numPartitions.is_instance_of::<pyo3::types::PyBool>()
+        {
+            return Err(not_expected_type(numPartitions, "numPartitions", 
"int"));
         }
+        let numPartitions = positive_partitions(py, numPartitions.extract()?)?;
         // A `ColumnOrName`, as in pyspark: a string is a column name, not a 
literal.
         let col = if let Ok(c) = partitionIdCol.extract::<Bound<'_, 
PyColumn>>() {
             c.borrow().column.clone()
@@ -582,9 +743,10 @@ impl PyDataFrame {
         numPartitions: Bound<'_, PyAny>,
         cols: Vec<Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
-        // First arg is an int -> a partition count; otherwise it's a 
partition column
-        // (str/Column) and the count defaults to the server's (passed as 0 = 
unset).
-        if let Ok(n) = numPartitions.extract::<i32>() {
+        // As PySpark's: a positive partition count, or a partition column 
(str/Column), at
+        // the server's count (passed as 0 = unset).
+        if numPartitions.is_instance_of::<PyInt>() {
+            let n = positive_partitions(py, numPartitions.extract()?)?;
             if cols.is_empty() {
                 return Ok(PyDataFrame::new(self.dataframe.repartition(n)));
             }
@@ -592,34 +754,49 @@ impl PyDataFrame {
                 .iter()
                 .map(|c| c.expression().clone())
                 .collect();
-            Ok(PyDataFrame::new(
+            return Ok(PyDataFrame::new(
                 self.dataframe.repartition_by_expressions(n, exprs),
-            ))
-        } else {
-            // Column-first form: the first arg is itself a partition column.
-            let mut all = vec![numPartitions];
-            all.extend(cols);
-            let _ = py;
-            let exprs: Vec<_> = to_column_list(all)?
-                .iter()
-                .map(|c| c.expression().clone())
-                .collect();
-            Ok(PyDataFrame::new(
-                self.dataframe.repartition_by_expressions(0, exprs),
-            ))
+            ));
+        }
+        if !(numPartitions.is_instance_of::<pyo3::types::PyString>()
+            || numPartitions.extract::<PyRef<'_, PyColumn>>().is_ok())
+        {
+            return Err(not_expected_type(
+                &numPartitions,
+                "numPartitions",
+                "Column or str",
+            ));
         }
+        let mut all = vec![numPartitions];
+        all.extend(cols);
+        let exprs: Vec<_> = to_column_list(all)?
+            .iter()
+            .map(|c| c.expression().clone())
+            .collect();
+        Ok(PyDataFrame::new(
+            self.dataframe.repartition_by_expressions(0, exprs),
+        ))
     }
 
     /// Coalesce.
-    fn coalesce(&self, numPartitions: i32) -> PyDataFrame {
-        PyDataFrame::new(self.dataframe.coalesce(numPartitions))
+    fn coalesce(&self, py: Python<'_>, numPartitions: i64) -> 
PyResult<PyDataFrame> {
+        let num_partitions = positive_partitions(py, numPartitions)?;
+        Ok(PyDataFrame::new(self.dataframe.coalesce(num_partitions)))
     }
 
     /// Convert to DataFrame with new column names.
     #[pyo3(signature = (*cols))]
-    fn toDF(&self, _py: Python<'_>, cols: Vec<String>) -> PyDataFrame {
-        let name_refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
-        PyDataFrame::new(self.dataframe.to_df(name_refs))
+    fn toDF(&self, cols: Vec<Bound<'_, PyAny>>) -> PyResult<PyDataFrame> {
+        // As PySpark's, of strings.
+        let mut names = Vec::with_capacity(cols.len());
+        for col in &cols {
+            if !col.is_instance_of::<pyo3::types::PyString>() {
+                return Err(not_expected_type(col, "cols", "list[str]"));
+            }
+            names.push(col.extract::<String>()?);
+        }
+        let name_refs: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
+        Ok(PyDataFrame::new(self.dataframe.to_df(name_refs)))
     }
 
     /// Alias this DataFrame.
@@ -672,11 +849,17 @@ impl PyDataFrame {
 
     /// Drop duplicates within the event-time watermark.
     #[pyo3(name = "dropDuplicatesWithinWatermark", signature = (subset=None))]
-    fn drop_duplicates_within_watermark(&self, subset: Option<Vec<String>>) -> 
PyDataFrame {
+    fn drop_duplicates_within_watermark(
+        &self,
+        subset: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<PyDataFrame> {
+        let subset = deduplicate_subset(subset)?;
         let refs: Option<Vec<&str>> = subset
             .as_ref()
             .map(|names| names.iter().map(|s| s.as_str()).collect());
-        PyDataFrame::new(self.dataframe.drop_duplicates_within_watermark(refs))
+        Ok(PyDataFrame::new(
+            self.dataframe.drop_duplicates_within_watermark(refs),
+        ))
     }
 
     /// `DataFrame.toJSON`, as PySpark Connect's: a DataFrame of each row as a 
JSON object
@@ -721,14 +904,18 @@ impl PyDataFrame {
 
     /// Rename multiple columns from an {old: new} mapping.
     #[pyo3(name = "withColumnsRenamed")]
-    fn with_columns_renamed(
-        &self,
-        colsMap: std::collections::HashMap<String, String>,
-    ) -> PyDataFrame {
-        PyDataFrame::new(
-            self.dataframe
-                .with_columns_renamed(colsMap.into_iter().collect()),
-        )
+    fn with_columns_renamed(&self, colsMap: &Bound<'_, PyAny>) -> 
PyResult<PyDataFrame> {
+        // As PySpark's, of a dict, whose renames it sends in order.
+        let Ok(cols_map) = colsMap.cast::<PyDict>() else {
+            return Err(not_expected_type(colsMap, "colsMap", "dict"));
+        };
+        let mut renames = Vec::with_capacity(cols_map.len());
+        for (existing, new) in cols_map.iter() {
+            renames.push((existing.extract::<String>()?, 
new.extract::<String>()?));
+        }
+        Ok(PyDataFrame::new(
+            self.dataframe.with_columns_renamed(renames),
+        ))
     }
 
     /// The columns of this DataFrame whose names match a regex, as a Column. 
Mirrors
@@ -1140,11 +1327,8 @@ impl PyDataFrame {
 
     /// Deprecated alias for dropDuplicates.
     #[pyo3(name = "drop_duplicates", signature = (subset=None))]
-    fn drop_duplicates(&self, subset: Option<Vec<String>>) -> PyDataFrame {
-        let refs: Option<Vec<&str>> = subset
-            .as_ref()
-            .map(|s| s.iter().map(|x| x.as_str()).collect());
-        PyDataFrame::new(self.dataframe.drop_duplicates(refs))
+    fn drop_duplicates(&self, subset: Option<&Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        self.dropDuplicates(subset)
     }
 
     /// Alias for groupBy.
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index 68d048f..f1e9120 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -631,10 +631,20 @@ impl DataFrame {
         name: &str,
         parameters: impl IntoIterator<Item = S>,
     ) -> DataFrame {
+        let parameters = parameters
+            .into_iter()
+            .map(|p| crate::plan::hint_parameter(&p.into()))
+            .collect();
+        self.hint_expressions(name, parameters)
+    }
+
+    /// Specify a hint of parameters given as expressions, mirroring 
`pyspark.sql.DataFrame.hint`,
+    /// which sends `lit` of each parameter: a literal, or a Column.
+    pub fn hint_expressions(&self, name: &str, parameters: Vec<Expression>) -> 
DataFrame {
         let plan = LogicalPlan::Hint {
             input: Arc::clone(&self.plan),
             name: name.to_string(),
-            parameters: parameters.into_iter().map(Into::into).collect(),
+            parameters,
         };
         self.with_cached_schema(plan)
     }
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index e9ef63d..36be8a3 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -177,7 +177,8 @@ pub enum LogicalPlan {
     Hint {
         input: Arc<LogicalPlan>,
         name: String,
-        parameters: Vec<String>,
+        /// The parameters, literal expressions or columns, as PySpark's `lit` 
of them.
+        parameters: Vec<Expression>,
     },
     /// Unpivot: `df.unpivot(...)`.
     Unpivot {
@@ -1020,19 +1021,8 @@ impl LogicalPlan {
                 let mut hint = proto::Hint::default();
                 hint.input = Some(Box::new(relation_of(input, plan_ids)));
                 hint.name = name.clone();
-                // Hint parameters are literal Expressions (e.g. `REPARTITION 
10`).
-                // An integer-looking parameter becomes an Integer (int32) 
literal -
-                // matching reference pyspark's `lit(int)` - otherwise a 
String literal.
-                // (A Long literal is rejected as a partitionNum by e.g. the 
REBALANCE
-                // hint, which expects an integral int.)
-                for p in parameters {
-                    let lit = if let Ok(n) = p.parse::<i32>() {
-                        crate::expression::LiteralExpression::int(n)
-                    } else {
-                        crate::expression::LiteralExpression::string(p.clone())
-                    };
-                    hint.parameters.push(lit.to_proto());
-                }
+                hint.parameters
+                    .extend(parameters.iter().map(|p| p.to_proto()));
                 relation.rel_type = 
Some(proto::relation::RelType::Hint(Box::new(hint)));
             }
 
@@ -1879,7 +1869,10 @@ pub fn to_schema(input: impl Into<Arc<LogicalPlan>>, 
schema: DataType) -> Logica
     }
 }
 
-/// Create a Hint plan.
+/// Create a Hint plan, of parameters given as strings: an integer-looking 
parameter is an
+/// Integer (int32) literal, matching PySpark's `lit(int)`, otherwise a String 
literal. (A
+/// Long literal is rejected as a partitionNum by e.g. the REBALANCE hint, 
which expects an
+/// integral int.)
 pub fn hint(
     input: impl Into<Arc<LogicalPlan>>,
     name: impl Into<String>,
@@ -1888,10 +1881,19 @@ pub fn hint(
     LogicalPlan::Hint {
         input: input.into(),
         name: name.into(),
-        parameters,
+        parameters: parameters.iter().map(|p| hint_parameter(p)).collect(),
     }
 }
 
+/// The literal of a hint parameter given as a string: an Integer for an 
integer-looking one,
+/// otherwise the String.
+pub(crate) fn hint_parameter(parameter: &str) -> Expression {
+    Expression::literal(match parameter.parse::<i32>() {
+        Ok(n) => LiteralExpression::int(n),
+        Err(_) => LiteralExpression::string(parameter),
+    })
+}
+
 /// Create an Unpivot plan.
 pub fn unpivot(
     input: impl Into<Arc<LogicalPlan>>,
@@ -2106,11 +2108,11 @@ mod argfix_tests {
 
     #[test]
     fn hint_carries_parameters() {
-        let p = LogicalPlan::Hint {
-            input: Arc::new(base()),
-            name: "REPARTITION".to_string(),
-            parameters: vec!["10".to_string(), "name".to_string()],
-        };
+        let p = hint(
+            base(),
+            "REPARTITION",
+            vec!["10".to_string(), "name".to_string()],
+        );
         match rel_type(p) {
             proto::relation::RelType::Hint(h) => {
                 assert_eq!(h.parameters.len(), 2, "parameters must be 
forwarded");
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 9852f80..75cd9f9 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -7997,6 +7997,65 @@ def 
test_writer_v2_setters_mutate_and_return_self(offline_spark):
     assert w.clusterBy("id") is w
 
 
+def test_dataframe_methods_raise_the_argument_errors_of_pyspark():
+    # As PySpark's DataFrame methods, which check their arguments on the 
client.
+    from pyspark.errors import PySparkTypeError, PySparkValueError
+
+    df = _offline_session().range(1)
+
+    def not_expected(arg_name, expected_type, arg_type):
+        return (PySparkTypeError, "NOT_EXPECTED_TYPE",
+                {"arg_name": arg_name, "expected_type": expected_type, 
"arg_type": arg_type})
+
+    def not_positive(value):
+        return (PySparkValueError, "VALUE_NOT_POSITIVE",
+                {"arg_name": "numPartitions", "arg_value": value})
+
+    allowed_hint_types = "str, float, int, Column, list[str], list[float], 
list[int]"
+    for call, (error_type, condition, parameters) in (
+        (lambda: df.select(1), not_expected("cols", "Column or str", "int")),
+        (lambda: df.select(["id", 1.5]), not_expected("cols", "Column or str", 
"float")),
+        (lambda: df.toDF(1), not_expected("cols", "list[str]", "int")),
+        (lambda: df.withColumn("a", 1), not_expected("col", "Column", "int")),
+        (lambda: df.withColumns([("a", 1)]), not_expected("colsMap", "dict", 
"list")),
+        (lambda: df.withColumnsRenamed([("id", "a")]), not_expected("colsMap", 
"dict", "list")),
+        (lambda: df.hint(1), not_expected("name", "str", "int")),
+        (lambda: df.hint("x", {"a": 1}),
+         (PySparkTypeError, "INVALID_ITEM_FOR_CONTAINER",
+          {"arg_name": "parameters", "allowed_types": allowed_hint_types, 
"item_type": "dict"})),
+        (lambda: df.hint("x", [None, 1]),
+         (PySparkTypeError, "INVALID_ITEM_FOR_CONTAINER",
+          {"arg_name": "parameters", "allowed_types": allowed_hint_types,
+           "item_type": "NoneType"})),
+        (lambda: df.hint("x", [1], [2, None]),
+         (PySparkTypeError, "INVALID_ITEM_FOR_CONTAINER",
+          {"arg_name": "parameters", "allowed_types": allowed_hint_types,
+           "item_type": "list[int]"})),
+        (lambda: df.dropDuplicates("id"), not_expected("subset", "list or 
tuple", "str")),
+        (lambda: df.drop_duplicates(["id", 1]), not_expected("subset", "str", 
"int")),
+        (lambda: df.dropDuplicatesWithinWatermark("id"),
+         not_expected("subset", "list or tuple", "str")),
+        (lambda: df.repartition(0), not_positive("0")),
+        (lambda: df.repartition(1.5), not_expected("numPartitions", "Column or 
str", "float")),
+        (lambda: df.repartition("id", 2), not_expected("cols", "Column or 
str", "int")),
+        (lambda: df.repartitionByRange(-1, "id"), not_positive("-1")),
+        (lambda: df.repartitionByRange(2), (PySparkValueError, 
"CANNOT_BE_EMPTY",
+                                            {"item": "cols"})),
+        (lambda: df.repartitionByRange(1.5, "id"),
+         not_expected("numPartitions", "Column, int or str", "float")),
+        (lambda: df.coalesce(0), not_positive("0")),
+        (lambda: df.repartitionById(True, "id"), not_expected("numPartitions", 
"int", "bool")),
+        (lambda: df.repartitionById(0, "id"), not_positive("0")),
+    ):
+        with pytest.raises(error_type) as error:
+            call()
+        assert error.value.getCondition() == condition
+        assert error.value.getMessageParameters() == parameters
+    for call in (lambda: df.withColumns(), lambda: df.withColumns({"a": 1})):
+        with pytest.raises(AssertionError):
+            call()
+
+
 def test_stream_writer_setters_mutate_and_return_self(offline_spark):
     w = offline_spark.readStream.format("rate").load().writeStream
     assert w.outputMode("append") is w


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to