This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 2f17367 [SPARK-61024][CONNECT][PYTHON] Raise the errors of PySpark
for the arguments of DataFrame methods
2f17367 is described below
commit 2f173672183bc17238af0b3141ef53f3982a4782
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 03:48:35 2026 +0900
[SPARK-61024][CONNECT][PYTHON] Raise the errors of PySpark for the
arguments of DataFrame methods
### What changes were proposed in this pull request?
[SPARK-61024](https://issues.apache.org/jira/browse/SPARK-61024): DataFrame
methods now check their arguments on the client and raise PySpark's errors.
- `select`, `repartition` and the other methods that take columns raise
`NOT_EXPECTED_TYPE` for a value that is neither a Column nor a string, as
PySpark's `_to_cols`. `toDF` does so for a value that is not a string.
- `withColumn` raises `NOT_EXPECTED_TYPE` for a value that is not a Column,
and `withColumns` and `withColumnsRenamed` for one that is not a dict.
`withColumns` asserts that it is given one dict of Columns.
`withColumnsRenamed` sends the renames in the order of the dict.
- `hint` raises `NOT_EXPECTED_TYPE` for a name that is not a string, and
`INVALID_ITEM_FOR_CONTAINER` for a parameter that is not a string, a float, an
int, a Column or a list of them. It sends each parameter as `lit` of it, a
literal of its type.
- `dropDuplicates`, `drop_duplicates` and `dropDuplicatesWithinWatermark`
raise `NOT_EXPECTED_TYPE` for a subset that is not a list or tuple of strings.
- `repartition`, `repartitionByRange`, `repartitionById` and `coalesce`
raise `VALUE_NOT_POSITIVE` for a number of partitions that is not positive, and
`NOT_EXPECTED_TYPE` for one of another type. `repartitionByRange` raises
`CANNOT_BE_EMPTY` without columns, and sorts by each column ascending unless it
is already a sort order.
In the Rust API, `DataFrame::hint_expressions` takes the parameters of a
hint as expressions.
### Why are the changes needed?
These methods raised `TypeError`s, such as `argument 'col2': 'int' object
is not an instance of 'str'`, whose `getCondition()` is not
`NOT_EXPECTED_TYPE`. They also:
- sent the parameters of `hint` as strings, and accepted parameters of
other types;
- sent numbers of partitions that were not positive to the server, which
failed;
- repartitioned by range by no columns.
### Does this PR introduce _any_ user-facing change?
Yes. These methods raise the errors of PySpark 4.3.0 for arguments of other
types.
### How was this patch tested?
- A new offline test checks the errors of 24 calls. It passes with official
PySpark 4.3.0, and fails without this change.
- Probes of these calls match official PySpark 4.3.0.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #226 from HyukjinKwon/SPARK-61024-type-errors.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 322 ++++++++++++++++++++++++++--------
crates/spark-connect/src/dataframe.rs | 12 +-
crates/spark-connect/src/plan.rs | 44 ++---
python/tests/test_dropin_offline.py | 59 +++++++
4 files changed, 346 insertions(+), 91 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index a1e0517..63bfc59 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -161,6 +161,65 @@ impl PyDataFrame {
}
}
+/// PySpark's `NOT_EXPECTED_TYPE` error of the argument `name`, `value`, which
should be of
+/// the type `expected`.
+fn not_expected_type(value: &Bound<'_, PyAny>, name: &str, expected: &str) ->
PyErr {
+ let type_name = value
+ .get_type()
+ .name()
+ .map(|name| name.to_string())
+ .unwrap_or_default();
+ crate::column::raise_pyspark(
+ value.py(),
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("expected_type", expected),
+ ("arg_name", name),
+ ("arg_type", &type_name),
+ ],
+ )
+}
+
+/// PySpark's `VALUE_NOT_POSITIVE` error of `numPartitions`, unless it is
positive.
+fn positive_partitions(py: Python<'_>, num_partitions: i64) -> PyResult<i32> {
+ if num_partitions > 0 {
+ return Ok(i32::try_from(num_partitions)?);
+ }
+ Err(crate::column::raise_pyspark(
+ py,
+ "PySparkValueError",
+ "VALUE_NOT_POSITIVE",
+ &[
+ ("arg_name", "numPartitions"),
+ ("arg_value", &num_partitions.to_string()),
+ ],
+ ))
+}
+
+/// The `subset` of `dropDuplicates` and `dropDuplicatesWithinWatermark`, as
PySpark's: None,
+/// or a list or tuple of column names.
+fn deduplicate_subset(subset: Option<&Bound<'_, PyAny>>) ->
PyResult<Option<Vec<String>>> {
+ let Some(subset) = subset else {
+ return Ok(None);
+ };
+ if !(subset.is_instance_of::<PyList>() ||
subset.is_instance_of::<PyTuple>()) {
+ return Err(not_expected_type(subset, "subset", "list or tuple"));
+ }
+ subset
+ .try_iter()?
+ .map(|column| {
+ let column = column?;
+ if column.is_instance_of::<pyo3::types::PyString>() {
+ column.extract()
+ } else {
+ Err(not_expected_type(&column, "subset", "str"))
+ }
+ })
+ .collect::<PyResult<Vec<String>>>()
+ .map(Some)
+}
+
/// Helper to convert arguments to a vector of Columns.
pub(crate) fn to_column_list(
mut _args: Vec<Bound<'_, PyAny>>,
@@ -187,9 +246,8 @@ pub(crate) fn to_column_list(
let _origin = crate::origin::with_origin("col");
cols.push(spark_connect::functions::col(&name));
} else {
- return Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(
- "Expected Column or column name (str)",
- ));
+ // As PySpark's `_to_cols`.
+ return Err(not_expected_type(&arg, "cols", "Column or str"));
}
}
}
@@ -300,9 +358,14 @@ impl PyDataFrame {
))
}
- /// Add or replace a column.
- fn withColumn(&self, colName: &str, col: &PyColumn) -> PyDataFrame {
- PyDataFrame::new(self.dataframe.with_column(colName,
col.column.clone()))
+ /// Add or replace a column, of a Column, as PySpark's
`DataFrame.withColumn`.
+ fn withColumn(&self, colName: &str, col: &Bound<'_, PyAny>) ->
PyResult<PyDataFrame> {
+ let Ok(column) = col.extract::<PyRef<'_, PyColumn>>() else {
+ return Err(not_expected_type(col, "col", "Column"));
+ };
+ Ok(PyDataFrame::new(
+ self.dataframe.with_column(colName, column.column.clone()),
+ ))
}
/// Rename a column.
@@ -310,16 +373,22 @@ impl PyDataFrame {
PyDataFrame::new(self.dataframe.with_column_renamed(existing, new))
}
- /// Add or replace multiple columns from a {name: Column} mapping.
+ /// Add or replace multiple columns from a {name: Column} mapping, as
PySpark's
+ /// `DataFrame.withColumns`, which asserts that it is given one, of
Columns.
#[pyo3(name = "withColumns", signature = (*colsMap))]
- fn with_columns(&self, colsMap: Vec<Bound<'_, PyDict>>) ->
PyResult<PyDataFrame> {
- let mut pairs = Vec::new();
- for d in &colsMap {
- for (k, v) in d.iter() {
- let name: String = k.extract()?;
- let pycol = v.extract::<PyRef<PyColumn>>()?;
- pairs.push((name, pycol.column.clone()));
- }
+ fn with_columns(&self, colsMap: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
+ let [cols_map] = colsMap.as_slice() else {
+ return Err(pyo3::exceptions::PyAssertionError::new_err(()));
+ };
+ let Ok(cols_map) = cols_map.cast::<PyDict>() else {
+ return Err(not_expected_type(cols_map, "colsMap", "dict"));
+ };
+ let mut pairs = Vec::with_capacity(cols_map.len());
+ for (name, column) in cols_map.iter() {
+ let Ok(column) = column.extract::<PyRef<'_, PyColumn>>() else {
+ return Err(pyo3::exceptions::PyAssertionError::new_err(()));
+ };
+ pairs.push((name.extract::<String>()?, column.column.clone()));
}
Ok(PyDataFrame::new(self.dataframe.with_columns(pairs)))
}
@@ -329,12 +398,68 @@ impl PyDataFrame {
/// accept any value and stringify it; the core parses a numeric string
back into a
/// long literal, matching reference semantics.
#[pyo3(signature = (name, *parameters))]
- fn hint(&self, name: &str, parameters: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
- let params: Vec<String> = parameters
- .iter()
- .map(|p| Ok(p.str()?.to_string()))
- .collect::<PyResult<_>>()?;
- Ok(PyDataFrame::new(self.dataframe.hint(name, params)))
+ fn hint(
+ &self,
+ py: Python<'_>,
+ name: &Bound<'_, PyAny>,
+ parameters: Vec<Bound<'_, PyAny>>,
+ ) -> PyResult<PyDataFrame> {
+ use pyo3::types::{PyFloat, PyString};
+ // As PySpark's: a single list is the parameters, each a str, a float,
an int, a
+ // Column or a list of str, float or int, sent as `lit` of it.
+ let parameters = match parameters.as_slice() {
+ [only] if only.is_instance_of::<PyList>() => {
+ only.try_iter()?.collect::<PyResult<Vec<_>>>()?
+ }
+ _ => parameters,
+ };
+ if !name.is_instance_of::<PyString>() {
+ return Err(not_expected_type(name, "name", "str"));
+ }
+ let primitive = |p: &Bound<'_, PyAny>| {
+ p.is_instance_of::<PyString>()
+ || p.is_instance_of::<PyFloat>()
+ || p.is_instance_of::<PyInt>()
+ };
+ let invalid = |item_type: &str| {
+ crate::column::raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "INVALID_ITEM_FOR_CONTAINER",
+ &[
+ ("arg_name", "parameters"),
+ (
+ "allowed_types",
+ "str, float, int, Column, list[str], list[float],
list[int]",
+ ),
+ ("item_type", item_type),
+ ],
+ )
+ };
+ let type_name =
+ |p: &Bound<'_, PyAny>| -> PyResult<String> {
Ok(p.get_type().name()?.to_string()) };
+ let mut expressions = Vec::with_capacity(parameters.len());
+ for p in ¶meters {
+ if p.is_instance_of::<PyList>() {
+ let items: Vec<Bound<'_, PyAny>> =
p.try_iter()?.collect::<PyResult<_>>()?;
+ if !items.iter().all(primitive) {
+ // As PySpark's, which names the type of the first item.
+ let first = items
+ .first()
+ .map(type_name)
+ .transpose()?
+ .unwrap_or_default();
+ return Err(invalid(&format!("list[{first}]")));
+ }
+ } else if !(primitive(p) || p.extract::<PyRef<'_,
PyColumn>>().is_ok()) {
+ return Err(invalid(&type_name(p)?));
+ }
+
expressions.push(crate::functions::to_column(p)?.expression().clone());
+ }
+ let name: String = name.extract()?;
+ Ok(PyDataFrame::new(
+ self.dataframe.hint_expressions(&name, expressions),
+ ))
}
/// Drop columns.
@@ -361,11 +486,12 @@ impl PyDataFrame {
/// Remove duplicate rows, optionally on specific columns.
#[pyo3(signature = (subset=None))]
- fn dropDuplicates(&self, _py: Python<'_>, subset: Option<Vec<String>>) ->
PyDataFrame {
+ fn dropDuplicates(&self, subset: Option<&Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
+ let subset = deduplicate_subset(subset)?;
let col_refs: Option<Vec<&str>> = subset
.as_ref()
.map(|names| names.iter().map(|s| s.as_str()).collect());
- PyDataFrame::new(self.dataframe.drop_duplicates(col_refs))
+ Ok(PyDataFrame::new(self.dataframe.drop_duplicates(col_refs)))
}
/// Sort rows. `ascending` (a bool or list of bools, via kwargs) mirrors
reference
@@ -484,13 +610,45 @@ impl PyDataFrame {
#[allow(non_snake_case)]
fn repartitionByRange(
&self,
- numPartitions: i32,
+ py: Python<'_>,
+ numPartitions: Bound<'_, PyAny>,
cols: Vec<Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
- let columns = to_column_list(cols)?;
- let exprs: Vec<_> = columns.iter().map(|c|
c.expression().clone()).collect();
+ // As PySpark's: a positive partition count and columns, or columns
alone, each sorted
+ // ascending unless it is a sort order.
+ let (num_partitions, cols) = if
numPartitions.is_instance_of::<PyInt>() {
+ let n = positive_partitions(py, numPartitions.extract()?)?;
+ if cols.is_empty() {
+ return Err(crate::column::raise_pyspark(
+ py,
+ "PySparkValueError",
+ "CANNOT_BE_EMPTY",
+ &[("item", "cols")],
+ ));
+ }
+ (n, cols)
+ } else if numPartitions.is_instance_of::<pyo3::types::PyString>()
+ || numPartitions.extract::<PyRef<'_, PyColumn>>().is_ok()
+ {
+ let mut all = vec![numPartitions];
+ all.extend(cols);
+ (0, all)
+ } else {
+ return Err(not_expected_type(
+ &numPartitions,
+ "numPartitions",
+ "Column, int or str",
+ ));
+ };
+ let exprs: Vec<_> = to_column_list(cols)?
+ .into_iter()
+ .map(|c| match c.expression().node() {
+ spark_connect::expression::ExpressionNode::SortOrder(_) =>
c.expression().clone(),
+ _ => c.asc().expression().clone(),
+ })
+ .collect();
Ok(PyDataFrame::new(
- self.dataframe.repartition_by_range(numPartitions, exprs),
+ self.dataframe.repartition_by_range(num_partitions, exprs),
))
}
@@ -500,14 +658,17 @@ impl PyDataFrame {
#[allow(non_snake_case)]
fn repartitionById(
&self,
- numPartitions: i32,
+ py: Python<'_>,
+ numPartitions: &Bound<'_, PyAny>,
partitionIdCol: &Bound<'_, PyAny>,
) -> PyResult<PyDataFrame> {
- if numPartitions <= 0 {
- return Err(pyo3::exceptions::PyValueError::new_err(format!(
- "numPartitions must be positive, got {numPartitions}"
- )));
+ // As PySpark's: an int that is not a bool, and positive.
+ if !numPartitions.is_instance_of::<PyInt>()
+ || numPartitions.is_instance_of::<pyo3::types::PyBool>()
+ {
+ return Err(not_expected_type(numPartitions, "numPartitions",
"int"));
}
+ let numPartitions = positive_partitions(py, numPartitions.extract()?)?;
// A `ColumnOrName`, as in pyspark: a string is a column name, not a
literal.
let col = if let Ok(c) = partitionIdCol.extract::<Bound<'_,
PyColumn>>() {
c.borrow().column.clone()
@@ -582,9 +743,10 @@ impl PyDataFrame {
numPartitions: Bound<'_, PyAny>,
cols: Vec<Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
- // First arg is an int -> a partition count; otherwise it's a
partition column
- // (str/Column) and the count defaults to the server's (passed as 0 =
unset).
- if let Ok(n) = numPartitions.extract::<i32>() {
+ // As PySpark's: a positive partition count, or a partition column
(str/Column), at
+ // the server's count (passed as 0 = unset).
+ if numPartitions.is_instance_of::<PyInt>() {
+ let n = positive_partitions(py, numPartitions.extract()?)?;
if cols.is_empty() {
return Ok(PyDataFrame::new(self.dataframe.repartition(n)));
}
@@ -592,34 +754,49 @@ impl PyDataFrame {
.iter()
.map(|c| c.expression().clone())
.collect();
- Ok(PyDataFrame::new(
+ return Ok(PyDataFrame::new(
self.dataframe.repartition_by_expressions(n, exprs),
- ))
- } else {
- // Column-first form: the first arg is itself a partition column.
- let mut all = vec![numPartitions];
- all.extend(cols);
- let _ = py;
- let exprs: Vec<_> = to_column_list(all)?
- .iter()
- .map(|c| c.expression().clone())
- .collect();
- Ok(PyDataFrame::new(
- self.dataframe.repartition_by_expressions(0, exprs),
- ))
+ ));
+ }
+ if !(numPartitions.is_instance_of::<pyo3::types::PyString>()
+ || numPartitions.extract::<PyRef<'_, PyColumn>>().is_ok())
+ {
+ return Err(not_expected_type(
+ &numPartitions,
+ "numPartitions",
+ "Column or str",
+ ));
}
+ let mut all = vec![numPartitions];
+ all.extend(cols);
+ let exprs: Vec<_> = to_column_list(all)?
+ .iter()
+ .map(|c| c.expression().clone())
+ .collect();
+ Ok(PyDataFrame::new(
+ self.dataframe.repartition_by_expressions(0, exprs),
+ ))
}
/// Coalesce.
- fn coalesce(&self, numPartitions: i32) -> PyDataFrame {
- PyDataFrame::new(self.dataframe.coalesce(numPartitions))
+ fn coalesce(&self, py: Python<'_>, numPartitions: i64) ->
PyResult<PyDataFrame> {
+ let num_partitions = positive_partitions(py, numPartitions)?;
+ Ok(PyDataFrame::new(self.dataframe.coalesce(num_partitions)))
}
/// Convert to DataFrame with new column names.
#[pyo3(signature = (*cols))]
- fn toDF(&self, _py: Python<'_>, cols: Vec<String>) -> PyDataFrame {
- let name_refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
- PyDataFrame::new(self.dataframe.to_df(name_refs))
+ fn toDF(&self, cols: Vec<Bound<'_, PyAny>>) -> PyResult<PyDataFrame> {
+ // As PySpark's, of strings.
+ let mut names = Vec::with_capacity(cols.len());
+ for col in &cols {
+ if !col.is_instance_of::<pyo3::types::PyString>() {
+ return Err(not_expected_type(col, "cols", "list[str]"));
+ }
+ names.push(col.extract::<String>()?);
+ }
+ let name_refs: Vec<&str> = names.iter().map(|s| s.as_str()).collect();
+ Ok(PyDataFrame::new(self.dataframe.to_df(name_refs)))
}
/// Alias this DataFrame.
@@ -672,11 +849,17 @@ impl PyDataFrame {
/// Drop duplicates within the event-time watermark.
#[pyo3(name = "dropDuplicatesWithinWatermark", signature = (subset=None))]
- fn drop_duplicates_within_watermark(&self, subset: Option<Vec<String>>) ->
PyDataFrame {
+ fn drop_duplicates_within_watermark(
+ &self,
+ subset: Option<&Bound<'_, PyAny>>,
+ ) -> PyResult<PyDataFrame> {
+ let subset = deduplicate_subset(subset)?;
let refs: Option<Vec<&str>> = subset
.as_ref()
.map(|names| names.iter().map(|s| s.as_str()).collect());
- PyDataFrame::new(self.dataframe.drop_duplicates_within_watermark(refs))
+ Ok(PyDataFrame::new(
+ self.dataframe.drop_duplicates_within_watermark(refs),
+ ))
}
/// `DataFrame.toJSON`, as PySpark Connect's: a DataFrame of each row as a
JSON object
@@ -721,14 +904,18 @@ impl PyDataFrame {
/// Rename multiple columns from an {old: new} mapping.
#[pyo3(name = "withColumnsRenamed")]
- fn with_columns_renamed(
- &self,
- colsMap: std::collections::HashMap<String, String>,
- ) -> PyDataFrame {
- PyDataFrame::new(
- self.dataframe
- .with_columns_renamed(colsMap.into_iter().collect()),
- )
+ fn with_columns_renamed(&self, colsMap: &Bound<'_, PyAny>) ->
PyResult<PyDataFrame> {
+ // As PySpark's, of a dict, whose renames it sends in order.
+ let Ok(cols_map) = colsMap.cast::<PyDict>() else {
+ return Err(not_expected_type(colsMap, "colsMap", "dict"));
+ };
+ let mut renames = Vec::with_capacity(cols_map.len());
+ for (existing, new) in cols_map.iter() {
+ renames.push((existing.extract::<String>()?,
new.extract::<String>()?));
+ }
+ Ok(PyDataFrame::new(
+ self.dataframe.with_columns_renamed(renames),
+ ))
}
/// The columns of this DataFrame whose names match a regex, as a Column.
Mirrors
@@ -1140,11 +1327,8 @@ impl PyDataFrame {
/// Deprecated alias for dropDuplicates.
#[pyo3(name = "drop_duplicates", signature = (subset=None))]
- fn drop_duplicates(&self, subset: Option<Vec<String>>) -> PyDataFrame {
- let refs: Option<Vec<&str>> = subset
- .as_ref()
- .map(|s| s.iter().map(|x| x.as_str()).collect());
- PyDataFrame::new(self.dataframe.drop_duplicates(refs))
+ fn drop_duplicates(&self, subset: Option<&Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
+ self.dropDuplicates(subset)
}
/// Alias for groupBy.
diff --git a/crates/spark-connect/src/dataframe.rs
b/crates/spark-connect/src/dataframe.rs
index 68d048f..f1e9120 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -631,10 +631,20 @@ impl DataFrame {
name: &str,
parameters: impl IntoIterator<Item = S>,
) -> DataFrame {
+ let parameters = parameters
+ .into_iter()
+ .map(|p| crate::plan::hint_parameter(&p.into()))
+ .collect();
+ self.hint_expressions(name, parameters)
+ }
+
+ /// Specify a hint of parameters given as expressions, mirroring
`pyspark.sql.DataFrame.hint`,
+ /// which sends `lit` of each parameter: a literal, or a Column.
+ pub fn hint_expressions(&self, name: &str, parameters: Vec<Expression>) ->
DataFrame {
let plan = LogicalPlan::Hint {
input: Arc::clone(&self.plan),
name: name.to_string(),
- parameters: parameters.into_iter().map(Into::into).collect(),
+ parameters,
};
self.with_cached_schema(plan)
}
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index e9ef63d..36be8a3 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -177,7 +177,8 @@ pub enum LogicalPlan {
Hint {
input: Arc<LogicalPlan>,
name: String,
- parameters: Vec<String>,
+ /// The parameters, literal expressions or columns, as PySpark's `lit`
of them.
+ parameters: Vec<Expression>,
},
/// Unpivot: `df.unpivot(...)`.
Unpivot {
@@ -1020,19 +1021,8 @@ impl LogicalPlan {
let mut hint = proto::Hint::default();
hint.input = Some(Box::new(relation_of(input, plan_ids)));
hint.name = name.clone();
- // Hint parameters are literal Expressions (e.g. `REPARTITION
10`).
- // An integer-looking parameter becomes an Integer (int32)
literal -
- // matching reference pyspark's `lit(int)` - otherwise a
String literal.
- // (A Long literal is rejected as a partitionNum by e.g. the
REBALANCE
- // hint, which expects an integral int.)
- for p in parameters {
- let lit = if let Ok(n) = p.parse::<i32>() {
- crate::expression::LiteralExpression::int(n)
- } else {
- crate::expression::LiteralExpression::string(p.clone())
- };
- hint.parameters.push(lit.to_proto());
- }
+ hint.parameters
+ .extend(parameters.iter().map(|p| p.to_proto()));
relation.rel_type =
Some(proto::relation::RelType::Hint(Box::new(hint)));
}
@@ -1879,7 +1869,10 @@ pub fn to_schema(input: impl Into<Arc<LogicalPlan>>,
schema: DataType) -> Logica
}
}
-/// Create a Hint plan.
+/// Create a Hint plan, of parameters given as strings: an integer-looking
parameter is an
+/// Integer (int32) literal, matching PySpark's `lit(int)`, otherwise a String
literal. (A
+/// Long literal is rejected as a partitionNum by e.g. the REBALANCE hint,
which expects an
+/// integral int.)
pub fn hint(
input: impl Into<Arc<LogicalPlan>>,
name: impl Into<String>,
@@ -1888,10 +1881,19 @@ pub fn hint(
LogicalPlan::Hint {
input: input.into(),
name: name.into(),
- parameters,
+ parameters: parameters.iter().map(|p| hint_parameter(p)).collect(),
}
}
+/// The literal of a hint parameter given as a string: an Integer for an
integer-looking one,
+/// otherwise the String.
+pub(crate) fn hint_parameter(parameter: &str) -> Expression {
+ Expression::literal(match parameter.parse::<i32>() {
+ Ok(n) => LiteralExpression::int(n),
+ Err(_) => LiteralExpression::string(parameter),
+ })
+}
+
/// Create an Unpivot plan.
pub fn unpivot(
input: impl Into<Arc<LogicalPlan>>,
@@ -2106,11 +2108,11 @@ mod argfix_tests {
#[test]
fn hint_carries_parameters() {
- let p = LogicalPlan::Hint {
- input: Arc::new(base()),
- name: "REPARTITION".to_string(),
- parameters: vec!["10".to_string(), "name".to_string()],
- };
+ let p = hint(
+ base(),
+ "REPARTITION",
+ vec!["10".to_string(), "name".to_string()],
+ );
match rel_type(p) {
proto::relation::RelType::Hint(h) => {
assert_eq!(h.parameters.len(), 2, "parameters must be
forwarded");
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 9852f80..75cd9f9 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -7997,6 +7997,65 @@ def
test_writer_v2_setters_mutate_and_return_self(offline_spark):
assert w.clusterBy("id") is w
+def test_dataframe_methods_raise_the_argument_errors_of_pyspark():
+ # As PySpark's DataFrame methods, which check their arguments on the
client.
+ from pyspark.errors import PySparkTypeError, PySparkValueError
+
+ df = _offline_session().range(1)
+
+ def not_expected(arg_name, expected_type, arg_type):
+ return (PySparkTypeError, "NOT_EXPECTED_TYPE",
+ {"arg_name": arg_name, "expected_type": expected_type,
"arg_type": arg_type})
+
+ def not_positive(value):
+ return (PySparkValueError, "VALUE_NOT_POSITIVE",
+ {"arg_name": "numPartitions", "arg_value": value})
+
+ allowed_hint_types = "str, float, int, Column, list[str], list[float],
list[int]"
+ for call, (error_type, condition, parameters) in (
+ (lambda: df.select(1), not_expected("cols", "Column or str", "int")),
+ (lambda: df.select(["id", 1.5]), not_expected("cols", "Column or str",
"float")),
+ (lambda: df.toDF(1), not_expected("cols", "list[str]", "int")),
+ (lambda: df.withColumn("a", 1), not_expected("col", "Column", "int")),
+ (lambda: df.withColumns([("a", 1)]), not_expected("colsMap", "dict",
"list")),
+ (lambda: df.withColumnsRenamed([("id", "a")]), not_expected("colsMap",
"dict", "list")),
+ (lambda: df.hint(1), not_expected("name", "str", "int")),
+ (lambda: df.hint("x", {"a": 1}),
+ (PySparkTypeError, "INVALID_ITEM_FOR_CONTAINER",
+ {"arg_name": "parameters", "allowed_types": allowed_hint_types,
"item_type": "dict"})),
+ (lambda: df.hint("x", [None, 1]),
+ (PySparkTypeError, "INVALID_ITEM_FOR_CONTAINER",
+ {"arg_name": "parameters", "allowed_types": allowed_hint_types,
+ "item_type": "NoneType"})),
+ (lambda: df.hint("x", [1], [2, None]),
+ (PySparkTypeError, "INVALID_ITEM_FOR_CONTAINER",
+ {"arg_name": "parameters", "allowed_types": allowed_hint_types,
+ "item_type": "list[int]"})),
+ (lambda: df.dropDuplicates("id"), not_expected("subset", "list or
tuple", "str")),
+ (lambda: df.drop_duplicates(["id", 1]), not_expected("subset", "str",
"int")),
+ (lambda: df.dropDuplicatesWithinWatermark("id"),
+ not_expected("subset", "list or tuple", "str")),
+ (lambda: df.repartition(0), not_positive("0")),
+ (lambda: df.repartition(1.5), not_expected("numPartitions", "Column or
str", "float")),
+ (lambda: df.repartition("id", 2), not_expected("cols", "Column or
str", "int")),
+ (lambda: df.repartitionByRange(-1, "id"), not_positive("-1")),
+ (lambda: df.repartitionByRange(2), (PySparkValueError,
"CANNOT_BE_EMPTY",
+ {"item": "cols"})),
+ (lambda: df.repartitionByRange(1.5, "id"),
+ not_expected("numPartitions", "Column, int or str", "float")),
+ (lambda: df.coalesce(0), not_positive("0")),
+ (lambda: df.repartitionById(True, "id"), not_expected("numPartitions",
"int", "bool")),
+ (lambda: df.repartitionById(0, "id"), not_positive("0")),
+ ):
+ with pytest.raises(error_type) as error:
+ call()
+ assert error.value.getCondition() == condition
+ assert error.value.getMessageParameters() == parameters
+ for call in (lambda: df.withColumns(), lambda: df.withColumns({"a": 1})):
+ with pytest.raises(AssertionError):
+ call()
+
+
def test_stream_writer_setters_mutate_and_return_self(offline_spark):
w = offline_spark.readStream.format("rate").load().writeStream
assert w.outputMode("append") is w
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]