This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 10ffe91 [SPARK-61015][CONNECT][PYTHON] Take the arguments of unpivot,
melt and replace as PySpark does
10ffe91 is described below
commit 10ffe915a6c0c8917ddea52679fe555e72f4e5ff
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 01:44:13 2026 +0900
[SPARK-61015][CONNECT][PYTHON] Take the arguments of unpivot, melt and
replace as PySpark does
### What changes were proposed in this pull request?
[SPARK-61015](https://issues.apache.org/jira/browse/SPARK-61015):
`unpivot`, `melt` and `replace` now take their arguments as PySpark does.
- `unpivot` and `melt`, which is the same function, take `ids` and `values`
that are a column name, a Column, or a list or tuple of them. All four
arguments are required.
- `replace` (and `na.replace`) takes a `subset` that is a column name, or a
list or tuple of them. It sends each value as a literal of its type, as PySpark
Connect builds the `NAReplace` relation: a bool, a double (an int as well), a
string, or `None` as a null.
- `replace` raises `ARGUMENT_REQUIRED`, `NOT_EXPECTED_TYPE`,
`LENGTH_SHOULD_BE_THE_SAME` and `MIXED_TYPE_REPLACEMENT` where PySpark does,
and warns that the `value` of a dict `to_replace` is ignored.
In the Rust API, `DataFrame::replace_literals` and
`NaFunctions::replace_literals` take the replacements as literals.
`DataFrame::replace` of strings is unchanged.
### Why are the changes needed?
They took lists only: `df.unpivot("id", ["v"], "var", "val")` and
`df.replace("a", "z", "k")` raised a `TypeError`. The values of `replace` were
sent as strings, and one that parses as a number as a double. So:
- `df.replace("1", "one")` and `df.replace({1: "x"})` failed on the server
with `CAST_INVALID_INPUT`.
- `df.replace(True, False)` replaced nothing.
- `df.replace({"a": None})` replaced `"a"` with the string `'None'`.
- `df.replace("a", None, ["k"])` replaced nothing.
### Does this PR introduce _any_ user-facing change?
Yes. `unpivot`, `melt` and `replace` take the arguments of PySpark 4.3.0,
replace as it does, and raise its errors.
### How was this patch tested?
- A new live test replaces strings, numbers, bools and nulls over a subset
of a name, a list or a tuple, and unpivots and melts by names and Columns. It
passes with official PySpark 4.3.0, and fails without this change.
- A new offline test checks the errors of the arguments, and the warning.
It passes with official PySpark 4.3.0, and fails without this change.
- New lib tests check the literals of the replacements.
- A probe of 47 calls matches official PySpark 4.3.0. The exceptions: two
messages of `TypeError` that name the function differently.
- The lib tests and every integration test, e2e included, against a local
Spark 4.3.0 server (673 passed, and 129 of the transport crate), the build of
all tests with `wasm-udf`, the offline Python suites (1625 passed, 8 skipped),
and the live drop-in and Arrow suites (95 passed) and live cases (10 passed).
Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #219 from HyukjinKwon/SPARK-61015-fix.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 350 +++++++++++++++++++++++----------
crates/spark-connect/src/dataframe.rs | 72 ++++++-
crates/spark-connect/src/expression.rs | 8 +
crates/spark-connect/src/group.rs | 13 ++
crates/spark-connect/src/plan.rs | 105 +++++++---
python/tests/test_dropin_e2e.py | 29 +++
python/tests/test_dropin_offline.py | 48 +++++
7 files changed, 495 insertions(+), 130 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index a0f59d0..a15eb44 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -988,37 +988,16 @@ impl PyDataFrame {
))
}
- /// Replace values. `to_replace` may be a {old: new} dict, or a value/list
with `value`.
- #[pyo3(signature = (to_replace, value=None, subset=None))]
+ /// Replace values, as PySpark's `DataFrame.replace`: `to_replace` is a
{old: new} dict, or
+ /// a value or a list or tuple of them with `value`, a value or a list or
tuple of them.
+ #[pyo3(signature = (to_replace, value=ReplaceValue::Omitted, subset=None))]
fn replace(
&self,
to_replace: &Bound<'_, PyAny>,
- value: Option<&Bound<'_, PyAny>>,
- subset: Option<Vec<String>>,
+ value: ReplaceValue,
+ subset: Option<&Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
- let mut pairs: Vec<(String, String)> = Vec::new();
- if let Ok(d) = to_replace.cast::<PyDict>() {
- for (k, v) in d.iter() {
- pairs.push((k.str()?.to_string(), v.str()?.to_string()));
- }
- } else if let (Ok(olds), Some(val)) =
(to_replace.extract::<Vec<Bound<'_, PyAny>>>(), value)
- {
- if let Ok(news) = val.extract::<Vec<Bound<'_, PyAny>>>() {
- for (o, n) in olds.iter().zip(news.iter()) {
- pairs.push((o.str()?.to_string(), n.str()?.to_string()));
- }
- } else {
- for o in olds {
- pairs.push((o.str()?.to_string(), val.str()?.to_string()));
- }
- }
- } else if let Some(val) = value {
- pairs.push((to_replace.str()?.to_string(),
val.str()?.to_string()));
- }
- let subset_refs: Option<Vec<&str>> = subset
- .as_ref()
- .map(|s| s.iter().map(|x| x.as_str()).collect());
- Ok(PyDataFrame::new(self.dataframe.replace(pairs, subset_refs)))
+ replace_values(&self.dataframe, to_replace, value, subset)
}
/// Reconcile to a target schema (StructType or DDL string).
@@ -1164,48 +1143,42 @@ impl PyDataFrame {
Ok(func.call(tuple, kwargs)?.unbind())
}
- /// Unpivot (wide-to-long). `values=None` unpivots all non-id columns.
- #[pyo3(signature = (ids, values=None, variableColumnName="variable",
valueColumnName="value"))]
+ /// Unpivot (wide-to-long), as PySpark's `DataFrame.unpivot`: `ids` and
`values` are each a
+ /// column or a column name, or a list or tuple of them, and `values=None`
unpivots all
+ /// non-id columns.
+ #[pyo3(signature = (ids, values, variableColumnName, valueColumnName))]
fn unpivot(
&self,
- ids: Vec<String>,
- values: Option<Vec<String>>,
+ ids: &Bound<'_, PyAny>,
+ values: Option<&Bound<'_, PyAny>>,
variableColumnName: &str,
valueColumnName: &str,
- ) -> PyDataFrame {
- let id_refs: Vec<&str> = ids.iter().map(|s| s.as_str()).collect();
- let owned = values;
- let val_refs = owned
- .as_ref()
- .map(|v| v.iter().map(|s| s.as_str()).collect());
- PyDataFrame::new(self.dataframe.melt(
- id_refs,
- val_refs,
+ ) -> PyResult<PyDataFrame> {
+ if ids.is_none() {
+ return Err(pyo3::exceptions::PyAssertionError::new_err(
+ "ids must not be None",
+ ));
+ }
+ let ids = unpivot_columns(ids)?;
+ let values = values.map(unpivot_columns).transpose()?;
+ Ok(PyDataFrame::new(self.dataframe.unpivot(
+ ids,
+ values,
variableColumnName,
valueColumnName,
- ))
+ )))
}
- /// Alias for unpivot.
- #[pyo3(signature = (ids, values=None, variableColumnName="variable",
valueColumnName="value"))]
+ /// The same function as `unpivot`, as PySpark's `DataFrame.melt`.
+ #[pyo3(signature = (ids, values, variableColumnName, valueColumnName))]
fn melt(
&self,
- ids: Vec<String>,
- values: Option<Vec<String>>,
+ ids: &Bound<'_, PyAny>,
+ values: Option<&Bound<'_, PyAny>>,
variableColumnName: &str,
valueColumnName: &str,
- ) -> PyDataFrame {
- let id_refs: Vec<&str> = ids.iter().map(|s| s.as_str()).collect();
- let owned = values;
- let val_refs = owned
- .as_ref()
- .map(|v| v.iter().map(|s| s.as_str()).collect());
- PyDataFrame::new(self.dataframe.melt(
- id_refs,
- val_refs,
- variableColumnName,
- valueColumnName,
- ))
+ ) -> PyResult<PyDataFrame> {
+ self.unpivot(ids, values, variableColumnName, valueColumnName)
}
/// Persist the DataFrame. A storage-level argument is accepted for API
parity;
@@ -1819,22 +1792,16 @@ impl PyDataFrameNaFunctions {
Ok(PyDataFrame::new(self.df.fillna_value(val, refs)))
}
- /// Replace values, mirroring `DataFrameNaFunctions.replace`:
- /// `replace([old...], [new...], subset)`, `replace({old: new}, subset)`,
or
- /// `replace(old, new, subset)`.
- #[pyo3(signature = (to_replace, value=None, subset=None))]
+ /// Replace values, as PySpark's `DataFrameNaFunctions.replace`, which is
+ /// `DataFrame.replace`.
+ #[pyo3(signature = (to_replace, value=ReplaceValue::Omitted, subset=None))]
fn replace(
&self,
to_replace: &Bound<'_, PyAny>,
- value: Option<&Bound<'_, PyAny>>,
+ value: ReplaceValue,
subset: Option<&Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
- let pairs = build_replacement_pairs(to_replace, value)?;
- let owned = to_subset(subset)?;
- let subset_refs: Option<Vec<&str>> = owned
- .as_ref()
- .map(|v| v.iter().map(String::as_str).collect());
- Ok(PyDataFrame::new(self.inner.replace(pairs, subset_refs)))
+ replace_values(&self.df, to_replace, value, subset)
}
}
@@ -1853,44 +1820,223 @@ fn to_subset(subset: Option<&Bound<'_, PyAny>>) ->
PyResult<Option<Vec<String>>>
}
}
-/// Build (old, new) string replacement pairs from the reference `replace`
forms.
-fn build_replacement_pairs(
+/// The `value` argument of `DataFrame.replace`, where PySpark tells an
omitted value apart
+/// from None: `replace(to_replace)` requires a dict `to_replace`, while
+/// `replace(to_replace, None)` replaces with nulls.
+pub enum ReplaceValue {
+ Omitted,
+ Given(Py<PyAny>),
+}
+
+impl<'a, 'py> FromPyObject<'a, 'py> for ReplaceValue {
+ type Error = PyErr;
+
+ fn extract(obj: Borrowed<'a, 'py, PyAny>) -> PyResult<Self> {
+ Ok(ReplaceValue::Given(obj.to_owned().unbind()))
+ }
+}
+
+/// `df.replace(to_replace, value, subset)`, checked and built as PySpark's
+/// `DataFrame.replace`: each value of `to_replace` (a {old: new} dict, or a
value or a list or
+/// tuple of them) is replaced by `value` (a value or a list or tuple of them,
or None), all
+/// bools, numbers or strings, sent as literals with an int as a float, in the
columns of
+/// `subset` (a column name, or a list or tuple of them).
+fn replace_values(
+ dataframe: &DataFrame,
to_replace: &Bound<'_, PyAny>,
- value: Option<&Bound<'_, PyAny>>,
-) -> PyResult<Vec<(String, String)>> {
- use pyo3::exceptions::PyValueError;
- use pyo3::types::{PyDict, PyList};
- if let Ok(d) = to_replace.cast::<PyDict>() {
- let mut pairs = Vec::with_capacity(d.len());
- for (k, v) in d.iter() {
- pairs.push((k.str()?.to_string(), v.str()?.to_string()));
+ value: ReplaceValue,
+ subset: Option<&Bound<'_, PyAny>>,
+) -> PyResult<PyDataFrame> {
+ use pyo3::types::{PyBool, PyFloat, PyString};
+ use spark_connect::expression::LiteralExpression;
+
+ let py = to_replace.py();
+ let raise = |class_name: &str, error_class: &str, params: &[(&str, &str)]|
{
+ crate::column::raise_pyspark(py, class_name, error_class, params)
+ };
+ let type_name = |v: &Bound<'_, PyAny>| -> String {
+ v.get_type()
+ .name()
+ .map(|name| name.to_string())
+ .unwrap_or_default()
+ };
+ let is_sequence =
+ |v: &Bound<'_, PyAny>| v.is_instance_of::<PyList>() ||
v.is_instance_of::<PyTuple>();
+ // A bool, a float, an int or a str: a bool is an int.
+ let is_scalar = |v: &Bound<'_, PyAny>| {
+ v.is_instance_of::<PyFloat>()
+ || v.is_instance_of::<PyInt>()
+ || v.is_instance_of::<PyString>()
+ };
+ let is_dict = to_replace.is_instance_of::<PyDict>();
+ let value = match value {
+ ReplaceValue::Given(value) => value.into_bound(py),
+ ReplaceValue::Omitted if is_dict => py.None().into_bound(py),
+ ReplaceValue::Omitted => {
+ return Err(raise(
+ "PySparkTypeError",
+ "ARGUMENT_REQUIRED",
+ &[("arg_name", "value"), ("condition", "`to_replace` is
dict")],
+ ))
}
- return Ok(pairs);
- }
- if let Ok(olds) = to_replace.cast::<PyList>() {
- let val = value.ok_or_else(|| {
- PyValueError::new_err("replace with a list to_replace requires a
value list")
- })?;
- let news = val
- .cast::<PyList>()
- .map_err(|_| PyValueError::new_err("value must be a list when
to_replace is a list"))?;
- if olds.len() != news.len() {
- return Err(PyValueError::new_err(
- "to_replace and value lists must be the same length",
- ));
+ };
+
+ if !(is_scalar(to_replace) || is_sequence(to_replace) || is_dict) {
+ return Err(raise(
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("expected_type", "bool, dict, float, int, str or tuple"),
+ ("arg_name", "to_replace"),
+ ("arg_type", &type_name(to_replace)),
+ ],
+ ));
+ }
+ if !(is_scalar(&value) || is_sequence(&value) || value.is_none() ||
is_dict) {
+ return Err(raise(
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ (
+ "expected_type",
+ "bool, float, int, list, None, str or tuple",
+ ),
+ ("arg_name", "value"),
+ ("arg_type", &type_name(&value)),
+ ],
+ ));
+ }
+ if is_sequence(to_replace) && is_sequence(&value) && to_replace.len()? !=
value.len()? {
+ return Err(raise(
+ "PySparkValueError",
+ "LENGTH_SHOULD_BE_THE_SAME",
+ &[
+ ("arg1", "to_replace"),
+ ("arg2", "value"),
+ ("arg1_length", &to_replace.len()?.to_string()),
+ ("arg2_length", &value.len()?.to_string()),
+ ],
+ ));
+ }
+ let subset: Option<Vec<String>> = match subset {
+ None => None,
+ Some(subset) if subset.is_instance_of::<PyString>() =>
Some(vec![subset.extract()?]),
+ Some(subset) if is_sequence(subset) => Some(subset.extract()?),
+ Some(subset) => {
+ return Err(raise(
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("arg_name", "subset"),
+ ("expected_type", "list, str or tuple"),
+ ("arg_type", &type_name(subset)),
+ ],
+ ))
}
- let mut pairs = Vec::with_capacity(olds.len());
- for (a, b) in olds.iter().zip(news.iter()) {
- pairs.push((a.str()?.to_string(), b.str()?.to_string()));
+ };
+
+ // The replacements as a dict, as PySpark builds it, so that equal values
are one key.
+ let replacements = if is_dict {
+ if !value.is_none() {
+ py.import("warnings")?.call_method1(
+ "warn",
+ ("to_replace is a dict and value is not None. value will be
ignored.",),
+ )?;
+ }
+ to_replace.cast::<PyDict>()?.clone()
+ } else {
+ let olds: Vec<Bound<'_, PyAny>> = if is_scalar(to_replace) {
+ vec![to_replace.clone()]
+ } else {
+ to_replace.try_iter()?.collect::<PyResult<_>>()?
+ };
+ let news: Vec<Bound<'_, PyAny>> = if is_scalar(&value) ||
value.is_none() {
+ vec![value.clone(); olds.len()]
+ } else {
+ value.try_iter()?.collect::<PyResult<_>>()?
+ };
+ let replacements = PyDict::new(py);
+ for (old, new) in olds.iter().zip(news.iter()) {
+ replacements.set_item(old, new)?;
}
- return Ok(pairs);
- }
- let val =
- value.ok_or_else(|| PyValueError::new_err("replace with a scalar
requires a value"))?;
- Ok(vec![(
- to_replace.str()?.to_string(),
- val.str()?.to_string(),
- )])
+ replacements
+ };
+
+ // All of the values to replace, and all of their replacements but None,
are bools,
+ // numbers or strings.
+ let olds: Vec<Bound<'_, PyAny>> = replacements.keys().iter().collect();
+ let news: Vec<Bound<'_, PyAny>> = replacements
+ .values()
+ .iter()
+ .filter(|new| !new.is_none())
+ .collect();
+ let all_of = |is_type: &dyn Fn(&Bound<'_, PyAny>) -> bool| {
+ olds.iter().all(is_type) && news.iter().all(is_type)
+ };
+ if !(all_of(&|v| v.is_instance_of::<PyBool>())
+ || all_of(&|v| v.is_instance_of::<PyString>())
+ || all_of(&|v| v.is_instance_of::<PyFloat>() ||
v.is_instance_of::<PyInt>()))
+ {
+ return Err(raise("PySparkValueError", "MIXED_TYPE_REPLACEMENT", &[]));
+ }
+
+ let literal = |v: &Bound<'_, PyAny>| -> PyResult<LiteralExpression> {
+ Ok(if v.is_none() {
+ LiteralExpression::null(spark_connect::types::DataType::Null)
+ } else if v.is_instance_of::<PyBool>() {
+ LiteralExpression::boolean(v.extract()?)
+ } else if v.is_instance_of::<PyString>() {
+ LiteralExpression::string(v.extract::<String>()?)
+ } else {
+ // As `float(v)`, for an int as well.
+ LiteralExpression::double(v.extract()?)
+ })
+ };
+ let replacements = replacements
+ .iter()
+ .map(|(old, new)| Ok((literal(&old)?, literal(&new)?)))
+ .collect::<PyResult<Vec<_>>>()?;
+ let subset: Option<Vec<&str>> = subset
+ .as_ref()
+ .map(|subset| subset.iter().map(String::as_str).collect());
+ Ok(PyDataFrame::new(
+ dataframe.replace_literals(replacements, subset),
+ ))
+}
+
+/// The columns of the `ids` or `values` of `df.unpivot`, as PySpark's: a list
or tuple of
+/// columns or column names, or one of them.
+fn unpivot_columns(cols: &Bound<'_, PyAny>) ->
PyResult<Vec<spark_connect::column::Column>> {
+ if cols.is_instance_of::<PyList>() || cols.is_instance_of::<PyTuple>() {
+ cols.try_iter()?.map(|col| column_or_name(&col?)).collect()
+ } else {
+ Ok(vec![column_or_name(cols)?])
+ }
+}
+
+/// A column, or a column by its name, as PySpark's `_to_col`.
+fn column_or_name(col: &Bound<'_, PyAny>) ->
PyResult<spark_connect::column::Column> {
+ if let Ok(column) = col.extract::<PyRef<'_, PyColumn>>() {
+ return Ok(column.column.clone());
+ }
+ if col.is_instance_of::<pyo3::types::PyString>() {
+ return Ok(spark_connect::functions::col(&col.extract::<String>()?));
+ }
+ let type_name = col
+ .get_type()
+ .name()
+ .map(|name| name.to_string())
+ .unwrap_or_default();
+ Err(crate::column::raise_pyspark(
+ col.py(),
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("expected_type", "Column or str"),
+ ("arg_name", "col"),
+ ("arg_type", &type_name),
+ ],
+ ))
}
/// Python wrapper for `MergeIntoWriter` (`df.mergeInto(...)`).
diff --git a/crates/spark-connect/src/dataframe.rs
b/crates/spark-connect/src/dataframe.rs
index ebd9385..98c79e3 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1463,18 +1463,43 @@ impl DataFrame {
DataFrame::new(self.session.clone(), plan)
}
- /// Replace values.
+ /// Replace values, each given as a string: a numeric-looking string is a
double, otherwise
+ /// a string. [`DataFrame::replace_literals`] takes the values as literals.
pub fn replace(
&self,
to_replace: Vec<(String, String)>,
subset: Option<Vec<&str>>,
+ ) -> DataFrame {
+ let replacements = to_replace
+ .iter()
+ .map(|(old, new)| {
+ (
+ crate::plan::numeric_or_string_literal(old),
+ crate::plan::numeric_or_string_literal(new),
+ )
+ })
+ .collect();
+ self.replace_literals(replacements, subset)
+ }
+
+ /// Replace values, mirroring `pyspark.sql.DataFrame.replace`: each pair
is a literal and
+ /// the literal that replaces it, which may be a null, in the columns of
`subset`, or else
+ /// in every column of the literal's type. The server replaces a boolean,
a double or a
+ /// string.
+ pub fn replace_literals(
+ &self,
+ replacements: Vec<(
+ crate::expression::LiteralExpression,
+ crate::expression::LiteralExpression,
+ )>,
+ subset: Option<Vec<&str>>,
) -> DataFrame {
let columns = subset
.map(|v| v.iter().map(|s| s.to_string()).collect())
.unwrap_or_default();
let plan = LogicalPlan::NAReplace {
input: Arc::clone(&self.plan),
- replacements: to_replace,
+ replacements,
columns,
};
DataFrame::new(self.session.clone(), plan)
@@ -3801,6 +3826,49 @@ mod plan_construction_tests {
}
}
+ #[test]
+ fn replace_literals_plan() {
+ use crate::expression::LiteralExpression;
+ let spark = session();
+ let df = spark.range(3).unwrap();
+ let replacements = vec![(
+ LiteralExpression::string("a"),
+ LiteralExpression::null(DataType::Null),
+ )];
+ for result in [
+ df.replace_literals(replacements.clone(), Some(vec!["k"])),
+ df.na()
+ .replace_literals(replacements.clone(), Some(vec!["k"])),
+ ] {
+ match result.plan.as_ref() {
+ LogicalPlan::NAReplace {
+ replacements: planned,
+ columns,
+ ..
+ } => {
+ assert_eq!(planned, &replacements);
+ assert_eq!(columns, &vec!["k".to_string()]);
+ }
+ _ => panic!("expected NAReplace plan"),
+ }
+ }
+ // A string replacement that parses as a number is a double, as before.
+ match df
+ .replace(vec![("1".to_string(), "a".to_string())], None)
+ .plan
+ .as_ref()
+ {
+ LogicalPlan::NAReplace { replacements, .. } => assert_eq!(
+ replacements,
+ &vec![(
+ LiteralExpression::double(1.0),
+ LiteralExpression::string("a")
+ )]
+ ),
+ _ => panic!("expected NAReplace plan"),
+ }
+ }
+
#[test]
fn replace_plan() {
let spark = session();
diff --git a/crates/spark-connect/src/expression.rs
b/crates/spark-connect/src/expression.rs
index e5f6365..b80fbfc 100644
--- a/crates/spark-connect/src/expression.rs
+++ b/crates/spark-connect/src/expression.rs
@@ -568,6 +568,14 @@ impl LiteralExpression {
expr
}
+ /// The literal as a proto `Literal`, as `to_proto` sets it in its
`Expression`.
+ pub fn to_literal_proto(&self) -> proto::expression::Literal {
+ match self.to_proto().expr_type {
+ Some(proto::expression::ExprType::Literal(literal)) => literal,
+ _ => proto::expression::Literal::default(),
+ }
+ }
+
/// Render the literal value, mirroring `LiteralExpression.__repr__`: the
Python value,
/// as `f"{value}"` shows it, and a date, time, timestamp or day-time
interval in the
/// formats that it gives them.
diff --git a/crates/spark-connect/src/group.rs
b/crates/spark-connect/src/group.rs
index b96c8cc..2840eb2 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -496,6 +496,19 @@ impl NaFunctions {
) -> DataFrame {
self.dataframe.replace(to_replace, subset)
}
+
+ /// Replace values given as literals. Mirrors
`DataFrameNaFunctions.replace`, as
+ /// [`DataFrame::replace_literals`].
+ pub fn replace_literals(
+ &self,
+ replacements: Vec<(
+ crate::expression::LiteralExpression,
+ crate::expression::LiteralExpression,
+ )>,
+ subset: Option<Vec<&str>>,
+ ) -> DataFrame {
+ self.dataframe.replace_literals(replacements, subset)
+ }
}
/// A pair of cogrouped [`GroupedData`], mirroring
`pyspark.sql.PandasCogroupedOps`.
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index 573026c..e9ef63d 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -8,7 +8,7 @@ use std::collections::HashMap;
use std::sync::{Arc, Mutex, Weak};
use crate::column::Column;
-use crate::expression::{Expression, ExpressionNode};
+use crate::expression::{Expression, ExpressionNode, LiteralExpression};
use crate::types::DataType;
use crate::udf::CommonInlineUserDefinedFunctionExpression;
@@ -206,10 +206,11 @@ pub enum LogicalPlan {
min_non_null: Option<i32>,
columns: Vec<String>,
},
- /// NAReplace: `df.replace(to_replace, value, subset)`.
+ /// NAReplace: `df.replace(to_replace, value, subset)`, each replacement a
literal and
+ /// the literal that replaces it.
NAReplace {
input: Arc<LogicalPlan>,
- replacements: Vec<(String, String)>,
+ replacements: Vec<(LiteralExpression, LiteralExpression)>,
columns: Vec<String>,
},
/// Describe: `df.describe(cols...)`.
@@ -1116,13 +1117,12 @@ impl LogicalPlan {
let mut na_replace = proto::NaReplace::default();
na_replace.input = Some(Box::new(relation_of(input,
plan_ids)));
for (old_val, new_val) in replacements.iter() {
- let mut replace =
proto::na_replace::Replacement::default();
- // A numeric-looking value becomes a Double literal;
otherwise a
- // String literal. Both sides are always set (never left
None, which
- // previously made string replacements a silent no-op).
- replace.old_value = Some(str_to_proto_literal(old_val));
- replace.new_value = Some(str_to_proto_literal(new_val));
- na_replace.replacements.push(replace);
+ na_replace
+ .replacements
+ .push(proto::na_replace::Replacement {
+ old_value: Some(old_val.to_literal_proto()),
+ new_value: Some(new_val.to_literal_proto()),
+ });
}
na_replace.cols.extend(columns.clone());
relation.rel_type =
Some(proto::relation::RelType::Replace(Box::new(na_replace)));
@@ -1961,17 +1961,13 @@ fn value_to_proto_literal(v: &crate::row::Value) ->
proto::expression::Literal {
lit
}
-/// Build a proto `Literal` from a string: a numeric-looking string becomes a
-/// Double literal, otherwise a String literal (used by NAReplace so string
-/// replacements are not silently dropped).
-fn str_to_proto_literal(s: &str) -> proto::expression::Literal {
- use proto::expression::literal::LiteralType;
- let mut lit = proto::expression::Literal::default();
- lit.literal_type = Some(match s.parse::<f64>() {
- Ok(v) => LiteralType::Double(v),
- Err(_) => LiteralType::String(s.to_string()),
- });
- lit
+/// The literal of a replacement given as a string: a double for a
numeric-looking string,
+/// otherwise the string (used by the NAReplace of strings).
+pub(crate) fn numeric_or_string_literal(s: &str) -> LiteralExpression {
+ match s.parse::<f64>() {
+ Ok(v) => LiteralExpression::double(v),
+ Err(_) => LiteralExpression::string(s),
+ }
}
/// Create a NADrop plan.
@@ -1989,7 +1985,8 @@ pub fn na_drop(
}
}
-/// Create a NAReplace plan.
+/// Create a NAReplace plan, of replacements given as strings: a
numeric-looking string is a
+/// double, otherwise a string.
pub fn na_replace(
input: impl Into<Arc<LogicalPlan>>,
replacements: Vec<(String, String)>,
@@ -1997,7 +1994,15 @@ pub fn na_replace(
) -> LogicalPlan {
LogicalPlan::NAReplace {
input: input.into(),
- replacements,
+ replacements: replacements
+ .iter()
+ .map(|(old, new)| {
+ (
+ numeric_or_string_literal(old),
+ numeric_or_string_literal(new),
+ )
+ })
+ .collect(),
columns,
}
}
@@ -2122,12 +2127,60 @@ mod argfix_tests {
}
#[test]
- fn replace_sets_string_literals() {
+ fn replace_sets_the_literals_of_their_types() {
+ // As PySpark's: a bool, a double, a string, or a null replacement, as
a literal of its
+ // type.
let p = LogicalPlan::NAReplace {
input: Arc::new(base()),
- replacements: vec![("foo".to_string(), "bar".to_string())],
- columns: vec![],
+ replacements: vec![
+ (
+ LiteralExpression::boolean(true),
+ LiteralExpression::boolean(false),
+ ),
+ (
+ LiteralExpression::double(1.0),
+ LiteralExpression::null(DataType::Null),
+ ),
+ (
+ LiteralExpression::string("1"),
+ LiteralExpression::string("a"),
+ ),
+ ],
+ columns: vec!["b".to_string()],
};
+ match rel_type(p) {
+ proto::relation::RelType::Replace(r) => {
+ let literals: Vec<(LiteralType, LiteralType)> = r
+ .replacements
+ .iter()
+ .map(|repl| {
+ let literal_type = |lit:
&Option<proto::expression::Literal>| {
+ lit.clone().unwrap().literal_type.unwrap()
+ };
+ (literal_type(&repl.old_value),
literal_type(&repl.new_value))
+ })
+ .collect();
+ assert!(matches!(
+ &literals[0],
+ (LiteralType::Boolean(true), LiteralType::Boolean(false))
+ ));
+ assert!(matches!(
+ &literals[1],
+ (LiteralType::Double(v), LiteralType::Null(_)) if *v == 1.0
+ ));
+ assert!(matches!(
+ &literals[2],
+ (LiteralType::String(old), LiteralType::String(new)) if
old == "1" && new == "a"
+ ));
+ assert_eq!(r.cols, vec!["b".to_string()]);
+ }
+ _ => panic!("expected Replace"),
+ }
+ }
+
+ #[test]
+ fn replace_sets_string_literals() {
+ let p = na_replace(base(), vec![("foo".to_string(),
"bar".to_string())], vec![]);
match rel_type(p) {
proto::relation::RelType::Replace(r) => {
let repl = &r.replacements[0];
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 0745cb7..96045ca 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -141,6 +141,35 @@ def
test_repr_shows_the_rows_with_eager_evaluation_as_pyspark(session):
assert repr(df) == "DataFrame[id: bigint]"
+def test_replace_and_unpivot_take_their_arguments_as_pyspark(session):
+ # As PySpark's: replace sends each value as a literal of its type, and
None as a null, over
+ # a subset of a column name or a list or tuple of them. unpivot and melt
take a column or a
+ # column name, or a list or tuple of them.
+ df = session.createDataFrame(
+ [(1, "a", 1.5, True, "1"), (2, "b", None, False, "2")],
+ "id int, k string, v double, b boolean, s string",
+ )
+
+ def rows(*changes):
+ base = [dict(id=1, k="a", v=1.5, b=True, s="1"), dict(id=2, k="b",
v=None, b=False, s="2")]
+ return [Row(**{**row, **change}) for row, change in zip(base, changes)]
+
+ assert df.replace("a", None, "k").collect() == rows({"k": None}, {})
+ # A string replaces in the string columns, and a number in the numeric
ones.
+ assert df.replace("1", "one").collect() == rows({"s": "one"}, {})
+ assert df.replace(1, 10, ("id",)).collect() == rows({"id": 10}, {})
+ assert df.replace([1.5, 2], [9.5, 20]).collect() == rows({"v": 9.5},
{"id": 20})
+ assert df.replace(True, False).collect() == rows({"b": False}, {})
+ assert df.na.replace({"a": "x", "b": "y"}, subset="k").collect() ==
rows({"k": "x"}, {"k": "y"})
+
+ wide = session.createDataFrame([(1, 11, 12)], "id int, x int, y int")
+ expected = [Row(id=1, var="x", val=11), Row(id=1, var="y", val=12)]
+ assert wide.unpivot("id", ["x", "y"], "var", "val").collect() == expected
+ assert wide.unpivot(F.col("id"), ("x", F.col("y")), "var",
"val").collect() == expected
+ assert wide.melt(["id"], None, "var", "val").collect() == expected
+ assert wide.melt("id", "x", "var", "val").collect() == expected[:1]
+
+
def test_instr_and_regexp_replace_apply_their_optional_arguments(session):
# Column b holds "c", so the literal "b" and the column b find different
positions.
df = session.createDataFrame([("abcabc", "c", "a", "z")], ["s", "b", "p",
"r"])
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index a85b979..9959071 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2844,6 +2844,54 @@ def test_functions_substring():
assert type(F.substr(str_col, F.lit(1), F.lit(5))).__name__ == "Column"
+def test_replace_and_unpivot_check_their_arguments_as_pyspark():
+ # As PySpark's DataFrame.replace and unpivot, which check their arguments
on the client.
+ import warnings
+
+ from pyspark.errors import PySparkTypeError, PySparkValueError
+
+ df = _offline_session().range(1)
+ for call, error_type, condition, parameters in (
+ (lambda: df.replace("a"), PySparkTypeError, "ARGUMENT_REQUIRED",
+ {"arg_name": "value", "condition": "`to_replace` is dict"}),
+ (lambda: df.na.replace([1]), PySparkTypeError, "ARGUMENT_REQUIRED",
+ {"arg_name": "value", "condition": "`to_replace` is dict"}),
+ (lambda: df.replace({1, 2}, 3), PySparkTypeError, "NOT_EXPECTED_TYPE",
+ {"expected_type": "bool, dict, float, int, str or tuple", "arg_name":
"to_replace",
+ "arg_type": "set"}),
+ (lambda: df.replace(1, {2: 3}), PySparkTypeError, "NOT_EXPECTED_TYPE",
+ {"expected_type": "bool, float, int, list, None, str or tuple",
"arg_name": "value",
+ "arg_type": "dict"}),
+ (lambda: df.replace([1, 2], [3]), PySparkValueError,
"LENGTH_SHOULD_BE_THE_SAME",
+ {"arg1": "to_replace", "arg2": "value", "arg1_length": "2",
"arg2_length": "1"}),
+ (lambda: df.replace("a", "b", 1), PySparkTypeError,
"NOT_EXPECTED_TYPE",
+ {"arg_name": "subset", "expected_type": "list, str or tuple",
"arg_type": "int"}),
+ (lambda: df.replace({1: "a"}), PySparkValueError,
"MIXED_TYPE_REPLACEMENT", {}),
+ (lambda: df.na.replace([1, "a"], "b"), PySparkValueError,
"MIXED_TYPE_REPLACEMENT", {}),
+ (lambda: df.unpivot(1, "id", "var", "val"), PySparkTypeError,
"NOT_EXPECTED_TYPE",
+ {"expected_type": "Column or str", "arg_name": "col", "arg_type":
"int"}),
+ (lambda: df.melt(["id"], [1], "var", "val"), PySparkTypeError,
"NOT_EXPECTED_TYPE",
+ {"expected_type": "Column or str", "arg_name": "col", "arg_type":
"int"}),
+ ):
+ with pytest.raises(error_type) as error:
+ call()
+ assert error.value.getCondition() == condition
+ assert error.value.getMessageParameters() == parameters
+ with pytest.raises(AssertionError, match="ids must not be None"):
+ df.unpivot(None, "id", "var", "val")
+ # All four arguments of unpivot, and of melt, which is the same function,
are required.
+ for unpivot in (df.unpivot, df.melt):
+ with pytest.raises(TypeError):
+ unpivot("id", "id")
+ # The value of a dict to_replace is ignored, with a warning.
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ df.replace({"a": "b"}, "c")
+ assert [str(w.message) for w in caught] == [
+ "to_replace is a dict and value is not None. value will be ignored."
+ ]
+
+
def test_functions_replace():
"""Test replace function."""
from pyspark.sql import functions as F
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]