This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 10ffe91  [SPARK-61015][CONNECT][PYTHON] Take the arguments of unpivot, 
melt and replace as PySpark does
10ffe91 is described below

commit 10ffe915a6c0c8917ddea52679fe555e72f4e5ff
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 01:44:13 2026 +0900

    [SPARK-61015][CONNECT][PYTHON] Take the arguments of unpivot, melt and 
replace as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61015](https://issues.apache.org/jira/browse/SPARK-61015): 
`unpivot`, `melt` and `replace` now take their arguments as PySpark does.
    
    - `unpivot` and `melt`, which is the same function, take `ids` and `values` 
that are a column name, a Column, or a list or tuple of them. All four 
arguments are required.
    - `replace` (and `na.replace`) takes a `subset` that is a column name, or a 
list or tuple of them. It sends each value as a literal of its type, as PySpark 
Connect builds the `NAReplace` relation: a bool, a double (an int as well), a 
string, or `None` as a null.
    - `replace` raises `ARGUMENT_REQUIRED`, `NOT_EXPECTED_TYPE`, 
`LENGTH_SHOULD_BE_THE_SAME` and `MIXED_TYPE_REPLACEMENT` where PySpark does, 
and warns that the `value` of a dict `to_replace` is ignored.
    
    In the Rust API, `DataFrame::replace_literals` and 
`NaFunctions::replace_literals` take the replacements as literals. 
`DataFrame::replace` of strings is unchanged.
    
    ### Why are the changes needed?
    
    They took lists only: `df.unpivot("id", ["v"], "var", "val")` and 
`df.replace("a", "z", "k")` raised a `TypeError`. The values of `replace` were 
sent as strings, and one that parses as a number as a double. So:
    
    - `df.replace("1", "one")` and `df.replace({1: "x"})` failed on the server 
with `CAST_INVALID_INPUT`.
    - `df.replace(True, False)` replaced nothing.
    - `df.replace({"a": None})` replaced `"a"` with the string `'None'`.
    - `df.replace("a", None, ["k"])` replaced nothing.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `unpivot`, `melt` and `replace` take the arguments of PySpark 4.3.0, 
replace as it does, and raise its errors.
    
    ### How was this patch tested?
    
    - A new live test replaces strings, numbers, bools and nulls over a subset 
of a name, a list or a tuple, and unpivots and melts by names and Columns. It 
passes with official PySpark 4.3.0, and fails without this change.
    - A new offline test checks the errors of the arguments, and the warning. 
It passes with official PySpark 4.3.0, and fails without this change.
    - New lib tests check the literals of the replacements.
    - A probe of 47 calls matches official PySpark 4.3.0. The exceptions: two 
messages of `TypeError` that name the function differently.
    - The lib tests and every integration test, e2e included, against a local 
Spark 4.3.0 server (673 passed, and 129 of the transport crate), the build of 
all tests with `wasm-udf`, the offline Python suites (1625 passed, 8 skipped), 
and the live drop-in and Arrow suites (95 passed) and live cases (10 passed). 
Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #219 from HyukjinKwon/SPARK-61015-fix.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs     | 350 +++++++++++++++++++++++----------
 crates/spark-connect/src/dataframe.rs  |  72 ++++++-
 crates/spark-connect/src/expression.rs |   8 +
 crates/spark-connect/src/group.rs      |  13 ++
 crates/spark-connect/src/plan.rs       | 105 +++++++---
 python/tests/test_dropin_e2e.py        |  29 +++
 python/tests/test_dropin_offline.py    |  48 +++++
 7 files changed, 495 insertions(+), 130 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index a0f59d0..a15eb44 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -988,37 +988,16 @@ impl PyDataFrame {
         ))
     }
 
-    /// Replace values. `to_replace` may be a {old: new} dict, or a value/list 
with `value`.
-    #[pyo3(signature = (to_replace, value=None, subset=None))]
+    /// Replace values, as PySpark's `DataFrame.replace`: `to_replace` is a 
{old: new} dict, or
+    /// a value or a list or tuple of them with `value`, a value or a list or 
tuple of them.
+    #[pyo3(signature = (to_replace, value=ReplaceValue::Omitted, subset=None))]
     fn replace(
         &self,
         to_replace: &Bound<'_, PyAny>,
-        value: Option<&Bound<'_, PyAny>>,
-        subset: Option<Vec<String>>,
+        value: ReplaceValue,
+        subset: Option<&Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
-        let mut pairs: Vec<(String, String)> = Vec::new();
-        if let Ok(d) = to_replace.cast::<PyDict>() {
-            for (k, v) in d.iter() {
-                pairs.push((k.str()?.to_string(), v.str()?.to_string()));
-            }
-        } else if let (Ok(olds), Some(val)) = 
(to_replace.extract::<Vec<Bound<'_, PyAny>>>(), value)
-        {
-            if let Ok(news) = val.extract::<Vec<Bound<'_, PyAny>>>() {
-                for (o, n) in olds.iter().zip(news.iter()) {
-                    pairs.push((o.str()?.to_string(), n.str()?.to_string()));
-                }
-            } else {
-                for o in olds {
-                    pairs.push((o.str()?.to_string(), val.str()?.to_string()));
-                }
-            }
-        } else if let Some(val) = value {
-            pairs.push((to_replace.str()?.to_string(), 
val.str()?.to_string()));
-        }
-        let subset_refs: Option<Vec<&str>> = subset
-            .as_ref()
-            .map(|s| s.iter().map(|x| x.as_str()).collect());
-        Ok(PyDataFrame::new(self.dataframe.replace(pairs, subset_refs)))
+        replace_values(&self.dataframe, to_replace, value, subset)
     }
 
     /// Reconcile to a target schema (StructType or DDL string).
@@ -1164,48 +1143,42 @@ impl PyDataFrame {
         Ok(func.call(tuple, kwargs)?.unbind())
     }
 
-    /// Unpivot (wide-to-long). `values=None` unpivots all non-id columns.
-    #[pyo3(signature = (ids, values=None, variableColumnName="variable", 
valueColumnName="value"))]
+    /// Unpivot (wide-to-long), as PySpark's `DataFrame.unpivot`: `ids` and 
`values` are each a
+    /// column or a column name, or a list or tuple of them, and `values=None` 
unpivots all
+    /// non-id columns.
+    #[pyo3(signature = (ids, values, variableColumnName, valueColumnName))]
     fn unpivot(
         &self,
-        ids: Vec<String>,
-        values: Option<Vec<String>>,
+        ids: &Bound<'_, PyAny>,
+        values: Option<&Bound<'_, PyAny>>,
         variableColumnName: &str,
         valueColumnName: &str,
-    ) -> PyDataFrame {
-        let id_refs: Vec<&str> = ids.iter().map(|s| s.as_str()).collect();
-        let owned = values;
-        let val_refs = owned
-            .as_ref()
-            .map(|v| v.iter().map(|s| s.as_str()).collect());
-        PyDataFrame::new(self.dataframe.melt(
-            id_refs,
-            val_refs,
+    ) -> PyResult<PyDataFrame> {
+        if ids.is_none() {
+            return Err(pyo3::exceptions::PyAssertionError::new_err(
+                "ids must not be None",
+            ));
+        }
+        let ids = unpivot_columns(ids)?;
+        let values = values.map(unpivot_columns).transpose()?;
+        Ok(PyDataFrame::new(self.dataframe.unpivot(
+            ids,
+            values,
             variableColumnName,
             valueColumnName,
-        ))
+        )))
     }
 
-    /// Alias for unpivot.
-    #[pyo3(signature = (ids, values=None, variableColumnName="variable", 
valueColumnName="value"))]
+    /// The same function as `unpivot`, as PySpark's `DataFrame.melt`.
+    #[pyo3(signature = (ids, values, variableColumnName, valueColumnName))]
     fn melt(
         &self,
-        ids: Vec<String>,
-        values: Option<Vec<String>>,
+        ids: &Bound<'_, PyAny>,
+        values: Option<&Bound<'_, PyAny>>,
         variableColumnName: &str,
         valueColumnName: &str,
-    ) -> PyDataFrame {
-        let id_refs: Vec<&str> = ids.iter().map(|s| s.as_str()).collect();
-        let owned = values;
-        let val_refs = owned
-            .as_ref()
-            .map(|v| v.iter().map(|s| s.as_str()).collect());
-        PyDataFrame::new(self.dataframe.melt(
-            id_refs,
-            val_refs,
-            variableColumnName,
-            valueColumnName,
-        ))
+    ) -> PyResult<PyDataFrame> {
+        self.unpivot(ids, values, variableColumnName, valueColumnName)
     }
 
     /// Persist the DataFrame. A storage-level argument is accepted for API 
parity;
@@ -1819,22 +1792,16 @@ impl PyDataFrameNaFunctions {
         Ok(PyDataFrame::new(self.df.fillna_value(val, refs)))
     }
 
-    /// Replace values, mirroring `DataFrameNaFunctions.replace`:
-    /// `replace([old...], [new...], subset)`, `replace({old: new}, subset)`, 
or
-    /// `replace(old, new, subset)`.
-    #[pyo3(signature = (to_replace, value=None, subset=None))]
+    /// Replace values, as PySpark's `DataFrameNaFunctions.replace`, which is
+    /// `DataFrame.replace`.
+    #[pyo3(signature = (to_replace, value=ReplaceValue::Omitted, subset=None))]
     fn replace(
         &self,
         to_replace: &Bound<'_, PyAny>,
-        value: Option<&Bound<'_, PyAny>>,
+        value: ReplaceValue,
         subset: Option<&Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
-        let pairs = build_replacement_pairs(to_replace, value)?;
-        let owned = to_subset(subset)?;
-        let subset_refs: Option<Vec<&str>> = owned
-            .as_ref()
-            .map(|v| v.iter().map(String::as_str).collect());
-        Ok(PyDataFrame::new(self.inner.replace(pairs, subset_refs)))
+        replace_values(&self.df, to_replace, value, subset)
     }
 }
 
@@ -1853,44 +1820,223 @@ fn to_subset(subset: Option<&Bound<'_, PyAny>>) -> 
PyResult<Option<Vec<String>>>
     }
 }
 
-/// Build (old, new) string replacement pairs from the reference `replace` 
forms.
-fn build_replacement_pairs(
+/// The `value` argument of `DataFrame.replace`, where PySpark tells an 
omitted value apart
+/// from None: `replace(to_replace)` requires a dict `to_replace`, while
+/// `replace(to_replace, None)` replaces with nulls.
+pub enum ReplaceValue {
+    Omitted,
+    Given(Py<PyAny>),
+}
+
+impl<'a, 'py> FromPyObject<'a, 'py> for ReplaceValue {
+    type Error = PyErr;
+
+    fn extract(obj: Borrowed<'a, 'py, PyAny>) -> PyResult<Self> {
+        Ok(ReplaceValue::Given(obj.to_owned().unbind()))
+    }
+}
+
+/// `df.replace(to_replace, value, subset)`, checked and built as PySpark's
+/// `DataFrame.replace`: each value of `to_replace` (a {old: new} dict, or a 
value or a list or
+/// tuple of them) is replaced by `value` (a value or a list or tuple of them, 
or None), all
+/// bools, numbers or strings, sent as literals with an int as a float, in the 
columns of
+/// `subset` (a column name, or a list or tuple of them).
+fn replace_values(
+    dataframe: &DataFrame,
     to_replace: &Bound<'_, PyAny>,
-    value: Option<&Bound<'_, PyAny>>,
-) -> PyResult<Vec<(String, String)>> {
-    use pyo3::exceptions::PyValueError;
-    use pyo3::types::{PyDict, PyList};
-    if let Ok(d) = to_replace.cast::<PyDict>() {
-        let mut pairs = Vec::with_capacity(d.len());
-        for (k, v) in d.iter() {
-            pairs.push((k.str()?.to_string(), v.str()?.to_string()));
+    value: ReplaceValue,
+    subset: Option<&Bound<'_, PyAny>>,
+) -> PyResult<PyDataFrame> {
+    use pyo3::types::{PyBool, PyFloat, PyString};
+    use spark_connect::expression::LiteralExpression;
+
+    let py = to_replace.py();
+    let raise = |class_name: &str, error_class: &str, params: &[(&str, &str)]| 
{
+        crate::column::raise_pyspark(py, class_name, error_class, params)
+    };
+    let type_name = |v: &Bound<'_, PyAny>| -> String {
+        v.get_type()
+            .name()
+            .map(|name| name.to_string())
+            .unwrap_or_default()
+    };
+    let is_sequence =
+        |v: &Bound<'_, PyAny>| v.is_instance_of::<PyList>() || 
v.is_instance_of::<PyTuple>();
+    // A bool, a float, an int or a str: a bool is an int.
+    let is_scalar = |v: &Bound<'_, PyAny>| {
+        v.is_instance_of::<PyFloat>()
+            || v.is_instance_of::<PyInt>()
+            || v.is_instance_of::<PyString>()
+    };
+    let is_dict = to_replace.is_instance_of::<PyDict>();
+    let value = match value {
+        ReplaceValue::Given(value) => value.into_bound(py),
+        ReplaceValue::Omitted if is_dict => py.None().into_bound(py),
+        ReplaceValue::Omitted => {
+            return Err(raise(
+                "PySparkTypeError",
+                "ARGUMENT_REQUIRED",
+                &[("arg_name", "value"), ("condition", "`to_replace` is 
dict")],
+            ))
         }
-        return Ok(pairs);
-    }
-    if let Ok(olds) = to_replace.cast::<PyList>() {
-        let val = value.ok_or_else(|| {
-            PyValueError::new_err("replace with a list to_replace requires a 
value list")
-        })?;
-        let news = val
-            .cast::<PyList>()
-            .map_err(|_| PyValueError::new_err("value must be a list when 
to_replace is a list"))?;
-        if olds.len() != news.len() {
-            return Err(PyValueError::new_err(
-                "to_replace and value lists must be the same length",
-            ));
+    };
+
+    if !(is_scalar(to_replace) || is_sequence(to_replace) || is_dict) {
+        return Err(raise(
+            "PySparkTypeError",
+            "NOT_EXPECTED_TYPE",
+            &[
+                ("expected_type", "bool, dict, float, int, str or tuple"),
+                ("arg_name", "to_replace"),
+                ("arg_type", &type_name(to_replace)),
+            ],
+        ));
+    }
+    if !(is_scalar(&value) || is_sequence(&value) || value.is_none() || 
is_dict) {
+        return Err(raise(
+            "PySparkTypeError",
+            "NOT_EXPECTED_TYPE",
+            &[
+                (
+                    "expected_type",
+                    "bool, float, int, list, None, str or tuple",
+                ),
+                ("arg_name", "value"),
+                ("arg_type", &type_name(&value)),
+            ],
+        ));
+    }
+    if is_sequence(to_replace) && is_sequence(&value) && to_replace.len()? != 
value.len()? {
+        return Err(raise(
+            "PySparkValueError",
+            "LENGTH_SHOULD_BE_THE_SAME",
+            &[
+                ("arg1", "to_replace"),
+                ("arg2", "value"),
+                ("arg1_length", &to_replace.len()?.to_string()),
+                ("arg2_length", &value.len()?.to_string()),
+            ],
+        ));
+    }
+    let subset: Option<Vec<String>> = match subset {
+        None => None,
+        Some(subset) if subset.is_instance_of::<PyString>() => 
Some(vec![subset.extract()?]),
+        Some(subset) if is_sequence(subset) => Some(subset.extract()?),
+        Some(subset) => {
+            return Err(raise(
+                "PySparkTypeError",
+                "NOT_EXPECTED_TYPE",
+                &[
+                    ("arg_name", "subset"),
+                    ("expected_type", "list, str or tuple"),
+                    ("arg_type", &type_name(subset)),
+                ],
+            ))
         }
-        let mut pairs = Vec::with_capacity(olds.len());
-        for (a, b) in olds.iter().zip(news.iter()) {
-            pairs.push((a.str()?.to_string(), b.str()?.to_string()));
+    };
+
+    // The replacements as a dict, as PySpark builds it, so that equal values 
are one key.
+    let replacements = if is_dict {
+        if !value.is_none() {
+            py.import("warnings")?.call_method1(
+                "warn",
+                ("to_replace is a dict and value is not None. value will be 
ignored.",),
+            )?;
+        }
+        to_replace.cast::<PyDict>()?.clone()
+    } else {
+        let olds: Vec<Bound<'_, PyAny>> = if is_scalar(to_replace) {
+            vec![to_replace.clone()]
+        } else {
+            to_replace.try_iter()?.collect::<PyResult<_>>()?
+        };
+        let news: Vec<Bound<'_, PyAny>> = if is_scalar(&value) || 
value.is_none() {
+            vec![value.clone(); olds.len()]
+        } else {
+            value.try_iter()?.collect::<PyResult<_>>()?
+        };
+        let replacements = PyDict::new(py);
+        for (old, new) in olds.iter().zip(news.iter()) {
+            replacements.set_item(old, new)?;
         }
-        return Ok(pairs);
-    }
-    let val =
-        value.ok_or_else(|| PyValueError::new_err("replace with a scalar 
requires a value"))?;
-    Ok(vec![(
-        to_replace.str()?.to_string(),
-        val.str()?.to_string(),
-    )])
+        replacements
+    };
+
+    // All of the values to replace, and all of their replacements but None, 
are bools,
+    // numbers or strings.
+    let olds: Vec<Bound<'_, PyAny>> = replacements.keys().iter().collect();
+    let news: Vec<Bound<'_, PyAny>> = replacements
+        .values()
+        .iter()
+        .filter(|new| !new.is_none())
+        .collect();
+    let all_of = |is_type: &dyn Fn(&Bound<'_, PyAny>) -> bool| {
+        olds.iter().all(is_type) && news.iter().all(is_type)
+    };
+    if !(all_of(&|v| v.is_instance_of::<PyBool>())
+        || all_of(&|v| v.is_instance_of::<PyString>())
+        || all_of(&|v| v.is_instance_of::<PyFloat>() || 
v.is_instance_of::<PyInt>()))
+    {
+        return Err(raise("PySparkValueError", "MIXED_TYPE_REPLACEMENT", &[]));
+    }
+
+    let literal = |v: &Bound<'_, PyAny>| -> PyResult<LiteralExpression> {
+        Ok(if v.is_none() {
+            LiteralExpression::null(spark_connect::types::DataType::Null)
+        } else if v.is_instance_of::<PyBool>() {
+            LiteralExpression::boolean(v.extract()?)
+        } else if v.is_instance_of::<PyString>() {
+            LiteralExpression::string(v.extract::<String>()?)
+        } else {
+            // As `float(v)`, for an int as well.
+            LiteralExpression::double(v.extract()?)
+        })
+    };
+    let replacements = replacements
+        .iter()
+        .map(|(old, new)| Ok((literal(&old)?, literal(&new)?)))
+        .collect::<PyResult<Vec<_>>>()?;
+    let subset: Option<Vec<&str>> = subset
+        .as_ref()
+        .map(|subset| subset.iter().map(String::as_str).collect());
+    Ok(PyDataFrame::new(
+        dataframe.replace_literals(replacements, subset),
+    ))
+}
+
+/// The columns of the `ids` or `values` of `df.unpivot`, as PySpark's: a list 
or tuple of
+/// columns or column names, or one of them.
+fn unpivot_columns(cols: &Bound<'_, PyAny>) -> 
PyResult<Vec<spark_connect::column::Column>> {
+    if cols.is_instance_of::<PyList>() || cols.is_instance_of::<PyTuple>() {
+        cols.try_iter()?.map(|col| column_or_name(&col?)).collect()
+    } else {
+        Ok(vec![column_or_name(cols)?])
+    }
+}
+
+/// A column, or a column by its name, as PySpark's `_to_col`.
+fn column_or_name(col: &Bound<'_, PyAny>) -> 
PyResult<spark_connect::column::Column> {
+    if let Ok(column) = col.extract::<PyRef<'_, PyColumn>>() {
+        return Ok(column.column.clone());
+    }
+    if col.is_instance_of::<pyo3::types::PyString>() {
+        return Ok(spark_connect::functions::col(&col.extract::<String>()?));
+    }
+    let type_name = col
+        .get_type()
+        .name()
+        .map(|name| name.to_string())
+        .unwrap_or_default();
+    Err(crate::column::raise_pyspark(
+        col.py(),
+        "PySparkTypeError",
+        "NOT_EXPECTED_TYPE",
+        &[
+            ("expected_type", "Column or str"),
+            ("arg_name", "col"),
+            ("arg_type", &type_name),
+        ],
+    ))
 }
 
 /// Python wrapper for `MergeIntoWriter` (`df.mergeInto(...)`).
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index ebd9385..98c79e3 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1463,18 +1463,43 @@ impl DataFrame {
         DataFrame::new(self.session.clone(), plan)
     }
 
-    /// Replace values.
+    /// Replace values, each given as a string: a numeric-looking string is a 
double, otherwise
+    /// a string. [`DataFrame::replace_literals`] takes the values as literals.
     pub fn replace(
         &self,
         to_replace: Vec<(String, String)>,
         subset: Option<Vec<&str>>,
+    ) -> DataFrame {
+        let replacements = to_replace
+            .iter()
+            .map(|(old, new)| {
+                (
+                    crate::plan::numeric_or_string_literal(old),
+                    crate::plan::numeric_or_string_literal(new),
+                )
+            })
+            .collect();
+        self.replace_literals(replacements, subset)
+    }
+
+    /// Replace values, mirroring `pyspark.sql.DataFrame.replace`: each pair 
is a literal and
+    /// the literal that replaces it, which may be a null, in the columns of 
`subset`, or else
+    /// in every column of the literal's type. The server replaces a boolean, 
a double or a
+    /// string.
+    pub fn replace_literals(
+        &self,
+        replacements: Vec<(
+            crate::expression::LiteralExpression,
+            crate::expression::LiteralExpression,
+        )>,
+        subset: Option<Vec<&str>>,
     ) -> DataFrame {
         let columns = subset
             .map(|v| v.iter().map(|s| s.to_string()).collect())
             .unwrap_or_default();
         let plan = LogicalPlan::NAReplace {
             input: Arc::clone(&self.plan),
-            replacements: to_replace,
+            replacements,
             columns,
         };
         DataFrame::new(self.session.clone(), plan)
@@ -3801,6 +3826,49 @@ mod plan_construction_tests {
         }
     }
 
+    #[test]
+    fn replace_literals_plan() {
+        use crate::expression::LiteralExpression;
+        let spark = session();
+        let df = spark.range(3).unwrap();
+        let replacements = vec![(
+            LiteralExpression::string("a"),
+            LiteralExpression::null(DataType::Null),
+        )];
+        for result in [
+            df.replace_literals(replacements.clone(), Some(vec!["k"])),
+            df.na()
+                .replace_literals(replacements.clone(), Some(vec!["k"])),
+        ] {
+            match result.plan.as_ref() {
+                LogicalPlan::NAReplace {
+                    replacements: planned,
+                    columns,
+                    ..
+                } => {
+                    assert_eq!(planned, &replacements);
+                    assert_eq!(columns, &vec!["k".to_string()]);
+                }
+                _ => panic!("expected NAReplace plan"),
+            }
+        }
+        // A string replacement that parses as a number is a double, as before.
+        match df
+            .replace(vec![("1".to_string(), "a".to_string())], None)
+            .plan
+            .as_ref()
+        {
+            LogicalPlan::NAReplace { replacements, .. } => assert_eq!(
+                replacements,
+                &vec![(
+                    LiteralExpression::double(1.0),
+                    LiteralExpression::string("a")
+                )]
+            ),
+            _ => panic!("expected NAReplace plan"),
+        }
+    }
+
     #[test]
     fn replace_plan() {
         let spark = session();
diff --git a/crates/spark-connect/src/expression.rs 
b/crates/spark-connect/src/expression.rs
index e5f6365..b80fbfc 100644
--- a/crates/spark-connect/src/expression.rs
+++ b/crates/spark-connect/src/expression.rs
@@ -568,6 +568,14 @@ impl LiteralExpression {
         expr
     }
 
+    /// The literal as a proto `Literal`, as `to_proto` sets it in its 
`Expression`.
+    pub fn to_literal_proto(&self) -> proto::expression::Literal {
+        match self.to_proto().expr_type {
+            Some(proto::expression::ExprType::Literal(literal)) => literal,
+            _ => proto::expression::Literal::default(),
+        }
+    }
+
     /// Render the literal value, mirroring `LiteralExpression.__repr__`: the 
Python value,
     /// as `f"{value}"` shows it, and a date, time, timestamp or day-time 
interval in the
     /// formats that it gives them.
diff --git a/crates/spark-connect/src/group.rs 
b/crates/spark-connect/src/group.rs
index b96c8cc..2840eb2 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -496,6 +496,19 @@ impl NaFunctions {
     ) -> DataFrame {
         self.dataframe.replace(to_replace, subset)
     }
+
+    /// Replace values given as literals. Mirrors 
`DataFrameNaFunctions.replace`, as
+    /// [`DataFrame::replace_literals`].
+    pub fn replace_literals(
+        &self,
+        replacements: Vec<(
+            crate::expression::LiteralExpression,
+            crate::expression::LiteralExpression,
+        )>,
+        subset: Option<Vec<&str>>,
+    ) -> DataFrame {
+        self.dataframe.replace_literals(replacements, subset)
+    }
 }
 
 /// A pair of cogrouped [`GroupedData`], mirroring 
`pyspark.sql.PandasCogroupedOps`.
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index 573026c..e9ef63d 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -8,7 +8,7 @@ use std::collections::HashMap;
 use std::sync::{Arc, Mutex, Weak};
 
 use crate::column::Column;
-use crate::expression::{Expression, ExpressionNode};
+use crate::expression::{Expression, ExpressionNode, LiteralExpression};
 use crate::types::DataType;
 use crate::udf::CommonInlineUserDefinedFunctionExpression;
 
@@ -206,10 +206,11 @@ pub enum LogicalPlan {
         min_non_null: Option<i32>,
         columns: Vec<String>,
     },
-    /// NAReplace: `df.replace(to_replace, value, subset)`.
+    /// NAReplace: `df.replace(to_replace, value, subset)`, each replacement a 
literal and
+    /// the literal that replaces it.
     NAReplace {
         input: Arc<LogicalPlan>,
-        replacements: Vec<(String, String)>,
+        replacements: Vec<(LiteralExpression, LiteralExpression)>,
         columns: Vec<String>,
     },
     /// Describe: `df.describe(cols...)`.
@@ -1116,13 +1117,12 @@ impl LogicalPlan {
                 let mut na_replace = proto::NaReplace::default();
                 na_replace.input = Some(Box::new(relation_of(input, 
plan_ids)));
                 for (old_val, new_val) in replacements.iter() {
-                    let mut replace = 
proto::na_replace::Replacement::default();
-                    // A numeric-looking value becomes a Double literal; 
otherwise a
-                    // String literal. Both sides are always set (never left 
None, which
-                    // previously made string replacements a silent no-op).
-                    replace.old_value = Some(str_to_proto_literal(old_val));
-                    replace.new_value = Some(str_to_proto_literal(new_val));
-                    na_replace.replacements.push(replace);
+                    na_replace
+                        .replacements
+                        .push(proto::na_replace::Replacement {
+                            old_value: Some(old_val.to_literal_proto()),
+                            new_value: Some(new_val.to_literal_proto()),
+                        });
                 }
                 na_replace.cols.extend(columns.clone());
                 relation.rel_type = 
Some(proto::relation::RelType::Replace(Box::new(na_replace)));
@@ -1961,17 +1961,13 @@ fn value_to_proto_literal(v: &crate::row::Value) -> 
proto::expression::Literal {
     lit
 }
 
-/// Build a proto `Literal` from a string: a numeric-looking string becomes a
-/// Double literal, otherwise a String literal (used by NAReplace so string
-/// replacements are not silently dropped).
-fn str_to_proto_literal(s: &str) -> proto::expression::Literal {
-    use proto::expression::literal::LiteralType;
-    let mut lit = proto::expression::Literal::default();
-    lit.literal_type = Some(match s.parse::<f64>() {
-        Ok(v) => LiteralType::Double(v),
-        Err(_) => LiteralType::String(s.to_string()),
-    });
-    lit
+/// The literal of a replacement given as a string: a double for a 
numeric-looking string,
+/// otherwise the string (used by the NAReplace of strings).
+pub(crate) fn numeric_or_string_literal(s: &str) -> LiteralExpression {
+    match s.parse::<f64>() {
+        Ok(v) => LiteralExpression::double(v),
+        Err(_) => LiteralExpression::string(s),
+    }
 }
 
 /// Create a NADrop plan.
@@ -1989,7 +1985,8 @@ pub fn na_drop(
     }
 }
 
-/// Create a NAReplace plan.
+/// Create a NAReplace plan, of replacements given as strings: a 
numeric-looking string is a
+/// double, otherwise a string.
 pub fn na_replace(
     input: impl Into<Arc<LogicalPlan>>,
     replacements: Vec<(String, String)>,
@@ -1997,7 +1994,15 @@ pub fn na_replace(
 ) -> LogicalPlan {
     LogicalPlan::NAReplace {
         input: input.into(),
-        replacements,
+        replacements: replacements
+            .iter()
+            .map(|(old, new)| {
+                (
+                    numeric_or_string_literal(old),
+                    numeric_or_string_literal(new),
+                )
+            })
+            .collect(),
         columns,
     }
 }
@@ -2122,12 +2127,60 @@ mod argfix_tests {
     }
 
     #[test]
-    fn replace_sets_string_literals() {
+    fn replace_sets_the_literals_of_their_types() {
+        // As PySpark's: a bool, a double, a string, or a null replacement, as 
a literal of its
+        // type.
         let p = LogicalPlan::NAReplace {
             input: Arc::new(base()),
-            replacements: vec![("foo".to_string(), "bar".to_string())],
-            columns: vec![],
+            replacements: vec![
+                (
+                    LiteralExpression::boolean(true),
+                    LiteralExpression::boolean(false),
+                ),
+                (
+                    LiteralExpression::double(1.0),
+                    LiteralExpression::null(DataType::Null),
+                ),
+                (
+                    LiteralExpression::string("1"),
+                    LiteralExpression::string("a"),
+                ),
+            ],
+            columns: vec!["b".to_string()],
         };
+        match rel_type(p) {
+            proto::relation::RelType::Replace(r) => {
+                let literals: Vec<(LiteralType, LiteralType)> = r
+                    .replacements
+                    .iter()
+                    .map(|repl| {
+                        let literal_type = |lit: 
&Option<proto::expression::Literal>| {
+                            lit.clone().unwrap().literal_type.unwrap()
+                        };
+                        (literal_type(&repl.old_value), 
literal_type(&repl.new_value))
+                    })
+                    .collect();
+                assert!(matches!(
+                    &literals[0],
+                    (LiteralType::Boolean(true), LiteralType::Boolean(false))
+                ));
+                assert!(matches!(
+                    &literals[1],
+                    (LiteralType::Double(v), LiteralType::Null(_)) if *v == 1.0
+                ));
+                assert!(matches!(
+                    &literals[2],
+                    (LiteralType::String(old), LiteralType::String(new)) if 
old == "1" && new == "a"
+                ));
+                assert_eq!(r.cols, vec!["b".to_string()]);
+            }
+            _ => panic!("expected Replace"),
+        }
+    }
+
+    #[test]
+    fn replace_sets_string_literals() {
+        let p = na_replace(base(), vec![("foo".to_string(), 
"bar".to_string())], vec![]);
         match rel_type(p) {
             proto::relation::RelType::Replace(r) => {
                 let repl = &r.replacements[0];
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 0745cb7..96045ca 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -141,6 +141,35 @@ def 
test_repr_shows_the_rows_with_eager_evaluation_as_pyspark(session):
     assert repr(df) == "DataFrame[id: bigint]"
 
 
+def test_replace_and_unpivot_take_their_arguments_as_pyspark(session):
+    # As PySpark's: replace sends each value as a literal of its type, and 
None as a null, over
+    # a subset of a column name or a list or tuple of them. unpivot and melt 
take a column or a
+    # column name, or a list or tuple of them.
+    df = session.createDataFrame(
+        [(1, "a", 1.5, True, "1"), (2, "b", None, False, "2")],
+        "id int, k string, v double, b boolean, s string",
+    )
+
+    def rows(*changes):
+        base = [dict(id=1, k="a", v=1.5, b=True, s="1"), dict(id=2, k="b", 
v=None, b=False, s="2")]
+        return [Row(**{**row, **change}) for row, change in zip(base, changes)]
+
+    assert df.replace("a", None, "k").collect() == rows({"k": None}, {})
+    # A string replaces in the string columns, and a number in the numeric 
ones.
+    assert df.replace("1", "one").collect() == rows({"s": "one"}, {})
+    assert df.replace(1, 10, ("id",)).collect() == rows({"id": 10}, {})
+    assert df.replace([1.5, 2], [9.5, 20]).collect() == rows({"v": 9.5}, 
{"id": 20})
+    assert df.replace(True, False).collect() == rows({"b": False}, {})
+    assert df.na.replace({"a": "x", "b": "y"}, subset="k").collect() == 
rows({"k": "x"}, {"k": "y"})
+
+    wide = session.createDataFrame([(1, 11, 12)], "id int, x int, y int")
+    expected = [Row(id=1, var="x", val=11), Row(id=1, var="y", val=12)]
+    assert wide.unpivot("id", ["x", "y"], "var", "val").collect() == expected
+    assert wide.unpivot(F.col("id"), ("x", F.col("y")), "var", 
"val").collect() == expected
+    assert wide.melt(["id"], None, "var", "val").collect() == expected
+    assert wide.melt("id", "x", "var", "val").collect() == expected[:1]
+
+
 def test_instr_and_regexp_replace_apply_their_optional_arguments(session):
     # Column b holds "c", so the literal "b" and the column b find different 
positions.
     df = session.createDataFrame([("abcabc", "c", "a", "z")], ["s", "b", "p", 
"r"])
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index a85b979..9959071 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2844,6 +2844,54 @@ def test_functions_substring():
     assert type(F.substr(str_col, F.lit(1), F.lit(5))).__name__ == "Column"
 
 
+def test_replace_and_unpivot_check_their_arguments_as_pyspark():
+    # As PySpark's DataFrame.replace and unpivot, which check their arguments 
on the client.
+    import warnings
+
+    from pyspark.errors import PySparkTypeError, PySparkValueError
+
+    df = _offline_session().range(1)
+    for call, error_type, condition, parameters in (
+        (lambda: df.replace("a"), PySparkTypeError, "ARGUMENT_REQUIRED",
+         {"arg_name": "value", "condition": "`to_replace` is dict"}),
+        (lambda: df.na.replace([1]), PySparkTypeError, "ARGUMENT_REQUIRED",
+         {"arg_name": "value", "condition": "`to_replace` is dict"}),
+        (lambda: df.replace({1, 2}, 3), PySparkTypeError, "NOT_EXPECTED_TYPE",
+         {"expected_type": "bool, dict, float, int, str or tuple", "arg_name": 
"to_replace",
+          "arg_type": "set"}),
+        (lambda: df.replace(1, {2: 3}), PySparkTypeError, "NOT_EXPECTED_TYPE",
+         {"expected_type": "bool, float, int, list, None, str or tuple", 
"arg_name": "value",
+          "arg_type": "dict"}),
+        (lambda: df.replace([1, 2], [3]), PySparkValueError, 
"LENGTH_SHOULD_BE_THE_SAME",
+         {"arg1": "to_replace", "arg2": "value", "arg1_length": "2", 
"arg2_length": "1"}),
+        (lambda: df.replace("a", "b", 1), PySparkTypeError, 
"NOT_EXPECTED_TYPE",
+         {"arg_name": "subset", "expected_type": "list, str or tuple", 
"arg_type": "int"}),
+        (lambda: df.replace({1: "a"}), PySparkValueError, 
"MIXED_TYPE_REPLACEMENT", {}),
+        (lambda: df.na.replace([1, "a"], "b"), PySparkValueError, 
"MIXED_TYPE_REPLACEMENT", {}),
+        (lambda: df.unpivot(1, "id", "var", "val"), PySparkTypeError, 
"NOT_EXPECTED_TYPE",
+         {"expected_type": "Column or str", "arg_name": "col", "arg_type": 
"int"}),
+        (lambda: df.melt(["id"], [1], "var", "val"), PySparkTypeError, 
"NOT_EXPECTED_TYPE",
+         {"expected_type": "Column or str", "arg_name": "col", "arg_type": 
"int"}),
+    ):
+        with pytest.raises(error_type) as error:
+            call()
+        assert error.value.getCondition() == condition
+        assert error.value.getMessageParameters() == parameters
+    with pytest.raises(AssertionError, match="ids must not be None"):
+        df.unpivot(None, "id", "var", "val")
+    # All four arguments of unpivot, and of melt, which is the same function, 
are required.
+    for unpivot in (df.unpivot, df.melt):
+        with pytest.raises(TypeError):
+            unpivot("id", "id")
+    # The value of a dict to_replace is ignored, with a warning.
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        df.replace({"a": "b"}, "c")
+    assert [str(w.message) for w in caught] == [
+        "to_replace is a dict and value is not None. value will be ignored."
+    ]
+
+
 def test_functions_replace():
     """Test replace function."""
     from pyspark.sql import functions as F


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to