This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 4a076e46 [SPARK-61140][CONNECT][PYTHON] Send the renames of 
DataFrame.withColumnsRenamed in their order, in the Rust core and the Python 
client, as PySpark does
4a076e46 is described below

commit 4a076e46c01c6a110cd7258f5810f3a8a05f78b3
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 13:04:36 2026 +0900

    [SPARK-61140][CONNECT][PYTHON] Send the renames of 
DataFrame.withColumnsRenamed in their order, in the Rust core and the Python 
client, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61140](https://issues.apache.org/jira/browse/SPARK-61140): PySpark's 
`withColumnsRenamed` sends its renames in the order of their dict (the repeated 
`renames` of `WithColumnsRenamed`), and the server applies them in that order. 
So `df.withColumnsRenamed({"id": "a", "a": "b"})` has the columns `["b"]`, and 
`df.withColumnsRenamed({"a": "b", "id": "a"})` has `["a"]`.
    
    The Rust core now keeps the renames of a `WithColumnsRenamed` plan in their 
order (a `Vec` of pairs, rather than a `HashMap`), and sends them in it:
    - `DataFrame::with_columns_renamed` keeps the order it is given, which the 
Python client gives of the dict.
    - `plan::with_columns_renamed` takes any iterator of pairs, so its callers 
with a `HashMap` still compile.
    
    ### Why are the changes needed?
    
    The core kept the renames in a `HashMap`, so it sent them in the order of 
their hashes, not in the order given. The columns of chained renames such as 
`{"id": "a", "a": "b"}` then depended on that order. Apache Spark's 
`test_ordering_of_with_columns_renamed` failed on this client: `['a'] != ['b']`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. The renames of `withColumnsRenamed` are applied in their order, as 
with PySpark 4.3.0. Before, chained renames gave columns that depended on 
hashing.
    
    For the Rust API, the `renames` of `LogicalPlan::WithColumnsRenamed` are 
now a `Vec<(String, String)>`.
    
    ### How was this patch tested?
    
    - A new Rust test (`plan_serialization.rs`) checks that the renames of a 
plan are sent in their order, also reversed.
    - A new live test checks the columns of chained renames in both orders, and 
of a chain of six. It passes with official PySpark 4.3.0.
    - Apache Spark's `test_ordering_of_with_columns_renamed` passes on this 
client against a local server.
    - The lib tests of both crates (590 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2201 passed, 8 skipped), and the 
live drop-in and Arrow suites (185 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #332 from HyukjinKwon/SPARK-61140-ordered-renames.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/spark-connect/src/dataframe.rs            | 14 +++++-------
 crates/spark-connect/src/plan.rs                 | 12 ++++++-----
 crates/spark-connect/tests/plan_serialization.rs | 27 ++++++++++++++++++++++++
 python/tests/test_dropin_e2e.py                  | 11 ++++++++++
 4 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index f622d4c5..c9ae717d 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -374,24 +374,20 @@ impl DataFrame {
 
     /// Rename a column.
     pub fn with_column_renamed(&self, existing: &str, new: &str) -> DataFrame {
-        let mut renames = HashMap::new();
-        renames.insert(existing.to_string(), new.to_string());
         let plan = LogicalPlan::WithColumnsRenamed {
             input: Arc::clone(&self.plan),
-            renames,
+            renames: vec![(existing.to_string(), new.to_string())],
         };
         DataFrame::new(self.session.clone(), plan)
     }
 
-    /// Rename multiple columns.
+    /// Rename multiple columns, of old and new names in their order, in which 
the server
+    /// applies them, as PySpark's `withColumnsRenamed` of a dict sends them: 
so
+    /// `[("id", "a"), ("a", "b")]` renames `id` to `b`.
     pub fn with_columns_renamed(&self, renames: Vec<(String, String)>) -> 
DataFrame {
-        let mut rename_map = HashMap::new();
-        for (old, new) in renames {
-            rename_map.insert(old, new);
-        }
         let plan = LogicalPlan::WithColumnsRenamed {
             input: Arc::clone(&self.plan),
-            renames: rename_map,
+            renames,
         };
         DataFrame::new(self.session.clone(), plan)
     }
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index 74406774..72ec7527 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -186,10 +186,11 @@ pub enum LogicalPlan {
         column: Expression,
         metadata_json: String,
     },
-    /// WithColumnsRenamed: `df.withColumnRenamed(old, new)` or 
`df.withColumnsRenamed(...)`.
+    /// WithColumnsRenamed: `df.withColumnRenamed(old, new)` or 
`df.withColumnsRenamed(...)`,
+    /// of the renames in their order, in which the server applies them, as 
PySpark sends them.
     WithColumnsRenamed {
         input: Arc<LogicalPlan>,
-        renames: HashMap<String, String>,
+        renames: Vec<(String, String)>,
     },
     /// Drop: `df.drop(column_names...)`.
     Drop {
@@ -1949,14 +1950,15 @@ pub fn with_columns(
     }
 }
 
-/// Create a WithColumnsRenamed plan.
+/// Create a WithColumnsRenamed plan of `renames`, of old and new names, in 
their order, in
+/// which the server applies them.
 pub fn with_columns_renamed(
     input: impl Into<Arc<LogicalPlan>>,
-    renames: HashMap<String, String>,
+    renames: impl IntoIterator<Item = (String, String)>,
 ) -> LogicalPlan {
     LogicalPlan::WithColumnsRenamed {
         input: input.into(),
-        renames,
+        renames: renames.into_iter().collect(),
     }
 }
 
diff --git a/crates/spark-connect/tests/plan_serialization.rs 
b/crates/spark-connect/tests/plan_serialization.rs
index f34f3886..0253d1e0 100644
--- a/crates/spark-connect/tests/plan_serialization.rs
+++ b/crates/spark-connect/tests/plan_serialization.rs
@@ -117,6 +117,33 @@ fn local_and_cached_relations_serialize() {
     }
 }
 
+#[test]
+fn renames_are_sent_in_their_order() {
+    // As PySpark's withColumnsRenamed of a dict: the renames in their order, 
in which the server
+    // applies them, so that ("id", "a") then ("a", "b") renames `id` to `b`.
+    let renames = |plan: LogicalPlan| match plan.to_proto().rel_type {
+        Some(proto::relation::RelType::WithColumnsRenamed(wcr)) => wcr
+            .renames
+            .iter()
+            .map(|r| (r.col_name.clone(), r.new_col_name.clone()))
+            .collect::<Vec<_>>(),
+        other => panic!("expected a WithColumnsRenamed, got {other:?}"),
+    };
+    let pairs: Vec<(String, String)> = ["id", "a", "b", "c", "d", "e", "f", 
"g"]
+        .windows(2)
+        .map(|w| (w[0].to_string(), w[1].to_string()))
+        .collect();
+    assert_eq!(
+        renames(plan::with_columns_renamed(base(), pairs.clone())),
+        pairs
+    );
+    let reversed: Vec<_> = pairs.iter().rev().cloned().collect();
+    assert_eq!(
+        renames(plan::with_columns_renamed(base(), reversed.clone())),
+        reversed
+    );
+}
+
 #[test]
 fn relation_changes_maps_to_correct_proto() {
     use std::collections::HashMap;
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index d3b79607..a6a19f2d 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -269,6 +269,17 @@ def 
test_udtfs_and_udfs_of_a_dataframe_or_the_session_raise_no_active_session_as
 
 
 
+def test_renames_are_applied_in_their_order_as_pyspark(session):
+    # As PySpark's withColumnsRenamed of a dict, whose renames the server 
applies in their order.
+    df = session.range(10)
+    assert df.withColumnsRenamed({"id": "a", "a": "b"}).columns == ["b"]
+    assert df.withColumnsRenamed({"a": "b", "id": "a"}).columns == ["a"]
+    renames = {"id": "c1", "c1": "c2", "c2": "c3", "c3": "c4", "c4": "c5", 
"c5": "c6"}
+    assert df.withColumnsRenamed(renames).columns == ["c6"]
+
+
+
+
 def test_join_with_an_unsupported_join_type_raises_as_pyspark(session):
     # As PySpark: the join type is checked when the join is built, which 
raises the
     # AnalysisException of UNSUPPORTED_JOIN_TYPE, naming the type as it is 
normalized.


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to