This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 0a5ad39  [SPARK-60981][CONNECT][PYTHON] Alias a Column with several 
names, and with metadata of any JSON values, as PySpark does
0a5ad39 is described below

commit 0a5ad394b6d0ebbb541ee91a9b099d779ee7be4b
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 13:17:49 2026 +0900

    [SPARK-60981][CONNECT][PYTHON] Alias a Column with several names, and with 
metadata of any JSON values, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60981](https://issues.apache.org/jira/browse/SPARK-60981): 
`Column.alias(*alias, **kwargs)`, and `Column.name`, now follow PySpark's 
`Expression.alias` and `ColumnAlias`:
    
    - **Names:** every given name is kept, so all the columns of a generator 
are named. For example, `F.explode("m").alias("k", "v")` names the key and the 
value of a map.
    - **Metadata:** metadata is sent as `json.dumps(metadata)`, so its values 
can be of any JSON type, such as `metadata={"k": 1, "n": [1, 2]}`.
    - **Errors, as in PySpark:**
      - metadata with more than one name raises `PySparkValueError` 
`ONLY_ALLOWED_FOR_SINGLE_COLUMN`;
      - another keyword argument raises `AssertionError: Unexpected kwargs 
where passed: {...}`.
    - **Repr:** the names render joined by commas, as `ColumnAlias.__repr__` 
does: `a AS x,y`.
    
    In the Rust crate, `Column` gains `aliases(names)` for several names, as 
Scala's `as(aliases: Seq[String])`, and `Alias` gains `with_names`.
    
    ### Why are the changes needed?
    
    - Only the first name was kept, by a note that the other names "are not 
applied here". So `spark.sql("SELECT map('k1', 1) AS 
m").select(F.explode("m").alias("k", "v"))` failed on the server with 
`UDTF_ALIAS_NUMBER_MISMATCH`, where PySpark 4.3.0 gives the columns `k` and `v`.
    - Metadata took string values only, so `F.col("id").alias("x", 
metadata={"k": 1})` raised `TypeError: 'int' object is not an instance of 
'str'`.
    - An unexpected keyword argument was ignored.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, `Column.alias` behaves as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A probe of 10 alias cases gives the same reprs and errors as official 
PySpark 4.3.0. The cases cover one, two or no names, metadata with one or two 
names, explicit `None` metadata, an unexpected keyword, `name` and `explode`.
    - Against a local Spark 4.3.0 server, the two failing cases above give the 
same columns, rows and metadata as official PySpark 4.3.0.
    - New unit tests check `Column::aliases` and the render.
    - A new offline test checks the reprs and errors, and a new live test 
checks the columns and metadata. Both also pass with official PySpark 4.3.0, 
and fail without this change.
    - The lib tests and every integration test, e2e included, against the local 
server (646 passed), the build of all tests with `wasm-udf`, the offline Python 
suites (652 passed, 8 live cases skipped), and the live drop-in and Arrow 
suites (64 passed) and live cases (10 passed). Clippy reports no diagnostics in 
the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #187 from HyukjinKwon/SPARK-60981-alias-names.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/column.rs        | 84 ++++++++++++++++++++++++----------
 crates/spark-connect/src/column.rs     | 21 +++++++++
 crates/spark-connect/src/expression.rs | 28 +++++++++---
 python/tests/test_dropin_e2e.py        |  8 ++++
 python/tests/test_dropin_offline.py    | 27 +++++++++++
 5 files changed, 136 insertions(+), 32 deletions(-)

diff --git a/crates/pyspark-rs/src/column.rs b/crates/pyspark-rs/src/column.rs
index 9472c6d..a4d1614 100644
--- a/crates/pyspark-rs/src/column.rs
+++ b/crates/pyspark-rs/src/column.rs
@@ -3,6 +3,7 @@
 use pyo3::prelude::*;
 use pyo3::types::PyDict;
 use spark_connect::column::Column;
+use spark_connect::expression::{Alias, ExpressionNode};
 
 use crate::functions::to_column;
 
@@ -63,37 +64,70 @@ fn py_obj_to_column(obj: &Bound<'_, PyAny>) -> 
PyResult<Column> {
 
 #[pymethods]
 impl PyColumn {
-    /// Alias the column, optionally attaching column metadata (a dict), 
mirroring
-    /// `Column.alias(name, metadata=...)`.
+    /// Alias the column with each given name, optionally attaching column 
metadata (a
+    /// dict). Mirrors `Column.alias(*alias, **kwargs)`, which names a 
generator's columns in
+    /// order, takes metadata for a single name only, and sends it as 
`json.dumps(metadata)`.
     #[pyo3(signature = (*alias, **kwargs))]
-    fn alias(&self, alias: Vec<String>, kwargs: Option<&Bound<'_, PyDict>>) -> 
PyResult<PyColumn> {
-        // Mirrors Column.alias(*alias, **kwargs): the common single-name form 
aliases the
-        // column; `metadata` (a dict) may be passed via kwargs. Extra names 
beyond the first
-        // (multi-output aliasing) are not applied here.
-        let metadata: Option<std::collections::HashMap<String, String>> = 
match kwargs {
-            Some(kw) => match kw.get_item("metadata")? {
-                Some(v) if !v.is_none() => Some(v.extract()?),
-                _ => None,
-            },
-            None => None,
-        };
-        let name = match alias.first() {
-            Some(n) => n.clone(),
-            None => return Ok(PyColumn::new(self.column.clone())),
-        };
-        Ok(match metadata {
-            None => PyColumn::new(self.column.clone().alias(&name)),
-            Some(m) => {
-                let md: std::collections::BTreeMap<String, String> = 
m.into_iter().collect();
-                PyColumn::new(self.column.clone().alias_with_metadata(&name, 
md))
+    fn alias(
+        &self,
+        py: Python<'_>,
+        alias: Vec<String>,
+        kwargs: Option<&Bound<'_, PyDict>>,
+    ) -> PyResult<PyColumn> {
+        let mut metadata = None;
+        if let Some(kwargs) = kwargs {
+            let rest = kwargs.copy()?;
+            if let Some(value) = rest.get_item("metadata")? {
+                rest.del_item("metadata")?;
+                if !value.is_none() {
+                    metadata = Some(value);
+                }
+            }
+            if !rest.is_empty() {
+                return Err(pyo3::exceptions::PyAssertionError::new_err(format!(
+                    "Unexpected kwargs where passed: {}",
+                    rest.repr()?
+                )));
             }
-        })
+        }
+        if alias.len() > 1 && metadata.is_some() {
+            return Err(raise_pyspark(
+                py,
+                "PySparkValueError",
+                "ONLY_ALLOWED_FOR_SINGLE_COLUMN",
+                &[("arg_name", "metadata")],
+            ));
+        }
+        let single = alias.len() == 1;
+        let mut node = Alias::with_names(self.column.expression().clone(), 
alias);
+        if let Some(metadata) = metadata {
+            if single {
+                let json = py.import("json")?.call_method1("dumps", 
(metadata,))?;
+                node = node.with_metadata(json.extract()?);
+            } else if metadata.is_truthy()? {
+                // The reference client raises this when it builds the plan of 
the alias.
+                return Err(raise_pyspark(
+                    py,
+                    "PySparkValueError",
+                    "CANNOT_PROVIDE_METADATA",
+                    &[],
+                ));
+            }
+        }
+        Ok(PyColumn::new(Column::new(
+            ExpressionNode::Alias(Box::new(node)).into(),
+        )))
     }
 
     /// Alias for `alias` (pyspark `Column.name`).
     #[pyo3(signature = (*alias, **kwargs))]
-    fn name(&self, alias: Vec<String>, kwargs: Option<&Bound<'_, PyDict>>) -> 
PyResult<PyColumn> {
-        self.alias(alias, kwargs)
+    fn name(
+        &self,
+        py: Python<'_>,
+        alias: Vec<String>,
+        kwargs: Option<&Bound<'_, PyDict>>,
+    ) -> PyResult<PyColumn> {
+        self.alias(py, alias, kwargs)
     }
 
     /// Apply a transformation function to this column. Mirrors 
`Column.transform(f)`,
diff --git a/crates/spark-connect/src/column.rs 
b/crates/spark-connect/src/column.rs
index e75fd02..ab736ce 100644
--- a/crates/spark-connect/src/column.rs
+++ b/crates/spark-connect/src/column.rs
@@ -54,6 +54,14 @@ impl Column {
         }
     }
 
+    /// `Column.alias(*alias)` of several names, such as the key and the value 
of a map's
+    /// `explode`. Mirrors Scala's `Column.as(aliases: Seq[String])`.
+    pub fn aliases<S: Into<String>>(self, names: impl IntoIterator<Item = S>) 
-> Column {
+        Column {
+            expr: ExpressionNode::Alias(Box::new(Alias::with_names(self.expr, 
names))).into(),
+        }
+    }
+
     /// Mirrors `pyspark.sql.column.Column.name` (alias for `alias`).
     pub fn name(self, name: &str) -> Column {
         self.alias(name)
@@ -656,6 +664,19 @@ pub fn when(condition: Column, value: Column) -> Column {
 mod tests {
     use super::*;
 
+    #[test]
+    fn aliases_name_every_column_of_a_generator() {
+        let column = crate::functions::explode(col("m")).aliases(["k", "v"]);
+        assert_eq!(column.expression().render(), "explode(m) AS k,v");
+        match column.to_proto().expr_type {
+            Some(spark_connect_proto::expression::ExprType::Alias(alias)) => {
+                assert_eq!(alias.name, vec!["k".to_string(), "v".to_string()]);
+                assert_eq!(alias.metadata, None);
+            }
+            other => panic!("expected an alias, got {other:?}"),
+        }
+    }
+
     #[test]
     fn test_col_creation() {
         let c = col("x");
diff --git a/crates/spark-connect/src/expression.rs 
b/crates/spark-connect/src/expression.rs
index 01ca28e..54e25c5 100644
--- a/crates/spark-connect/src/expression.rs
+++ b/crates/spark-connect/src/expression.rs
@@ -136,13 +136,9 @@ impl Expression {
                 Some(t) => t.clone(),
                 None => "*".to_string(),
             },
+            // As `ColumnAlias.__repr__`: the names joined by commas.
             ExpressionNode::Alias(alias) => {
-                let name = if alias.names.len() == 1 {
-                    alias.names[0].clone()
-                } else {
-                    format!("({})", alias.names.join(", "))
-                };
-                format!("{} AS {}", alias.child.render(), name)
+                format!("{} AS {}", alias.child.render(), 
alias.names.join(","))
             }
             ExpressionNode::Cast(cast) => {
                 let type_str = match &cast.target {
@@ -609,6 +605,19 @@ impl Alias {
         }
     }
 
+    /// An alias of several names, or of none, as the reference client's 
`ColumnAlias`
+    /// takes them: a generator's columns are named in order.
+    pub fn with_names<S: Into<String>>(
+        child: Expression,
+        names: impl IntoIterator<Item = S>,
+    ) -> Self {
+        Self {
+            child,
+            names: names.into_iter().map(Into::into).collect(),
+            metadata: None,
+        }
+    }
+
     pub fn with_metadata(mut self, metadata: String) -> Self {
         self.metadata = Some(metadata);
         self
@@ -1416,7 +1425,12 @@ mod tests {
             metadata: None,
         };
         let expr: Expression = ExpressionNode::Alias(Box::new(alias)).into();
-        assert_eq!(expr.render(), "x AS (a, b)");
+        // As `ColumnAlias.__repr__`: the names joined by commas.
+        assert_eq!(expr.render(), "x AS a,b");
+        let none: Expression =
+            ExpressionNode::Alias(Box::new(Alias::with_names(col("x"), 
Vec::<String>::new())))
+                .into();
+        assert_eq!(none.render(), "x AS ");
     }
 
     #[test]
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 833b357..46c3a3a 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -613,6 +613,14 @@ def 
test_sql_keyword_arguments_are_not_parameters_as_pyspark(session):
     assert error.value.getCondition() == "UNBOUND_SQL_PARAMETER"
 
 
+def test_column_alias_names_every_column_and_keeps_its_metadata(session):
+    df = session.sql("SELECT map('k1', 1) AS 
m").select(F.explode("m").alias("k", "v"))
+    assert df.columns == ["k", "v"]
+    assert [tuple(r) for r in df.collect()] == [("k1", 1)]
+    df = session.range(1).select(F.col("id").alias("x", metadata={"k": 1, "n": 
[1, 2], "s": "v"}))
+    assert df.schema.fields[0].metadata == {"k": 1, "n": [1, 2], "s": "v"}
+
+
 def test_maps_keep_their_entries_in_order_with_keys_of_their_type(session):
     row = session.range(1).selectExpr(
         "map(2, 'b', 1, 'a') AS ints",
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index b7b8683..e6a6b3f 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1876,6 +1876,33 @@ def test_functions_pad_functions():
     assert type(F.rpad(str_col, 10, " ")).__name__ == "Column"
 
 
[email protected]("build,expected", [
+    # As Column.alias(*alias, **kwargs) of PySpark: every name, metadata for 
one name only.
+    (lambda c: c.alias("x"), "a AS x"),
+    (lambda c: c.alias("x", "y"), "a AS x,y"),
+    (lambda c: c.name("x", "y"), "a AS x,y"),
+    (lambda c: c.alias(), "a AS "),
+    (lambda c: c.alias("x", metadata={"k": 1, "n": [1, 2], "s": "v"}), "a AS 
x"),
+    (lambda c: c.alias("x", "y", metadata=None), "a AS x,y"),
+])
+def test_column_alias_takes_every_name_as_pyspark(build, expected):
+    from pyspark.sql import functions as F
+
+    assert repr(build(F.col("a"))) == f"Column<'{expected}'>"
+
+
+def test_column_alias_errors_as_pyspark():
+    from pyspark.errors import PySparkValueError
+    from pyspark.sql import functions as F
+
+    with pytest.raises(PySparkValueError) as error:
+        F.col("a").alias("x", "y", metadata={"k": 1})
+    assert error.value.getCondition() == "ONLY_ALLOWED_FOR_SINGLE_COLUMN"
+    assert error.value.getMessageParameters() == {"arg_name": "metadata"}
+    with pytest.raises(AssertionError, match=r"^Unexpected kwargs where 
passed: \{'foo': 1\}$"):
+        F.col("a").alias("x", foo=1)
+
+
 def test_functions_instr():
     """Test instr function."""
     from pyspark.sql import functions as F


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to