This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 0a5ad39 [SPARK-60981][CONNECT][PYTHON] Alias a Column with several
names, and with metadata of any JSON values, as PySpark does
0a5ad39 is described below
commit 0a5ad394b6d0ebbb541ee91a9b099d779ee7be4b
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 13:17:49 2026 +0900
[SPARK-60981][CONNECT][PYTHON] Alias a Column with several names, and with
metadata of any JSON values, as PySpark does
### What changes were proposed in this pull request?
[SPARK-60981](https://issues.apache.org/jira/browse/SPARK-60981):
`Column.alias(*alias, **kwargs)`, and `Column.name`, now follow PySpark's
`Expression.alias` and `ColumnAlias`:
- **Names:** every given name is kept, so all the columns of a generator
are named. For example, `F.explode("m").alias("k", "v")` names the key and the
value of a map.
- **Metadata:** metadata is sent as `json.dumps(metadata)`, so its values
can be of any JSON type, such as `metadata={"k": 1, "n": [1, 2]}`.
- **Errors, as in PySpark:**
- metadata with more than one name raises `PySparkValueError`
`ONLY_ALLOWED_FOR_SINGLE_COLUMN`;
- another keyword argument raises `AssertionError: Unexpected kwargs
where passed: {...}`.
- **Repr:** the names render joined by commas, as `ColumnAlias.__repr__`
does: `a AS x,y`.
In the Rust crate, `Column` gains `aliases(names)` for several names, as
Scala's `as(aliases: Seq[String])`, and `Alias` gains `with_names`.
### Why are the changes needed?
- Only the first name was kept, by a note that the other names "are not
applied here". So `spark.sql("SELECT map('k1', 1) AS
m").select(F.explode("m").alias("k", "v"))` failed on the server with
`UDTF_ALIAS_NUMBER_MISMATCH`, where PySpark 4.3.0 gives the columns `k` and `v`.
- Metadata took string values only, so `F.col("id").alias("x",
metadata={"k": 1})` raised `TypeError: 'int' object is not an instance of
'str'`.
- An unexpected keyword argument was ignored.
### Does this PR introduce _any_ user-facing change?
Yes, `Column.alias` behaves as in PySpark 4.3.0.
### How was this patch tested?
- A probe of 10 alias cases gives the same reprs and errors as official
PySpark 4.3.0. The cases cover one, two or no names, metadata with one or two
names, explicit `None` metadata, an unexpected keyword, `name` and `explode`.
- Against a local Spark 4.3.0 server, the two failing cases above give the
same columns, rows and metadata as official PySpark 4.3.0.
- New unit tests check `Column::aliases` and the render.
- A new offline test checks the reprs and errors, and a new live test
checks the columns and metadata. Both also pass with official PySpark 4.3.0,
and fail without this change.
- The lib tests and every integration test, e2e included, against the local
server (646 passed), the build of all tests with `wasm-udf`, the offline Python
suites (652 passed, 8 live cases skipped), and the live drop-in and Arrow
suites (64 passed) and live cases (10 passed). Clippy reports no diagnostics in
the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #187 from HyukjinKwon/SPARK-60981-alias-names.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/column.rs | 84 ++++++++++++++++++++++++----------
crates/spark-connect/src/column.rs | 21 +++++++++
crates/spark-connect/src/expression.rs | 28 +++++++++---
python/tests/test_dropin_e2e.py | 8 ++++
python/tests/test_dropin_offline.py | 27 +++++++++++
5 files changed, 136 insertions(+), 32 deletions(-)
diff --git a/crates/pyspark-rs/src/column.rs b/crates/pyspark-rs/src/column.rs
index 9472c6d..a4d1614 100644
--- a/crates/pyspark-rs/src/column.rs
+++ b/crates/pyspark-rs/src/column.rs
@@ -3,6 +3,7 @@
use pyo3::prelude::*;
use pyo3::types::PyDict;
use spark_connect::column::Column;
+use spark_connect::expression::{Alias, ExpressionNode};
use crate::functions::to_column;
@@ -63,37 +64,70 @@ fn py_obj_to_column(obj: &Bound<'_, PyAny>) ->
PyResult<Column> {
#[pymethods]
impl PyColumn {
- /// Alias the column, optionally attaching column metadata (a dict),
mirroring
- /// `Column.alias(name, metadata=...)`.
+ /// Alias the column with each given name, optionally attaching column
metadata (a
+ /// dict). Mirrors `Column.alias(*alias, **kwargs)`, which names a
generator's columns in
+ /// order, takes metadata for a single name only, and sends it as
`json.dumps(metadata)`.
#[pyo3(signature = (*alias, **kwargs))]
- fn alias(&self, alias: Vec<String>, kwargs: Option<&Bound<'_, PyDict>>) ->
PyResult<PyColumn> {
- // Mirrors Column.alias(*alias, **kwargs): the common single-name form
aliases the
- // column; `metadata` (a dict) may be passed via kwargs. Extra names
beyond the first
- // (multi-output aliasing) are not applied here.
- let metadata: Option<std::collections::HashMap<String, String>> =
match kwargs {
- Some(kw) => match kw.get_item("metadata")? {
- Some(v) if !v.is_none() => Some(v.extract()?),
- _ => None,
- },
- None => None,
- };
- let name = match alias.first() {
- Some(n) => n.clone(),
- None => return Ok(PyColumn::new(self.column.clone())),
- };
- Ok(match metadata {
- None => PyColumn::new(self.column.clone().alias(&name)),
- Some(m) => {
- let md: std::collections::BTreeMap<String, String> =
m.into_iter().collect();
- PyColumn::new(self.column.clone().alias_with_metadata(&name,
md))
+ fn alias(
+ &self,
+ py: Python<'_>,
+ alias: Vec<String>,
+ kwargs: Option<&Bound<'_, PyDict>>,
+ ) -> PyResult<PyColumn> {
+ let mut metadata = None;
+ if let Some(kwargs) = kwargs {
+ let rest = kwargs.copy()?;
+ if let Some(value) = rest.get_item("metadata")? {
+ rest.del_item("metadata")?;
+ if !value.is_none() {
+ metadata = Some(value);
+ }
+ }
+ if !rest.is_empty() {
+ return Err(pyo3::exceptions::PyAssertionError::new_err(format!(
+ "Unexpected kwargs where passed: {}",
+ rest.repr()?
+ )));
}
- })
+ }
+ if alias.len() > 1 && metadata.is_some() {
+ return Err(raise_pyspark(
+ py,
+ "PySparkValueError",
+ "ONLY_ALLOWED_FOR_SINGLE_COLUMN",
+ &[("arg_name", "metadata")],
+ ));
+ }
+ let single = alias.len() == 1;
+ let mut node = Alias::with_names(self.column.expression().clone(),
alias);
+ if let Some(metadata) = metadata {
+ if single {
+ let json = py.import("json")?.call_method1("dumps",
(metadata,))?;
+ node = node.with_metadata(json.extract()?);
+ } else if metadata.is_truthy()? {
+ // The reference client raises this when it builds the plan of
the alias.
+ return Err(raise_pyspark(
+ py,
+ "PySparkValueError",
+ "CANNOT_PROVIDE_METADATA",
+ &[],
+ ));
+ }
+ }
+ Ok(PyColumn::new(Column::new(
+ ExpressionNode::Alias(Box::new(node)).into(),
+ )))
}
/// Alias for `alias` (pyspark `Column.name`).
#[pyo3(signature = (*alias, **kwargs))]
- fn name(&self, alias: Vec<String>, kwargs: Option<&Bound<'_, PyDict>>) ->
PyResult<PyColumn> {
- self.alias(alias, kwargs)
+ fn name(
+ &self,
+ py: Python<'_>,
+ alias: Vec<String>,
+ kwargs: Option<&Bound<'_, PyDict>>,
+ ) -> PyResult<PyColumn> {
+ self.alias(py, alias, kwargs)
}
/// Apply a transformation function to this column. Mirrors
`Column.transform(f)`,
diff --git a/crates/spark-connect/src/column.rs
b/crates/spark-connect/src/column.rs
index e75fd02..ab736ce 100644
--- a/crates/spark-connect/src/column.rs
+++ b/crates/spark-connect/src/column.rs
@@ -54,6 +54,14 @@ impl Column {
}
}
+ /// `Column.alias(*alias)` of several names, such as the key and the value
of a map's
+ /// `explode`. Mirrors Scala's `Column.as(aliases: Seq[String])`.
+ pub fn aliases<S: Into<String>>(self, names: impl IntoIterator<Item = S>)
-> Column {
+ Column {
+ expr: ExpressionNode::Alias(Box::new(Alias::with_names(self.expr,
names))).into(),
+ }
+ }
+
/// Mirrors `pyspark.sql.column.Column.name` (alias for `alias`).
pub fn name(self, name: &str) -> Column {
self.alias(name)
@@ -656,6 +664,19 @@ pub fn when(condition: Column, value: Column) -> Column {
mod tests {
use super::*;
+ #[test]
+ fn aliases_name_every_column_of_a_generator() {
+ let column = crate::functions::explode(col("m")).aliases(["k", "v"]);
+ assert_eq!(column.expression().render(), "explode(m) AS k,v");
+ match column.to_proto().expr_type {
+ Some(spark_connect_proto::expression::ExprType::Alias(alias)) => {
+ assert_eq!(alias.name, vec!["k".to_string(), "v".to_string()]);
+ assert_eq!(alias.metadata, None);
+ }
+ other => panic!("expected an alias, got {other:?}"),
+ }
+ }
+
#[test]
fn test_col_creation() {
let c = col("x");
diff --git a/crates/spark-connect/src/expression.rs
b/crates/spark-connect/src/expression.rs
index 01ca28e..54e25c5 100644
--- a/crates/spark-connect/src/expression.rs
+++ b/crates/spark-connect/src/expression.rs
@@ -136,13 +136,9 @@ impl Expression {
Some(t) => t.clone(),
None => "*".to_string(),
},
+ // As `ColumnAlias.__repr__`: the names joined by commas.
ExpressionNode::Alias(alias) => {
- let name = if alias.names.len() == 1 {
- alias.names[0].clone()
- } else {
- format!("({})", alias.names.join(", "))
- };
- format!("{} AS {}", alias.child.render(), name)
+ format!("{} AS {}", alias.child.render(),
alias.names.join(","))
}
ExpressionNode::Cast(cast) => {
let type_str = match &cast.target {
@@ -609,6 +605,19 @@ impl Alias {
}
}
+ /// An alias of several names, or of none, as the reference client's
`ColumnAlias`
+ /// takes them: a generator's columns are named in order.
+ pub fn with_names<S: Into<String>>(
+ child: Expression,
+ names: impl IntoIterator<Item = S>,
+ ) -> Self {
+ Self {
+ child,
+ names: names.into_iter().map(Into::into).collect(),
+ metadata: None,
+ }
+ }
+
pub fn with_metadata(mut self, metadata: String) -> Self {
self.metadata = Some(metadata);
self
@@ -1416,7 +1425,12 @@ mod tests {
metadata: None,
};
let expr: Expression = ExpressionNode::Alias(Box::new(alias)).into();
- assert_eq!(expr.render(), "x AS (a, b)");
+ // As `ColumnAlias.__repr__`: the names joined by commas.
+ assert_eq!(expr.render(), "x AS a,b");
+ let none: Expression =
+ ExpressionNode::Alias(Box::new(Alias::with_names(col("x"),
Vec::<String>::new())))
+ .into();
+ assert_eq!(none.render(), "x AS ");
}
#[test]
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 833b357..46c3a3a 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -613,6 +613,14 @@ def
test_sql_keyword_arguments_are_not_parameters_as_pyspark(session):
assert error.value.getCondition() == "UNBOUND_SQL_PARAMETER"
+def test_column_alias_names_every_column_and_keeps_its_metadata(session):
+ df = session.sql("SELECT map('k1', 1) AS
m").select(F.explode("m").alias("k", "v"))
+ assert df.columns == ["k", "v"]
+ assert [tuple(r) for r in df.collect()] == [("k1", 1)]
+ df = session.range(1).select(F.col("id").alias("x", metadata={"k": 1, "n":
[1, 2], "s": "v"}))
+ assert df.schema.fields[0].metadata == {"k": 1, "n": [1, 2], "s": "v"}
+
+
def test_maps_keep_their_entries_in_order_with_keys_of_their_type(session):
row = session.range(1).selectExpr(
"map(2, 'b', 1, 'a') AS ints",
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index b7b8683..e6a6b3f 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1876,6 +1876,33 @@ def test_functions_pad_functions():
assert type(F.rpad(str_col, 10, " ")).__name__ == "Column"
[email protected]("build,expected", [
+ # As Column.alias(*alias, **kwargs) of PySpark: every name, metadata for
one name only.
+ (lambda c: c.alias("x"), "a AS x"),
+ (lambda c: c.alias("x", "y"), "a AS x,y"),
+ (lambda c: c.name("x", "y"), "a AS x,y"),
+ (lambda c: c.alias(), "a AS "),
+ (lambda c: c.alias("x", metadata={"k": 1, "n": [1, 2], "s": "v"}), "a AS
x"),
+ (lambda c: c.alias("x", "y", metadata=None), "a AS x,y"),
+])
+def test_column_alias_takes_every_name_as_pyspark(build, expected):
+ from pyspark.sql import functions as F
+
+ assert repr(build(F.col("a"))) == f"Column<'{expected}'>"
+
+
+def test_column_alias_errors_as_pyspark():
+ from pyspark.errors import PySparkValueError
+ from pyspark.sql import functions as F
+
+ with pytest.raises(PySparkValueError) as error:
+ F.col("a").alias("x", "y", metadata={"k": 1})
+ assert error.value.getCondition() == "ONLY_ALLOWED_FOR_SINGLE_COLUMN"
+ assert error.value.getMessageParameters() == {"arg_name": "metadata"}
+ with pytest.raises(AssertionError, match=r"^Unexpected kwargs where
passed: \{'foo': 1\}$"):
+ F.col("a").alias("x", foo=1)
+
+
def test_functions_instr():
"""Test instr function."""
from pyspark.sql import functions as F
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]