This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 59e2059 [SPARK-61011][CONNECT][PYTHON] Refer to the columns of a
DataFrame by its plan ID as PySpark does
59e2059 is described below
commit 59e2059a2f42d9daeccfb692174cee4f2011c128
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 23:12:36 2026 +0900
[SPARK-61011][CONNECT][PYTHON] Refer to the columns of a DataFrame by its
plan ID as PySpark does
### What changes were proposed in this pull request?
[SPARK-61011](https://issues.apache.org/jira/browse/SPARK-61011): the
columns of a DataFrame now refer to it by the plan ID of its plan, as PySpark
Connect's do. A DataFrame's relation is sent with that plan ID.
- `df["name"]`, `df.name` and `df._col(name)` are the `ColumnReference` of
the name and the plan ID.
- `df["*"]` is the `UnresolvedStar` of the plan ID.
- `df.metadataColumn(name)` is the metadata `ColumnReference` of it.
- `df.colRegex(regex)` is the Column of the `UnresolvedRegex` of it, rather
than a DataFrame.
The server resolves them against that plan. So `df1.join(df2, df1["id"] ==
df2["id"]).select(df1["id"], df2["w"])` selects the `id` of `df1`, and a
self-join of a DataFrame and one derived from it raises
`AMBIGUOUS_COLUMN_REFERENCE`, as for PySpark.
In the core, the DataFrame has `col`, `star` and `col_regex` for these, and
the star and regex expressions take an optional plan ID.
### Why are the changes needed?
The columns referred to their names only, so the common join above raised
`AMBIGUOUS_REFERENCE`, as did `df1.join(df2, df1.id == df2.id)`. PySpark 4.3.0
runs both. `colRegex` also returned a DataFrame rather than a Column.
### Does this PR introduce _any_ user-facing change?
Yes. The columns of DataFrames resolve as in PySpark 4.3.0, and `colRegex`
returns a Column. In the Rust API, `DataFrame::col_regex` returns a Column too.
### How was this patch tested?
- A new live test runs joins that select the columns of each side by name,
by attribute, by `df["*"]` and by `colRegex`. It also checks the error of an
ambiguous self-join. It passes with official PySpark 4.3.0, and fails without
this change.
- A new offline test checks that the columns sent to a mock Connect server
have the plan IDs of the relations of their DataFrames. It fails without this
change.
- A new lib test checks the plan IDs of the columns, the star, the regex
and the metadata column.
- The lib tests and every integration test, e2e included, against a local
Spark 4.3.0 server (665 passed, and 125 of the transport crate), the build of
all tests with `wasm-udf`, the offline Python suites (1485 passed, 8 skipped),
and the live drop-in and Arrow suites (88 passed) and live cases (10 passed).
Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #216 from HyukjinKwon/SPARK-61011-plan-id-columns.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 34 +++---
crates/spark-connect/src/dataframe.rs | 114 ++++++++++++++++++---
crates/spark-connect/src/expression.rs | 54 +++++-----
crates/spark-connect/src/functions.rs | 8 +-
.../spark-connect/tests/dataframe_extra_golden.rs | 2 +-
crates/spark-connect/tests/e2e_coverage.rs | 2 +-
crates/spark-connect/tests/plans_golden.rs | 9 +-
python/tests/test_dropin_e2e.py | 18 ++++
python/tests/test_dropin_offline.py | 32 ++++++
9 files changed, 210 insertions(+), 63 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index dc9ead8..1bac91a 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -731,10 +731,11 @@ impl PyDataFrame {
)
}
- /// Select columns whose names match a regex.
+ /// The columns of this DataFrame whose names match a regex, as a Column.
Mirrors
+ /// `DataFrame.colRegex`.
#[pyo3(name = "colRegex")]
- fn col_regex(&self, colName: &str) -> PyDataFrame {
- PyDataFrame::new(self.dataframe.col_regex(colName))
+ fn col_regex(&self, colName: &str) -> PyColumn {
+ PyColumn::new(self.dataframe.col_regex(colName))
}
/// Basic statistics for the given columns (count/mean/stddev/min/max).
@@ -1242,15 +1243,16 @@ impl PyDataFrame {
Ok(PyDataFrame::new(df))
}
- /// Column access by attribute (`df.colname` -> Column), mirroring
reference pyspark.
- /// Names beginning with `_` raise AttributeError so normal attribute
lookup works.
+ /// Column access by attribute (`df.colname` -> Column), mirroring
reference pyspark: the
+ /// column of this DataFrame, by its plan ID. Names beginning with `_`
raise
+ /// AttributeError so normal attribute lookup works.
fn __getattr__(&self, name: &str) -> PyResult<PyColumn> {
if name.starts_with('_') {
return Err(PyErr::new::<pyo3::exceptions::PyAttributeError, _>(
name.to_string(),
));
}
- Ok(PyColumn::new(spark_connect::functions::col(name)))
+ Ok(PyColumn::new(self.dataframe.col(name)))
}
/// Class attributes and all column names, matching Python Connect.
@@ -1603,14 +1605,21 @@ impl PyDataFrame {
}
}
- /// As PySpark's `DataFrame.__getitem__`: a column by name (`df["col"]`)
or position
- /// (`df[0]`), the rows that match a Column (`df[df.a > 1]`, as `filter`),
or the
- /// projection of a list or tuple of columns (`df[["a", "b"]]`, as
`select`).
+ /// As PySpark's `DataFrame.__getitem__`: a column by name (`df["col"]`),
of this DataFrame
+ /// by its plan ID, as its star of `df["*"]`, or by position (`df[0]`),
the rows that match
+ /// a Column (`df[df.a > 1]`, as `filter`), or the projection of a list or
tuple of columns
+ /// (`df[["a", "b"]]`, as `select`).
fn __getitem__(&self, py: Python<'_>, item: &Bound<'_, PyAny>) ->
PyResult<Py<PyAny>> {
use pyo3::IntoPyObjectExt;
let name = if let Ok(name) = item.cast::<pyo3::types::PyString>() {
- name.extract::<String>()?
+ let name = name.to_cow()?;
+ let column = if name == "*" {
+ self.dataframe.star()
+ } else {
+ self.dataframe.col(&name)
+ };
+ return PyColumn::new(column).into_py_any(py);
} else if item.is_instance_of::<PyColumn>() {
return self.filter(item)?.into_py_any(py);
} else if item.is_instance_of::<PyList>() ||
item.is_instance_of::<PyTuple>() {
@@ -1641,9 +1650,10 @@ impl PyDataFrame {
}
/// Private column accessor used by pandas-on-Spark (`sdf._col(name)`);
mirrors the
- /// connect DataFrame's `_col`. Backtick-quoted names arrive from
`scol_for`.
+ /// connect DataFrame's `_col`, the column of this DataFrame by its plan
ID. Backtick-quoted
+ /// names arrive from `scol_for`.
fn _col(&self, item: &str) -> PyColumn {
- PyColumn::new(spark_connect::functions::col(item))
+ PyColumn::new(self.dataframe.col(item))
}
/// As PySpark's `DataFrame.__repr__`: the table of `_show_string` when
the session's
diff --git a/crates/spark-connect/src/dataframe.rs
b/crates/spark-connect/src/dataframe.rs
index b04de08..6eb2b14 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1500,25 +1500,41 @@ impl DataFrame {
DataFrame::new(self.session.clone(), plan)
}
- /// Select columns by regex pattern.
- pub fn col_regex(&self, col_name: &str) -> DataFrame {
- let plan = LogicalPlan::ColRegex {
- input: Arc::clone(&self.plan),
- col_name: col_name.to_string(),
- };
- DataFrame::new(self.session.clone(), plan)
+ /// The plan ID of this DataFrame's plan in its session, by which its
columns refer to it.
+ fn plan_id(&self) -> i64 {
+ self.session.plan_ids().id_of(&self.plan)
}
- /// Select a metadata column by name. Mirrors
`pyspark.sql.DataFrame.metadataColumn`.
- pub fn metadata_column(&self, name: &str) -> Column {
+ /// The column `name` of this DataFrame. It refers to this DataFrame by
its plan ID, so that
+ /// the server resolves it against this DataFrame's plan, as in a join of
DataFrames of the
+ /// same column names. Mirrors `df[name]`, `df.name` and `DataFrame._col`
of PySpark Connect.
+ pub fn col(&self, name: &str) -> Column {
+ let reference =
crate::expression::ColumnReference::new(name).with_plan_id(self.plan_id());
+ Column::new(ExpressionNode::ColumnReference(reference).into())
+ }
+
+ /// The star of the columns of this DataFrame, as `df["*"]` of PySpark
Connect.
+ pub fn star(&self) -> Column {
+ Column::new(ExpressionNode::UnresolvedStar(None,
Some(self.plan_id())).into())
+ }
+
+ /// The columns of this DataFrame whose names match the regex `col_name`,
in backticks, as
+ /// a Column. Mirrors `DataFrame.colRegex`.
+ pub fn col_regex(&self, col_name: &str) -> Column {
Column::new(
- ExpressionNode::ColumnReference(
- crate::expression::ColumnReference::new(name).metadata(),
- )
- .into(),
+ ExpressionNode::UnresolvedRegex(col_name.to_string(),
Some(self.plan_id())).into(),
)
}
+ /// The metadata column `name` of this DataFrame, which refers to it by
its plan ID.
+ /// Mirrors `pyspark.sql.DataFrame.metadataColumn`.
+ pub fn metadata_column(&self, name: &str) -> Column {
+ let reference = crate::expression::ColumnReference::new(name)
+ .with_plan_id(self.plan_id())
+ .metadata();
+ Column::new(ExpressionNode::ColumnReference(reference).into())
+ }
+
/// Group by with rollup.
pub fn rollup<C: Into<Column>>(
&self,
@@ -1891,7 +1907,7 @@ impl DataFrame {
/// `DataFrame.zipWithIndex(indexColName="index")`:
/// `self.select(col("*"), distributed_sequence_id().alias(indexColName))`.
pub fn zip_with_index(&self, index_col_name: &str) -> DataFrame {
- let star = Column::new(ExpressionNode::UnresolvedStar(None).into());
+ let star = Column::new(ExpressionNode::UnresolvedStar(None,
None).into());
let seq = Column::new(Expression::function("distributed_sequence_id",
vec![]))
.alias(index_col_name);
self.select(vec![star, seq])
@@ -3841,7 +3857,7 @@ mod plan_construction_tests {
ser(&df.alias("t"));
ser(&df.sample(0.5, Some(1)));
ser(&df.select_expr(vec!["id + 1"]));
- ser(&df.col_regex("id"));
+ ser(&df.select(vec![df.col_regex("`id`")]));
ser(&df.describe(vec!["id"]));
ser(&df.summary(vec!["count"]));
ser(&df.as_table("t2"));
@@ -4435,3 +4451,71 @@ mod arrow_value_tests {
assert!(micros_to_time_string(1).contains('.'));
}
}
+
+#[cfg(test)]
+mod plan_id_column_tests {
+ use super::*;
+ use crate::session::SparkSession;
+ use proto::relation::RelType;
+
+ fn session() -> SparkSession {
+ SparkSession::builder()
+ .remote("sc://localhost:15002")
+ .get_or_create()
+ .expect("failed to build session")
+ }
+
+ /// As PySpark Connect's `df[name]`, `df["*"]`, `colRegex` and
`metadataColumn`: the columns
+ /// of a DataFrame refer to it by the plan ID that its relation is sent
with.
+ #[test]
+ fn columns_refer_to_their_dataframe_by_its_plan_id() {
+ use proto::expression::ExprType;
+
+ let spark = session();
+ let left = spark.range(2).unwrap();
+ let right = spark.range(3).unwrap();
+ let joined = left
+ .join(
+ &right,
+ Some(left.col("id").eq(right.col("id"))),
+ JoinType::Inner,
+ )
+ .select(vec![left.col("id"), right.star(),
left.col_regex("`i.*`")]);
+ let relation = joined.relation();
+ let Some(RelType::Project(project)) = &relation.rel_type else {
+ panic!("expected Project: {relation:?}");
+ };
+ let Some(RelType::Join(join)) =
&project.input.as_deref().unwrap().rel_type else {
+ panic!("expected Join");
+ };
+ let plan_id = |r: &proto::Relation| r.common.as_ref().and_then(|c|
c.plan_id);
+ let (left_id, right_id) = (
+ plan_id(join.left.as_deref().unwrap()),
+ plan_id(join.right.as_deref().unwrap()),
+ );
+ assert!(left_id.is_some() && left_id != right_id);
+ let ids: Vec<Option<i64>> = project
+ .expressions
+ .iter()
+ .map(|e| match &e.expr_type {
+ Some(ExprType::UnresolvedAttribute(a)) => a.plan_id,
+ Some(ExprType::UnresolvedStar(s)) => s.plan_id,
+ Some(ExprType::UnresolvedRegex(r)) => r.plan_id,
+ other => panic!("unexpected {other:?}"),
+ })
+ .collect();
+ assert_eq!(ids, vec![left_id, right_id, left_id]);
+ let Some(ExprType::UnresolvedAttribute(metadata)) = left
+ .metadata_column("_metadata")
+ .expression()
+ .to_proto()
+ .expr_type
+ else {
+ panic!("expected an attribute");
+ };
+ assert_eq!(
+ (metadata.plan_id, metadata.is_metadata_column),
+ (left_id, Some(true))
+ );
+ }
+}
diff --git a/crates/spark-connect/src/expression.rs
b/crates/spark-connect/src/expression.rs
index 3dcf67f..4731f87 100644
--- a/crates/spark-connect/src/expression.rs
+++ b/crates/spark-connect/src/expression.rs
@@ -59,8 +59,9 @@ pub enum ExpressionNode {
/// `pyspark.sql.connect.expressions.UnresolvedFunction` - a function call.
UnresolvedFunction(UnresolvedFunction),
/// `pyspark.sql.connect.expressions.UnresolvedStar` - a star expression
(`*`).
- /// Carries an optional target ending in `.*` (e.g. `df.*` from
`col("df.*")`).
- UnresolvedStar(Option<String>),
+ /// Carries an optional target ending in `.*` (e.g. `df.*` from
`col("df.*")`), and the plan
+ /// ID of the DataFrame of `df["*"]`, whose columns it is.
+ UnresolvedStar(Option<String>, Option<i64>),
/// `pyspark.sql.connect.expressions.ColumnAlias` / `Alias` - an aliased
expression.
Alias(Box<Alias>),
/// `pyspark.sql.connect.expressions.CastExpression` - a cast expression.
@@ -68,8 +69,9 @@ pub enum ExpressionNode {
/// `spark.connect.Expression.DirectShufflePartitionID` - wraps a child
expression
/// that evaluates to a partition id (used by `DataFrame.repartitionById`).
DirectShufflePartitionId(Expression),
- /// `pyspark.sql.connect.expressions.UnresolvedRegex` - a regex column
reference.
- UnresolvedRegex(String),
+ /// `pyspark.sql.connect.expressions.UnresolvedRegex` - a regex column
reference, of the
+ /// columns of the DataFrame of the plan ID of `df.colRegex(...)`.
+ UnresolvedRegex(String, Option<i64>),
/// `pyspark.sql.connect.expressions.SortOrder` - a sort order expression.
SortOrder(Box<SortOrder>),
/// `pyspark.sql.connect.expressions.CaseWhen` - a CASE WHEN expression.
@@ -155,7 +157,7 @@ impl Expression {
ExpressionNode::Literal(lit) => lit.render(),
ExpressionNode::ColumnReference(col_ref) => col_ref.name.clone(),
ExpressionNode::UnresolvedFunction(func) => func.render(),
- ExpressionNode::UnresolvedStar(target) => match target {
+ ExpressionNode::UnresolvedStar(target, _) => match target {
Some(t) => format!("unresolvedstar({t})"),
None => "unresolvedstar()".to_string(),
},
@@ -180,7 +182,7 @@ impl Expression {
ExpressionNode::DirectShufflePartitionId(child) => {
format!("DirectShufflePartitionID(child={})", child.render())
}
- ExpressionNode::UnresolvedRegex(col_name) =>
format!("UnresolvedRegex({col_name})"),
+ ExpressionNode::UnresolvedRegex(col_name, _) =>
format!("UnresolvedRegex({col_name})"),
ExpressionNode::SortOrder(sort) => sort.render(),
ExpressionNode::CaseWhen(case_when) => case_when.render(),
ExpressionNode::UnresolvedExtractValue(ev) => {
@@ -228,16 +230,15 @@ impl Expression {
ExpressionNode::Literal(lit) => lit.to_proto(),
ExpressionNode::ColumnReference(col_ref) => col_ref.to_proto(),
ExpressionNode::UnresolvedFunction(func) => func.to_proto(),
- ExpressionNode::UnresolvedStar(target) => {
- let mut expr = proto::Expression::default();
- expr.expr_type =
Some(proto::expression::ExprType::UnresolvedStar(
+ ExpressionNode::UnresolvedStar(target, plan_id) =>
proto::Expression {
+ expr_type: Some(proto::expression::ExprType::UnresolvedStar(
proto::expression::UnresolvedStar {
unparsed_target: target.clone(),
- plan_id: None,
+ plan_id: *plan_id,
},
- ));
- expr
- }
+ )),
+ ..Default::default()
+ },
ExpressionNode::Alias(alias) => alias.to_proto(),
ExpressionNode::Cast(cast) => cast.to_proto(),
ExpressionNode::DirectShufflePartitionId(child) => {
@@ -249,16 +250,15 @@ impl Expression {
));
expr
}
- ExpressionNode::UnresolvedRegex(col_name) => {
- let mut expr = proto::Expression::default();
- expr.expr_type =
Some(proto::expression::ExprType::UnresolvedRegex(
+ ExpressionNode::UnresolvedRegex(col_name, plan_id) =>
proto::Expression {
+ expr_type: Some(proto::expression::ExprType::UnresolvedRegex(
proto::expression::UnresolvedRegex {
col_name: col_name.clone(),
- plan_id: None,
+ plan_id: *plan_id,
},
- ));
- expr
- }
+ )),
+ ..Default::default()
+ },
ExpressionNode::SortOrder(sort) => sort.to_proto(),
ExpressionNode::CaseWhen(case_when) => case_when.to_proto(),
ExpressionNode::UnresolvedExtractValue(ev) => ev.to_proto(),
@@ -1349,7 +1349,7 @@ mod tests {
"CAST(x AS INT)"
);
assert_eq!(
- Expression::from(ExpressionNode::UnresolvedStar(None)).render(),
+ Expression::from(ExpressionNode::UnresolvedStar(None,
None)).render(),
"unresolvedstar()"
);
@@ -1531,13 +1531,14 @@ mod tests {
#[test]
fn test_expression_render_unresolved_star_with_target() {
- let expr: Expression =
ExpressionNode::UnresolvedStar(Some("table.*".to_string())).into();
+ let expr: Expression =
+ ExpressionNode::UnresolvedStar(Some("table.*".to_string()),
None).into();
assert_eq!(expr.render(), "unresolvedstar(table.*)");
}
#[test]
fn test_expression_render_unresolved_regex() {
- let expr: Expression =
ExpressionNode::UnresolvedRegex("`col_.*`".to_string()).into();
+ let expr: Expression =
ExpressionNode::UnresolvedRegex("`col_.*`".to_string(), None).into();
assert_eq!(expr.render(), "UnresolvedRegex(`col_.*`)");
}
@@ -1944,7 +1945,7 @@ mod tests {
#[test]
fn test_to_proto_unresolved_star_none() {
- let expr: Expression = ExpressionNode::UnresolvedStar(None).into();
+ let expr: Expression = ExpressionNode::UnresolvedStar(None,
None).into();
let proto = expr.to_proto();
assert!(proto.expr_type.is_some());
if let Some(proto::expression::ExprType::UnresolvedStar(star)) =
proto.expr_type {
@@ -1956,7 +1957,8 @@ mod tests {
#[test]
fn test_to_proto_unresolved_star_with_target() {
- let expr: Expression =
ExpressionNode::UnresolvedStar(Some("table.*".to_string())).into();
+ let expr: Expression =
+ ExpressionNode::UnresolvedStar(Some("table.*".to_string()),
None).into();
let proto = expr.to_proto();
assert!(proto.expr_type.is_some());
if let Some(proto::expression::ExprType::UnresolvedStar(star)) =
proto.expr_type {
@@ -1968,7 +1970,7 @@ mod tests {
#[test]
fn test_to_proto_unresolved_regex() {
- let expr: Expression =
ExpressionNode::UnresolvedRegex("`col_.*`".to_string()).into();
+ let expr: Expression =
ExpressionNode::UnresolvedRegex("`col_.*`".to_string(), None).into();
let proto = expr.to_proto();
assert!(proto.expr_type.is_some());
if let Some(proto::expression::ExprType::UnresolvedRegex(regex)) =
proto.expr_type {
diff --git a/crates/spark-connect/src/functions.rs
b/crates/spark-connect/src/functions.rs
index 49f7cdb..e13f50b 100644
--- a/crates/spark-connect/src/functions.rs
+++ b/crates/spark-connect/src/functions.rs
@@ -67,9 +67,9 @@ fn lit_binary(b: Vec<u8>) -> Expression {
/// `select("*")`, `col("t.*")` resolve on the server), matching pyspark.
pub fn col(name: &str) -> Column {
if name == "*" {
- Column::new(ExpressionNode::UnresolvedStar(None).into())
+ Column::new(ExpressionNode::UnresolvedStar(None, None).into())
} else if let Some(prefix) = name.strip_suffix(".*") {
-
Column::new(ExpressionNode::UnresolvedStar(Some(format!("{prefix}.*"))).into())
+
Column::new(ExpressionNode::UnresolvedStar(Some(format!("{prefix}.*")),
None).into())
} else {
Column::new(Expression::column(name))
}
@@ -4649,13 +4649,13 @@ mod tests {
let star = col("*");
match star.expression().node() {
- ExpressionNode::UnresolvedStar(None) => {}
+ ExpressionNode::UnresolvedStar(None, None) => {}
_ => panic!("Expected UnresolvedStar(None)"),
}
let table_star = col("t.*");
match table_star.expression().node() {
- ExpressionNode::UnresolvedStar(Some(_)) => {}
+ ExpressionNode::UnresolvedStar(Some(_), None) => {}
_ => panic!("Expected UnresolvedStar with prefix"),
}
}
diff --git a/crates/spark-connect/tests/dataframe_extra_golden.rs
b/crates/spark-connect/tests/dataframe_extra_golden.rs
index eab357a..5cc69c4 100644
--- a/crates/spark-connect/tests/dataframe_extra_golden.rs
+++ b/crates/spark-connect/tests/dataframe_extra_golden.rs
@@ -44,7 +44,7 @@ fn test_summary_golden() {
#[test]
#[ignore]
fn test_col_regex_golden() {
- // Test: df.selectExpr("id", "id as id2").col_regex("id.*")
+ // Test: df.selectExpr("id", "id as id2").select(col_regex("id.*"))
}
#[test]
diff --git a/crates/spark-connect/tests/e2e_coverage.rs
b/crates/spark-connect/tests/e2e_coverage.rs
index b76db10..d1b949a 100644
--- a/crates/spark-connect/tests/e2e_coverage.rs
+++ b/crates/spark-connect/tests/e2e_coverage.rs
@@ -681,7 +681,7 @@ fn dataframe_more_surface() {
.unwrap();
let _ = df.to_df(vec!["a", "b", "c"]).collect().unwrap();
let _ = df.sample(0.5, Some(1)).count().unwrap();
- let _ = df.col_regex("`.*`").collect().unwrap();
+ let _ = df.select(vec![df.col_regex("`.*`")]).collect().unwrap();
// cache / persist / unpersist round-trip.
let cached = df.cache().unwrap();
diff --git a/crates/spark-connect/tests/plans_golden.rs
b/crates/spark-connect/tests/plans_golden.rs
index 1f551cf..1768b25 100644
--- a/crates/spark-connect/tests/plans_golden.rs
+++ b/crates/spark-connect/tests/plans_golden.rs
@@ -297,8 +297,9 @@ fn build(name: &str) -> Option<LogicalPlan> {
}
"select_star" => {
let range = plan::range(0, 10, 1);
- let star_col =
-
spark_connect::column::Column::new(ExpressionNode::UnresolvedStar(None).into());
+ let star_col = spark_connect::column::Column::new(
+ ExpressionNode::UnresolvedStar(None, None).into(),
+ );
plan::project(range, vec![star_col])
}
"select_alias" => {
@@ -393,7 +394,7 @@ fn build(name: &str) -> Option<LogicalPlan> {
.clone();
let agg_count =
spark_connect::column::Column::new(Expression::function(
"count",
- vec![ExpressionNode::UnresolvedStar(None).into()],
+ vec![ExpressionNode::UnresolvedStar(None, None).into()],
))
.alias("c")
.expression()
@@ -604,7 +605,7 @@ fn build(name: &str) -> Option<LogicalPlan> {
"col_regex" => {
let range = plan::range(0, 10, 1);
let regex_col = spark_connect::column::Column::new(
- ExpressionNode::UnresolvedRegex("`id`".to_string()).into(),
+ ExpressionNode::UnresolvedRegex("`id`".to_string(),
None).into(),
);
plan::project(range, vec![regex_col])
}
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 5eeee7c..64217f5 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -609,6 +609,24 @@ def
test_grouped_map_functions_take_iterators_by_their_type_hints(session):
assert sorted(tuple(r) for r in result.collect()) == expected
+def test_dataframe_columns_resolve_against_their_dataframe_as_pyspark(session):
+ # As PySpark Connect's: df[name] and df.name refer to the columns of their
DataFrame by its
+ # plan ID, so a join selects the columns of each side, and an ambiguous
self-join of a
+ # derived DataFrame raises PySpark's error.
+ left = session.createDataFrame([(1, "a"), (2, "b")], "id int, v string")
+ right = session.createDataFrame([(1, "x"), (3, "y")], "id int, w string")
+ joined = left.join(right, left["id"] == right["id"])
+ assert joined.select(left["id"], right["w"]).collect() == [Row(id=1,
w="x")]
+ assert joined.select(left.v, right.w).collect() == [Row(v="a", w="x")]
+ assert joined.select(right["*"]).collect() == [Row(id=1, w="x")]
+ assert joined.select(left.colRegex("`v`")).collect() == [Row(v="a")]
+ base = session.range(3)
+ derived = base.withColumn("x", F.col("id") * 2)
+ with pytest.raises(AnalysisException) as error:
+ base.join(derived, base["id"] ==
derived["id"]).select(base["id"]).collect()
+ assert error.value.getCondition() == "AMBIGUOUS_COLUMN_REFERENCE"
+
+
def test_collect_reads_a_result_across_stream_ends(session):
# Takes several response streams when the server's senderMaxStreamSize is
small.
n = 300_000
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index ce58183..4e12e66 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1389,6 +1389,38 @@ def test_broadcast_type_check():
F.broadcast("not a dataframe")
+def test_dataframe_columns_refer_to_it_by_its_plan_id_as_pyspark(mock_connect):
+ # As PySpark Connect's df[name], df.name, df["*"] and colRegex: the
columns of a DataFrame
+ # refer to it by the plan ID that its relation is sent with, and colRegex
is a Column.
+ from pyspark.sql import Column
+
+ spark, service = mock_connect
+ left, right = spark.range(2), spark.range(3)
+ assert isinstance(left.colRegex("`i.*`"), Column)
+ left.join(right, left["id"] == right.id).select(
+ left.id, right["*"], left.colRegex("`i.*`"), left["id"]
+ ).collect()
+ # ExecutePlanRequest.plan=3, Plan.root=1, Relation.project=3: input=1,
expressions=3;
+ # Relation.join=5: left=1, right=2; Relation.common=1,
RelationCommon.plan_id=2.
+ plan =
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+ project = parse_message(parse_message(plan)[3][0])
+ join = parse_message(parse_message(project[1][0])[5][0])
+
+ def plan_id(relation):
+ return parse_message(parse_message(relation)[1][0])[2][0]
+
+ left_id, right_id = plan_id(join[1][0]), plan_id(join[2][0])
+ assert left_id != right_id
+ # Expression.unresolved_attribute=2 (plan_id=2), unresolved_star=5
(plan_id=2),
+ # unresolved_regex=8 (plan_id=2).
+ ids = []
+ for expression in project[3]:
+ expression = parse_message(expression)
+ (field,) = set(expression) & {2, 5, 8}
+ ids.append(parse_message(expression[field][0])[2][0])
+ assert ids == [left_id, right_id, left_id, left_id]
+
+
def test_row_count_and_index():
from pyspark.sql import Row
r = Row(a=1, b=2, c=1)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]