This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 59e2059  [SPARK-61011][CONNECT][PYTHON] Refer to the columns of a 
DataFrame by its plan ID as PySpark does
59e2059 is described below

commit 59e2059a2f42d9daeccfb692174cee4f2011c128
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 23:12:36 2026 +0900

    [SPARK-61011][CONNECT][PYTHON] Refer to the columns of a DataFrame by its 
plan ID as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61011](https://issues.apache.org/jira/browse/SPARK-61011): the 
columns of a DataFrame now refer to it by the plan ID of its plan, as PySpark 
Connect's do. A DataFrame's relation is sent with that plan ID.
    
    - `df["name"]`, `df.name` and `df._col(name)` are the `ColumnReference` of 
the name and the plan ID.
    - `df["*"]` is the `UnresolvedStar` of the plan ID.
    - `df.metadataColumn(name)` is the metadata `ColumnReference` of it.
    - `df.colRegex(regex)` is the Column of the `UnresolvedRegex` of it, rather 
than a DataFrame.
    
    The server resolves them against that plan. So `df1.join(df2, df1["id"] == 
df2["id"]).select(df1["id"], df2["w"])` selects the `id` of `df1`, and a 
self-join of a DataFrame and one derived from it raises 
`AMBIGUOUS_COLUMN_REFERENCE`, as for PySpark.
    
    In the core, the DataFrame has `col`, `star` and `col_regex` for these, and 
the star and regex expressions take an optional plan ID.
    
    ### Why are the changes needed?
    
    The columns referred to their names only, so the common join above raised 
`AMBIGUOUS_REFERENCE`, as did `df1.join(df2, df1.id == df2.id)`. PySpark 4.3.0 
runs both. `colRegex` also returned a DataFrame rather than a Column.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. The columns of DataFrames resolve as in PySpark 4.3.0, and `colRegex` 
returns a Column. In the Rust API, `DataFrame::col_regex` returns a Column too.
    
    ### How was this patch tested?
    
    - A new live test runs joins that select the columns of each side by name, 
by attribute, by `df["*"]` and by `colRegex`. It also checks the error of an 
ambiguous self-join. It passes with official PySpark 4.3.0, and fails without 
this change.
    - A new offline test checks that the columns sent to a mock Connect server 
have the plan IDs of the relations of their DataFrames. It fails without this 
change.
    - A new lib test checks the plan IDs of the columns, the star, the regex 
and the metadata column.
    - The lib tests and every integration test, e2e included, against a local 
Spark 4.3.0 server (665 passed, and 125 of the transport crate), the build of 
all tests with `wasm-udf`, the offline Python suites (1485 passed, 8 skipped), 
and the live drop-in and Arrow suites (88 passed) and live cases (10 passed). 
Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #216 from HyukjinKwon/SPARK-61011-plan-id-columns.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs                 |  34 +++---
 crates/spark-connect/src/dataframe.rs              | 114 ++++++++++++++++++---
 crates/spark-connect/src/expression.rs             |  54 +++++-----
 crates/spark-connect/src/functions.rs              |   8 +-
 .../spark-connect/tests/dataframe_extra_golden.rs  |   2 +-
 crates/spark-connect/tests/e2e_coverage.rs         |   2 +-
 crates/spark-connect/tests/plans_golden.rs         |   9 +-
 python/tests/test_dropin_e2e.py                    |  18 ++++
 python/tests/test_dropin_offline.py                |  32 ++++++
 9 files changed, 210 insertions(+), 63 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index dc9ead8..1bac91a 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -731,10 +731,11 @@ impl PyDataFrame {
         )
     }
 
-    /// Select columns whose names match a regex.
+    /// The columns of this DataFrame whose names match a regex, as a Column. 
Mirrors
+    /// `DataFrame.colRegex`.
     #[pyo3(name = "colRegex")]
-    fn col_regex(&self, colName: &str) -> PyDataFrame {
-        PyDataFrame::new(self.dataframe.col_regex(colName))
+    fn col_regex(&self, colName: &str) -> PyColumn {
+        PyColumn::new(self.dataframe.col_regex(colName))
     }
 
     /// Basic statistics for the given columns (count/mean/stddev/min/max).
@@ -1242,15 +1243,16 @@ impl PyDataFrame {
         Ok(PyDataFrame::new(df))
     }
 
-    /// Column access by attribute (`df.colname` -> Column), mirroring 
reference pyspark.
-    /// Names beginning with `_` raise AttributeError so normal attribute 
lookup works.
+    /// Column access by attribute (`df.colname` -> Column), mirroring 
reference pyspark: the
+    /// column of this DataFrame, by its plan ID. Names beginning with `_` 
raise
+    /// AttributeError so normal attribute lookup works.
     fn __getattr__(&self, name: &str) -> PyResult<PyColumn> {
         if name.starts_with('_') {
             return Err(PyErr::new::<pyo3::exceptions::PyAttributeError, _>(
                 name.to_string(),
             ));
         }
-        Ok(PyColumn::new(spark_connect::functions::col(name)))
+        Ok(PyColumn::new(self.dataframe.col(name)))
     }
 
     /// Class attributes and all column names, matching Python Connect.
@@ -1603,14 +1605,21 @@ impl PyDataFrame {
         }
     }
 
-    /// As PySpark's `DataFrame.__getitem__`: a column by name (`df["col"]`) 
or position
-    /// (`df[0]`), the rows that match a Column (`df[df.a > 1]`, as `filter`), 
or the
-    /// projection of a list or tuple of columns (`df[["a", "b"]]`, as 
`select`).
+    /// As PySpark's `DataFrame.__getitem__`: a column by name (`df["col"]`), 
of this DataFrame
+    /// by its plan ID, as its star of `df["*"]`, or by position (`df[0]`), 
the rows that match
+    /// a Column (`df[df.a > 1]`, as `filter`), or the projection of a list or 
tuple of columns
+    /// (`df[["a", "b"]]`, as `select`).
     fn __getitem__(&self, py: Python<'_>, item: &Bound<'_, PyAny>) -> 
PyResult<Py<PyAny>> {
         use pyo3::IntoPyObjectExt;
 
         let name = if let Ok(name) = item.cast::<pyo3::types::PyString>() {
-            name.extract::<String>()?
+            let name = name.to_cow()?;
+            let column = if name == "*" {
+                self.dataframe.star()
+            } else {
+                self.dataframe.col(&name)
+            };
+            return PyColumn::new(column).into_py_any(py);
         } else if item.is_instance_of::<PyColumn>() {
             return self.filter(item)?.into_py_any(py);
         } else if item.is_instance_of::<PyList>() || 
item.is_instance_of::<PyTuple>() {
@@ -1641,9 +1650,10 @@ impl PyDataFrame {
     }
 
     /// Private column accessor used by pandas-on-Spark (`sdf._col(name)`); 
mirrors the
-    /// connect DataFrame's `_col`. Backtick-quoted names arrive from 
`scol_for`.
+    /// connect DataFrame's `_col`, the column of this DataFrame by its plan 
ID. Backtick-quoted
+    /// names arrive from `scol_for`.
     fn _col(&self, item: &str) -> PyColumn {
-        PyColumn::new(spark_connect::functions::col(item))
+        PyColumn::new(self.dataframe.col(item))
     }
 
     /// As PySpark's `DataFrame.__repr__`: the table of `_show_string` when 
the session's
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index b04de08..6eb2b14 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1500,25 +1500,41 @@ impl DataFrame {
         DataFrame::new(self.session.clone(), plan)
     }
 
-    /// Select columns by regex pattern.
-    pub fn col_regex(&self, col_name: &str) -> DataFrame {
-        let plan = LogicalPlan::ColRegex {
-            input: Arc::clone(&self.plan),
-            col_name: col_name.to_string(),
-        };
-        DataFrame::new(self.session.clone(), plan)
+    /// The plan ID of this DataFrame's plan in its session, by which its 
columns refer to it.
+    fn plan_id(&self) -> i64 {
+        self.session.plan_ids().id_of(&self.plan)
     }
 
-    /// Select a metadata column by name. Mirrors 
`pyspark.sql.DataFrame.metadataColumn`.
-    pub fn metadata_column(&self, name: &str) -> Column {
+    /// The column `name` of this DataFrame. It refers to this DataFrame by 
its plan ID, so that
+    /// the server resolves it against this DataFrame's plan, as in a join of 
DataFrames of the
+    /// same column names. Mirrors `df[name]`, `df.name` and `DataFrame._col` 
of PySpark Connect.
+    pub fn col(&self, name: &str) -> Column {
+        let reference = 
crate::expression::ColumnReference::new(name).with_plan_id(self.plan_id());
+        Column::new(ExpressionNode::ColumnReference(reference).into())
+    }
+
+    /// The star of the columns of this DataFrame, as `df["*"]` of PySpark 
Connect.
+    pub fn star(&self) -> Column {
+        Column::new(ExpressionNode::UnresolvedStar(None, 
Some(self.plan_id())).into())
+    }
+
+    /// The columns of this DataFrame whose names match the regex `col_name`, 
in backticks, as
+    /// a Column. Mirrors `DataFrame.colRegex`.
+    pub fn col_regex(&self, col_name: &str) -> Column {
         Column::new(
-            ExpressionNode::ColumnReference(
-                crate::expression::ColumnReference::new(name).metadata(),
-            )
-            .into(),
+            ExpressionNode::UnresolvedRegex(col_name.to_string(), 
Some(self.plan_id())).into(),
         )
     }
 
+    /// The metadata column `name` of this DataFrame, which refers to it by 
its plan ID.
+    /// Mirrors `pyspark.sql.DataFrame.metadataColumn`.
+    pub fn metadata_column(&self, name: &str) -> Column {
+        let reference = crate::expression::ColumnReference::new(name)
+            .with_plan_id(self.plan_id())
+            .metadata();
+        Column::new(ExpressionNode::ColumnReference(reference).into())
+    }
+
     /// Group by with rollup.
     pub fn rollup<C: Into<Column>>(
         &self,
@@ -1891,7 +1907,7 @@ impl DataFrame {
     /// `DataFrame.zipWithIndex(indexColName="index")`:
     /// `self.select(col("*"), distributed_sequence_id().alias(indexColName))`.
     pub fn zip_with_index(&self, index_col_name: &str) -> DataFrame {
-        let star = Column::new(ExpressionNode::UnresolvedStar(None).into());
+        let star = Column::new(ExpressionNode::UnresolvedStar(None, 
None).into());
         let seq = Column::new(Expression::function("distributed_sequence_id", 
vec![]))
             .alias(index_col_name);
         self.select(vec![star, seq])
@@ -3841,7 +3857,7 @@ mod plan_construction_tests {
         ser(&df.alias("t"));
         ser(&df.sample(0.5, Some(1)));
         ser(&df.select_expr(vec!["id + 1"]));
-        ser(&df.col_regex("id"));
+        ser(&df.select(vec![df.col_regex("`id`")]));
         ser(&df.describe(vec!["id"]));
         ser(&df.summary(vec!["count"]));
         ser(&df.as_table("t2"));
@@ -4435,3 +4451,71 @@ mod arrow_value_tests {
         assert!(micros_to_time_string(1).contains('.'));
     }
 }
+
+#[cfg(test)]
+mod plan_id_column_tests {
+    use super::*;
+    use crate::session::SparkSession;
+    use proto::relation::RelType;
+
+    fn session() -> SparkSession {
+        SparkSession::builder()
+            .remote("sc://localhost:15002")
+            .get_or_create()
+            .expect("failed to build session")
+    }
+
+    /// As PySpark Connect's `df[name]`, `df["*"]`, `colRegex` and 
`metadataColumn`: the columns
+    /// of a DataFrame refer to it by the plan ID that its relation is sent 
with.
+    #[test]
+    fn columns_refer_to_their_dataframe_by_its_plan_id() {
+        use proto::expression::ExprType;
+
+        let spark = session();
+        let left = spark.range(2).unwrap();
+        let right = spark.range(3).unwrap();
+        let joined = left
+            .join(
+                &right,
+                Some(left.col("id").eq(right.col("id"))),
+                JoinType::Inner,
+            )
+            .select(vec![left.col("id"), right.star(), 
left.col_regex("`i.*`")]);
+        let relation = joined.relation();
+        let Some(RelType::Project(project)) = &relation.rel_type else {
+            panic!("expected Project: {relation:?}");
+        };
+        let Some(RelType::Join(join)) = 
&project.input.as_deref().unwrap().rel_type else {
+            panic!("expected Join");
+        };
+        let plan_id = |r: &proto::Relation| r.common.as_ref().and_then(|c| 
c.plan_id);
+        let (left_id, right_id) = (
+            plan_id(join.left.as_deref().unwrap()),
+            plan_id(join.right.as_deref().unwrap()),
+        );
+        assert!(left_id.is_some() && left_id != right_id);
+        let ids: Vec<Option<i64>> = project
+            .expressions
+            .iter()
+            .map(|e| match &e.expr_type {
+                Some(ExprType::UnresolvedAttribute(a)) => a.plan_id,
+                Some(ExprType::UnresolvedStar(s)) => s.plan_id,
+                Some(ExprType::UnresolvedRegex(r)) => r.plan_id,
+                other => panic!("unexpected {other:?}"),
+            })
+            .collect();
+        assert_eq!(ids, vec![left_id, right_id, left_id]);
+        let Some(ExprType::UnresolvedAttribute(metadata)) = left
+            .metadata_column("_metadata")
+            .expression()
+            .to_proto()
+            .expr_type
+        else {
+            panic!("expected an attribute");
+        };
+        assert_eq!(
+            (metadata.plan_id, metadata.is_metadata_column),
+            (left_id, Some(true))
+        );
+    }
+}
diff --git a/crates/spark-connect/src/expression.rs 
b/crates/spark-connect/src/expression.rs
index 3dcf67f..4731f87 100644
--- a/crates/spark-connect/src/expression.rs
+++ b/crates/spark-connect/src/expression.rs
@@ -59,8 +59,9 @@ pub enum ExpressionNode {
     /// `pyspark.sql.connect.expressions.UnresolvedFunction` - a function call.
     UnresolvedFunction(UnresolvedFunction),
     /// `pyspark.sql.connect.expressions.UnresolvedStar` - a star expression 
(`*`).
-    /// Carries an optional target ending in `.*` (e.g. `df.*` from 
`col("df.*")`).
-    UnresolvedStar(Option<String>),
+    /// Carries an optional target ending in `.*` (e.g. `df.*` from 
`col("df.*")`), and the plan
+    /// ID of the DataFrame of `df["*"]`, whose columns it is.
+    UnresolvedStar(Option<String>, Option<i64>),
     /// `pyspark.sql.connect.expressions.ColumnAlias` / `Alias` - an aliased 
expression.
     Alias(Box<Alias>),
     /// `pyspark.sql.connect.expressions.CastExpression` - a cast expression.
@@ -68,8 +69,9 @@ pub enum ExpressionNode {
     /// `spark.connect.Expression.DirectShufflePartitionID` - wraps a child 
expression
     /// that evaluates to a partition id (used by `DataFrame.repartitionById`).
     DirectShufflePartitionId(Expression),
-    /// `pyspark.sql.connect.expressions.UnresolvedRegex` - a regex column 
reference.
-    UnresolvedRegex(String),
+    /// `pyspark.sql.connect.expressions.UnresolvedRegex` - a regex column 
reference, of the
+    /// columns of the DataFrame of the plan ID of `df.colRegex(...)`.
+    UnresolvedRegex(String, Option<i64>),
     /// `pyspark.sql.connect.expressions.SortOrder` - a sort order expression.
     SortOrder(Box<SortOrder>),
     /// `pyspark.sql.connect.expressions.CaseWhen` - a CASE WHEN expression.
@@ -155,7 +157,7 @@ impl Expression {
             ExpressionNode::Literal(lit) => lit.render(),
             ExpressionNode::ColumnReference(col_ref) => col_ref.name.clone(),
             ExpressionNode::UnresolvedFunction(func) => func.render(),
-            ExpressionNode::UnresolvedStar(target) => match target {
+            ExpressionNode::UnresolvedStar(target, _) => match target {
                 Some(t) => format!("unresolvedstar({t})"),
                 None => "unresolvedstar()".to_string(),
             },
@@ -180,7 +182,7 @@ impl Expression {
             ExpressionNode::DirectShufflePartitionId(child) => {
                 format!("DirectShufflePartitionID(child={})", child.render())
             }
-            ExpressionNode::UnresolvedRegex(col_name) => 
format!("UnresolvedRegex({col_name})"),
+            ExpressionNode::UnresolvedRegex(col_name, _) => 
format!("UnresolvedRegex({col_name})"),
             ExpressionNode::SortOrder(sort) => sort.render(),
             ExpressionNode::CaseWhen(case_when) => case_when.render(),
             ExpressionNode::UnresolvedExtractValue(ev) => {
@@ -228,16 +230,15 @@ impl Expression {
             ExpressionNode::Literal(lit) => lit.to_proto(),
             ExpressionNode::ColumnReference(col_ref) => col_ref.to_proto(),
             ExpressionNode::UnresolvedFunction(func) => func.to_proto(),
-            ExpressionNode::UnresolvedStar(target) => {
-                let mut expr = proto::Expression::default();
-                expr.expr_type = 
Some(proto::expression::ExprType::UnresolvedStar(
+            ExpressionNode::UnresolvedStar(target, plan_id) => 
proto::Expression {
+                expr_type: Some(proto::expression::ExprType::UnresolvedStar(
                     proto::expression::UnresolvedStar {
                         unparsed_target: target.clone(),
-                        plan_id: None,
+                        plan_id: *plan_id,
                     },
-                ));
-                expr
-            }
+                )),
+                ..Default::default()
+            },
             ExpressionNode::Alias(alias) => alias.to_proto(),
             ExpressionNode::Cast(cast) => cast.to_proto(),
             ExpressionNode::DirectShufflePartitionId(child) => {
@@ -249,16 +250,15 @@ impl Expression {
                 ));
                 expr
             }
-            ExpressionNode::UnresolvedRegex(col_name) => {
-                let mut expr = proto::Expression::default();
-                expr.expr_type = 
Some(proto::expression::ExprType::UnresolvedRegex(
+            ExpressionNode::UnresolvedRegex(col_name, plan_id) => 
proto::Expression {
+                expr_type: Some(proto::expression::ExprType::UnresolvedRegex(
                     proto::expression::UnresolvedRegex {
                         col_name: col_name.clone(),
-                        plan_id: None,
+                        plan_id: *plan_id,
                     },
-                ));
-                expr
-            }
+                )),
+                ..Default::default()
+            },
             ExpressionNode::SortOrder(sort) => sort.to_proto(),
             ExpressionNode::CaseWhen(case_when) => case_when.to_proto(),
             ExpressionNode::UnresolvedExtractValue(ev) => ev.to_proto(),
@@ -1349,7 +1349,7 @@ mod tests {
             "CAST(x AS INT)"
         );
         assert_eq!(
-            Expression::from(ExpressionNode::UnresolvedStar(None)).render(),
+            Expression::from(ExpressionNode::UnresolvedStar(None, 
None)).render(),
             "unresolvedstar()"
         );
 
@@ -1531,13 +1531,14 @@ mod tests {
 
     #[test]
     fn test_expression_render_unresolved_star_with_target() {
-        let expr: Expression = 
ExpressionNode::UnresolvedStar(Some("table.*".to_string())).into();
+        let expr: Expression =
+            ExpressionNode::UnresolvedStar(Some("table.*".to_string()), 
None).into();
         assert_eq!(expr.render(), "unresolvedstar(table.*)");
     }
 
     #[test]
     fn test_expression_render_unresolved_regex() {
-        let expr: Expression = 
ExpressionNode::UnresolvedRegex("`col_.*`".to_string()).into();
+        let expr: Expression = 
ExpressionNode::UnresolvedRegex("`col_.*`".to_string(), None).into();
         assert_eq!(expr.render(), "UnresolvedRegex(`col_.*`)");
     }
 
@@ -1944,7 +1945,7 @@ mod tests {
 
     #[test]
     fn test_to_proto_unresolved_star_none() {
-        let expr: Expression = ExpressionNode::UnresolvedStar(None).into();
+        let expr: Expression = ExpressionNode::UnresolvedStar(None, 
None).into();
         let proto = expr.to_proto();
         assert!(proto.expr_type.is_some());
         if let Some(proto::expression::ExprType::UnresolvedStar(star)) = 
proto.expr_type {
@@ -1956,7 +1957,8 @@ mod tests {
 
     #[test]
     fn test_to_proto_unresolved_star_with_target() {
-        let expr: Expression = 
ExpressionNode::UnresolvedStar(Some("table.*".to_string())).into();
+        let expr: Expression =
+            ExpressionNode::UnresolvedStar(Some("table.*".to_string()), 
None).into();
         let proto = expr.to_proto();
         assert!(proto.expr_type.is_some());
         if let Some(proto::expression::ExprType::UnresolvedStar(star)) = 
proto.expr_type {
@@ -1968,7 +1970,7 @@ mod tests {
 
     #[test]
     fn test_to_proto_unresolved_regex() {
-        let expr: Expression = 
ExpressionNode::UnresolvedRegex("`col_.*`".to_string()).into();
+        let expr: Expression = 
ExpressionNode::UnresolvedRegex("`col_.*`".to_string(), None).into();
         let proto = expr.to_proto();
         assert!(proto.expr_type.is_some());
         if let Some(proto::expression::ExprType::UnresolvedRegex(regex)) = 
proto.expr_type {
diff --git a/crates/spark-connect/src/functions.rs 
b/crates/spark-connect/src/functions.rs
index 49f7cdb..e13f50b 100644
--- a/crates/spark-connect/src/functions.rs
+++ b/crates/spark-connect/src/functions.rs
@@ -67,9 +67,9 @@ fn lit_binary(b: Vec<u8>) -> Expression {
 /// `select("*")`, `col("t.*")` resolve on the server), matching pyspark.
 pub fn col(name: &str) -> Column {
     if name == "*" {
-        Column::new(ExpressionNode::UnresolvedStar(None).into())
+        Column::new(ExpressionNode::UnresolvedStar(None, None).into())
     } else if let Some(prefix) = name.strip_suffix(".*") {
-        
Column::new(ExpressionNode::UnresolvedStar(Some(format!("{prefix}.*"))).into())
+        
Column::new(ExpressionNode::UnresolvedStar(Some(format!("{prefix}.*")), 
None).into())
     } else {
         Column::new(Expression::column(name))
     }
@@ -4649,13 +4649,13 @@ mod tests {
 
         let star = col("*");
         match star.expression().node() {
-            ExpressionNode::UnresolvedStar(None) => {}
+            ExpressionNode::UnresolvedStar(None, None) => {}
             _ => panic!("Expected UnresolvedStar(None)"),
         }
 
         let table_star = col("t.*");
         match table_star.expression().node() {
-            ExpressionNode::UnresolvedStar(Some(_)) => {}
+            ExpressionNode::UnresolvedStar(Some(_), None) => {}
             _ => panic!("Expected UnresolvedStar with prefix"),
         }
     }
diff --git a/crates/spark-connect/tests/dataframe_extra_golden.rs 
b/crates/spark-connect/tests/dataframe_extra_golden.rs
index eab357a..5cc69c4 100644
--- a/crates/spark-connect/tests/dataframe_extra_golden.rs
+++ b/crates/spark-connect/tests/dataframe_extra_golden.rs
@@ -44,7 +44,7 @@ fn test_summary_golden() {
 #[test]
 #[ignore]
 fn test_col_regex_golden() {
-    // Test: df.selectExpr("id", "id as id2").col_regex("id.*")
+    // Test: df.selectExpr("id", "id as id2").select(col_regex("id.*"))
 }
 
 #[test]
diff --git a/crates/spark-connect/tests/e2e_coverage.rs 
b/crates/spark-connect/tests/e2e_coverage.rs
index b76db10..d1b949a 100644
--- a/crates/spark-connect/tests/e2e_coverage.rs
+++ b/crates/spark-connect/tests/e2e_coverage.rs
@@ -681,7 +681,7 @@ fn dataframe_more_surface() {
         .unwrap();
     let _ = df.to_df(vec!["a", "b", "c"]).collect().unwrap();
     let _ = df.sample(0.5, Some(1)).count().unwrap();
-    let _ = df.col_regex("`.*`").collect().unwrap();
+    let _ = df.select(vec![df.col_regex("`.*`")]).collect().unwrap();
 
     // cache / persist / unpersist round-trip.
     let cached = df.cache().unwrap();
diff --git a/crates/spark-connect/tests/plans_golden.rs 
b/crates/spark-connect/tests/plans_golden.rs
index 1f551cf..1768b25 100644
--- a/crates/spark-connect/tests/plans_golden.rs
+++ b/crates/spark-connect/tests/plans_golden.rs
@@ -297,8 +297,9 @@ fn build(name: &str) -> Option<LogicalPlan> {
         }
         "select_star" => {
             let range = plan::range(0, 10, 1);
-            let star_col =
-                
spark_connect::column::Column::new(ExpressionNode::UnresolvedStar(None).into());
+            let star_col = spark_connect::column::Column::new(
+                ExpressionNode::UnresolvedStar(None, None).into(),
+            );
             plan::project(range, vec![star_col])
         }
         "select_alias" => {
@@ -393,7 +394,7 @@ fn build(name: &str) -> Option<LogicalPlan> {
                 .clone();
             let agg_count = 
spark_connect::column::Column::new(Expression::function(
                 "count",
-                vec![ExpressionNode::UnresolvedStar(None).into()],
+                vec![ExpressionNode::UnresolvedStar(None, None).into()],
             ))
             .alias("c")
             .expression()
@@ -604,7 +605,7 @@ fn build(name: &str) -> Option<LogicalPlan> {
         "col_regex" => {
             let range = plan::range(0, 10, 1);
             let regex_col = spark_connect::column::Column::new(
-                ExpressionNode::UnresolvedRegex("`id`".to_string()).into(),
+                ExpressionNode::UnresolvedRegex("`id`".to_string(), 
None).into(),
             );
             plan::project(range, vec![regex_col])
         }
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 5eeee7c..64217f5 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -609,6 +609,24 @@ def 
test_grouped_map_functions_take_iterators_by_their_type_hints(session):
         assert sorted(tuple(r) for r in result.collect()) == expected
 
 
+def test_dataframe_columns_resolve_against_their_dataframe_as_pyspark(session):
+    # As PySpark Connect's: df[name] and df.name refer to the columns of their 
DataFrame by its
+    # plan ID, so a join selects the columns of each side, and an ambiguous 
self-join of a
+    # derived DataFrame raises PySpark's error.
+    left = session.createDataFrame([(1, "a"), (2, "b")], "id int, v string")
+    right = session.createDataFrame([(1, "x"), (3, "y")], "id int, w string")
+    joined = left.join(right, left["id"] == right["id"])
+    assert joined.select(left["id"], right["w"]).collect() == [Row(id=1, 
w="x")]
+    assert joined.select(left.v, right.w).collect() == [Row(v="a", w="x")]
+    assert joined.select(right["*"]).collect() == [Row(id=1, w="x")]
+    assert joined.select(left.colRegex("`v`")).collect() == [Row(v="a")]
+    base = session.range(3)
+    derived = base.withColumn("x", F.col("id") * 2)
+    with pytest.raises(AnalysisException) as error:
+        base.join(derived, base["id"] == 
derived["id"]).select(base["id"]).collect()
+    assert error.value.getCondition() == "AMBIGUOUS_COLUMN_REFERENCE"
+
+
 def test_collect_reads_a_result_across_stream_ends(session):
     # Takes several response streams when the server's senderMaxStreamSize is 
small.
     n = 300_000
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index ce58183..4e12e66 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1389,6 +1389,38 @@ def test_broadcast_type_check():
         F.broadcast("not a dataframe")
 
 
+def test_dataframe_columns_refer_to_it_by_its_plan_id_as_pyspark(mock_connect):
+    # As PySpark Connect's df[name], df.name, df["*"] and colRegex: the 
columns of a DataFrame
+    # refer to it by the plan ID that its relation is sent with, and colRegex 
is a Column.
+    from pyspark.sql import Column
+
+    spark, service = mock_connect
+    left, right = spark.range(2), spark.range(3)
+    assert isinstance(left.colRegex("`i.*`"), Column)
+    left.join(right, left["id"] == right.id).select(
+        left.id, right["*"], left.colRegex("`i.*`"), left["id"]
+    ).collect()
+    # ExecutePlanRequest.plan=3, Plan.root=1, Relation.project=3: input=1, 
expressions=3;
+    # Relation.join=5: left=1, right=2; Relation.common=1, 
RelationCommon.plan_id=2.
+    plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+    project = parse_message(parse_message(plan)[3][0])
+    join = parse_message(parse_message(project[1][0])[5][0])
+
+    def plan_id(relation):
+        return parse_message(parse_message(relation)[1][0])[2][0]
+
+    left_id, right_id = plan_id(join[1][0]), plan_id(join[2][0])
+    assert left_id != right_id
+    # Expression.unresolved_attribute=2 (plan_id=2), unresolved_star=5 
(plan_id=2),
+    # unresolved_regex=8 (plan_id=2).
+    ids = []
+    for expression in project[3]:
+        expression = parse_message(expression)
+        (field,) = set(expression) & {2, 5, 8}
+        ids.append(parse_message(expression[field][0])[2][0])
+    assert ids == [left_id, right_id, left_id, left_id]
+
+
 def test_row_count_and_index():
     from pyspark.sql import Row
     r = Row(a=1, b=2, c=1)


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to