This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new aa28b047 [SPARK-61150][CONNECT] Make plan IDs unique across the 
sessions of the process, as PySpark does
aa28b047 is described below

commit aa28b04765eb49fc8abf66842e8ef1c267ff3697
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 12:40:15 2026 +0900

    [SPARK-61150][CONNECT] Make plan IDs unique across the sessions of the 
process, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61150](https://issues.apache.org/jira/browse/SPARK-61150): Draw the 
plan IDs of all the sessions of the process from one counter, as PySpark does.
    
    `PlanIds` numbered the nodes of each session from 1. PySpark's 
`LogicalPlan._nextPlanId` is one counter of the process. This change makes the 
counter a process-wide `AtomicI64` (`NEXT_PLAN_ID`) that every `PlanIds` draws 
from, and `assign` raises it past the ID it is given, as before. Each session 
keeps its own node-to-ID map, with its pruning, so a node keeps its ID across 
re-executions (SPARK-60898).
    
    The unit test that asserted that each session numbers its own plans from 1 
now asserts that the sessions never hand out the same ID. The test that checked 
a returned relation's plan ID of 1 now compares it with the ID of that node.
    
    ### Why are the changes needed?
    
    `a = spark.createDataFrame([(1,)], "id int")` and `b = 
spark.newSession().createDataFrame([(2,)], "id int")` gave 
`a.select(b.id).collect()` as `[Row(id=1)]`, and `a.filter(b.id == 2).count()` 
as `0`. The column of `b` carried a plan ID that also named a node of `a`'s 
plan, so the server silently resolved it to `a`'s column of the same name. 
PySpark raises `AnalysisException` `CANNOT_RESOLVE_DATAFRAME_COLUMN` for both.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. A column of a DataFrame of another session no longer resolves against 
this session's DataFrame, and raises `CANNOT_RESOLVE_DATAFRAME_COLUMN` as in 
PySpark.
    
    ### How was this patch tested?
    
    - `the_sessions_of_a_process_never_hand_out_the_same_plan_id` (Rust unit 
test): plans of two sessions never get the same plan ID. It fails with the 
per-session counter.
    - The existing unit tests of stable plan IDs, pruning and returned 
relations pass with the global counter.
    - `test_columns_of_another_session_do_not_resolve_in_this_one` (live): 
select and filter with a column of another session raise 
`CANNOT_RESOLVE_DATAFRAME_COLUMN`, and a column of this session still selects. 
The same calls give the same condition with official PySpark 4.3.0.
    - The lib tests of both crates (590 passed), the build of all tests, also 
with `wasm-udf`, and the offline Python suites (2201 passed, 8 skipped), and 
the live tests of sessions, plan IDs, columns, joins and pickling (42 passed) 
against a local Spark 4.3.0 server; rustfmt passes, and clippy reports no 
diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #340 from HyukjinKwon/SPARK-61150-process-wide-plan-ids.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/spark-connect/src/dataframe.rs | 24 +++++++++++++++++-------
 crates/spark-connect/src/plan.rs      | 13 ++++++++-----
 python/tests/test_dropin_e2e.py       | 16 ++++++++++++++++
 3 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index 1f164b94..c5a87976 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -4086,7 +4086,7 @@ mod plan_construction_tests {
     #[test]
     fn a_relation_the_server_returned_is_sent_back_with_its_own_plan_id() {
         use proto::relation::RelType;
-        // A session of its own, whose plan IDs start at 1.
+        // A session of its own.
         let spark = session().new_session();
         let returned = |rel_type: RelType| {
             let relation = proto::Relation {
@@ -4115,7 +4115,10 @@ mod plan_construction_tests {
             Some(RelType::Sql(sql)) => assert_eq!(sql.query, "SELECT 1"),
             other => panic!("expected the returned SQL relation, got 
{other:?}"),
         }
-        assert_eq!(relation.common.and_then(|common| common.plan_id), Some(1));
+        assert_eq!(
+            relation.common.and_then(|common| common.plan_id),
+            Some(spark.plan_ids().id_of(&query.plan))
+        );
     }
 
     #[test]
@@ -4378,13 +4381,20 @@ mod plan_construction_tests {
     }
 
     #[test]
-    fn a_new_session_numbers_its_own_plans() {
+    fn the_sessions_of_a_process_never_hand_out_the_same_plan_id() {
+        // As PySpark's one counter of the process: a plan of one session 
never has the plan ID
+        // of a plan of another, so a column of another session names no node 
of this plan.
         let spark = session();
         let (one, two) = (spark.new_session(), spark.new_session());
-        assert_eq!(plan_ids_of(&one.range(1).unwrap().relation()), vec![1]);
-        assert_eq!(plan_ids_of(&one.range(2).unwrap().relation()), vec![2]);
-        // The plans of one session do not advance the numbering of another.
-        assert_eq!(plan_ids_of(&two.range(1).unwrap().relation()), vec![1]);
+        let mut seen = std::collections::HashSet::new();
+        for end in 0..10 {
+            for plans in [&one, &two] {
+                for id in plan_ids_of(&plans.range(end).unwrap().relation()) {
+                    assert!(seen.insert(id), "plan ID {id} was assigned 
twice");
+                }
+            }
+        }
+        assert_eq!(seen.len(), 20);
     }
 
     #[test]
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index 185cca19..74406774 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -5,6 +5,7 @@
 
 use spark_connect_proto as proto;
 use std::collections::HashMap;
+use std::sync::atomic::{AtomicI64, Ordering};
 use std::sync::{Arc, Mutex, Weak};
 
 use crate::column::Column;
@@ -486,6 +487,11 @@ pub enum SetOpType {
     Except,
 }
 
+/// The plan ID of the next new node, of all the sessions of the process: as 
PySpark's
+/// `LogicalPlan._nextPlanId`, one counter, so that a column of a DataFrame of 
another session
+/// names no node of this session's plan.
+static NEXT_PLAN_ID: AtomicI64 = AtomicI64::new(1);
+
 /// Plan IDs that stay the same each time a plan node is serialized, as 
PySpark assigns
 /// them once per plan. The server caches resolved plans keyed by the whole 
relation, so
 /// re-executing a DataFrame, or building on one it already resolved, reuses 
that work.
@@ -501,7 +507,6 @@ pub(crate) struct PlanIds {
 /// The state of `PlanIds`, all guarded by its one lock.
 #[derive(Debug)]
 struct PlanIdNodes {
-    next: i64,
     ids: HashMap<usize, (Weak<LogicalPlan>, i64)>,
     prune_at: usize,
 }
@@ -512,7 +517,6 @@ impl PlanIds {
     pub(crate) fn new() -> Self {
         PlanIds {
             nodes: Mutex::new(PlanIdNodes {
-                next: 1,
                 ids: HashMap::new(),
                 prune_at: Self::MIN_PRUNE_AT,
             }),
@@ -543,8 +547,7 @@ impl PlanIds {
             nodes.ids.retain(|_, (node, _)| node.strong_count() > 0);
             nodes.prune_at = (nodes.ids.len() * 2).max(Self::MIN_PRUNE_AT);
         }
-        let id = nodes.next;
-        nodes.next += 1;
+        let id = NEXT_PLAN_ID.fetch_add(1, Ordering::Relaxed);
         nodes.ids.insert(key, (Arc::downgrade(node), id));
         id
     }
@@ -552,8 +555,8 @@ impl PlanIds {
     /// Give `node` the plan ID `id`, of a relation that this session sent: so 
a DataFrame of the
     /// relation, such as one that was pickled, keeps the plan ID of the 
DataFrame it was of.
     pub(crate) fn assign(&self, node: &Arc<LogicalPlan>, id: i64) {
+        NEXT_PLAN_ID.fetch_max(id.saturating_add(1), Ordering::Relaxed);
         let mut nodes = self.nodes.lock().unwrap_or_else(|e| e.into_inner());
-        nodes.next = nodes.next.max(id.saturating_add(1));
         nodes
             .ids
             .insert(Arc::as_ptr(node) as usize, (Arc::downgrade(node), id));
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 4afd6911..cb17b2ef 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1314,6 +1314,22 @@ def 
test_subqueries_and_table_arguments_return_the_rows_of_pyspark(session):
         session.range(1).asTable().orderBy("id")
 
 
+def test_columns_of_another_session_do_not_resolve_in_this_one(session):
+    # As PySpark's, plan IDs come from one counter of the process, so a column 
of a DataFrame of
+    # another session names no node of this one: 
CANNOT_RESOLVE_DATAFRAME_COLUMN, not the
+    # column of the same name.
+    other = session.newSession()
+    a = session.createDataFrame([(1,)], "id int")
+    b = other.createDataFrame([(2,)], "id int")
+    with pytest.raises(AnalysisException) as e:
+        a.select(b.id).collect()
+    assert e.value.getCondition() == "CANNOT_RESOLVE_DATAFRAME_COLUMN"
+    with pytest.raises(AnalysisException) as e:
+        a.filter(b.id == 2).count()
+    assert e.value.getCondition() == "CANNOT_RESOLVE_DATAFRAME_COLUMN"
+    assert a.select(a.id).collect() == [Row(id=1)]
+
+
 def test_tags_are_those_of_each_thread_as_pyspark(session):
     # As PySpark's: the tags of a thread are on its operations only, so 
interruptTag of the tag
     # of one thread interrupts its operation, but not that of another thread.


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to