This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new aa28b047 [SPARK-61150][CONNECT] Make plan IDs unique across the
sessions of the process, as PySpark does
aa28b047 is described below
commit aa28b04765eb49fc8abf66842e8ef1c267ff3697
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 12:40:15 2026 +0900
[SPARK-61150][CONNECT] Make plan IDs unique across the sessions of the
process, as PySpark does
### What changes were proposed in this pull request?
[SPARK-61150](https://issues.apache.org/jira/browse/SPARK-61150): Draw the
plan IDs of all the sessions of the process from one counter, as PySpark does.
`PlanIds` numbered the nodes of each session from 1. PySpark's
`LogicalPlan._nextPlanId` is one counter of the process. This change makes the
counter a process-wide `AtomicI64` (`NEXT_PLAN_ID`) that every `PlanIds` draws
from, and `assign` raises it past the ID it is given, as before. Each session
keeps its own node-to-ID map, with its pruning, so a node keeps its ID across
re-executions (SPARK-60898).
The unit test that asserted that each session numbers its own plans from 1
now asserts that the sessions never hand out the same ID. The test that checked
a returned relation's plan ID of 1 now compares it with the ID of that node.
### Why are the changes needed?
`a = spark.createDataFrame([(1,)], "id int")` and `b =
spark.newSession().createDataFrame([(2,)], "id int")` gave
`a.select(b.id).collect()` as `[Row(id=1)]`, and `a.filter(b.id == 2).count()`
as `0`. The column of `b` carried a plan ID that also named a node of `a`'s
plan, so the server silently resolved it to `a`'s column of the same name.
PySpark raises `AnalysisException` `CANNOT_RESOLVE_DATAFRAME_COLUMN` for both.
### Does this PR introduce _any_ user-facing change?
Yes. A column of a DataFrame of another session no longer resolves against
this session's DataFrame, and raises `CANNOT_RESOLVE_DATAFRAME_COLUMN` as in
PySpark.
### How was this patch tested?
- `the_sessions_of_a_process_never_hand_out_the_same_plan_id` (Rust unit
test): plans of two sessions never get the same plan ID. It fails with the
per-session counter.
- The existing unit tests of stable plan IDs, pruning and returned
relations pass with the global counter.
- `test_columns_of_another_session_do_not_resolve_in_this_one` (live):
select and filter with a column of another session raise
`CANNOT_RESOLVE_DATAFRAME_COLUMN`, and a column of this session still selects.
The same calls give the same condition with official PySpark 4.3.0.
- The lib tests of both crates (590 passed), the build of all tests, also
with `wasm-udf`, and the offline Python suites (2201 passed, 8 skipped), and
the live tests of sessions, plan IDs, columns, joins and pickling (42 passed)
against a local Spark 4.3.0 server; rustfmt passes, and clippy reports no
diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #340 from HyukjinKwon/SPARK-61150-process-wide-plan-ids.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/spark-connect/src/dataframe.rs | 24 +++++++++++++++++-------
crates/spark-connect/src/plan.rs | 13 ++++++++-----
python/tests/test_dropin_e2e.py | 16 ++++++++++++++++
3 files changed, 41 insertions(+), 12 deletions(-)
diff --git a/crates/spark-connect/src/dataframe.rs
b/crates/spark-connect/src/dataframe.rs
index 1f164b94..c5a87976 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -4086,7 +4086,7 @@ mod plan_construction_tests {
#[test]
fn a_relation_the_server_returned_is_sent_back_with_its_own_plan_id() {
use proto::relation::RelType;
- // A session of its own, whose plan IDs start at 1.
+ // A session of its own.
let spark = session().new_session();
let returned = |rel_type: RelType| {
let relation = proto::Relation {
@@ -4115,7 +4115,10 @@ mod plan_construction_tests {
Some(RelType::Sql(sql)) => assert_eq!(sql.query, "SELECT 1"),
other => panic!("expected the returned SQL relation, got
{other:?}"),
}
- assert_eq!(relation.common.and_then(|common| common.plan_id), Some(1));
+ assert_eq!(
+ relation.common.and_then(|common| common.plan_id),
+ Some(spark.plan_ids().id_of(&query.plan))
+ );
}
#[test]
@@ -4378,13 +4381,20 @@ mod plan_construction_tests {
}
#[test]
- fn a_new_session_numbers_its_own_plans() {
+ fn the_sessions_of_a_process_never_hand_out_the_same_plan_id() {
+ // As PySpark's one counter of the process: a plan of one session
never has the plan ID
+ // of a plan of another, so a column of another session names no node
of this plan.
let spark = session();
let (one, two) = (spark.new_session(), spark.new_session());
- assert_eq!(plan_ids_of(&one.range(1).unwrap().relation()), vec![1]);
- assert_eq!(plan_ids_of(&one.range(2).unwrap().relation()), vec![2]);
- // The plans of one session do not advance the numbering of another.
- assert_eq!(plan_ids_of(&two.range(1).unwrap().relation()), vec![1]);
+ let mut seen = std::collections::HashSet::new();
+ for end in 0..10 {
+ for plans in [&one, &two] {
+ for id in plan_ids_of(&plans.range(end).unwrap().relation()) {
+ assert!(seen.insert(id), "plan ID {id} was assigned
twice");
+ }
+ }
+ }
+ assert_eq!(seen.len(), 20);
}
#[test]
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index 185cca19..74406774 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -5,6 +5,7 @@
use spark_connect_proto as proto;
use std::collections::HashMap;
+use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::{Arc, Mutex, Weak};
use crate::column::Column;
@@ -486,6 +487,11 @@ pub enum SetOpType {
Except,
}
+/// The plan ID of the next new node, of all the sessions of the process: as
PySpark's
+/// `LogicalPlan._nextPlanId`, one counter, so that a column of a DataFrame of
another session
+/// names no node of this session's plan.
+static NEXT_PLAN_ID: AtomicI64 = AtomicI64::new(1);
+
/// Plan IDs that stay the same each time a plan node is serialized, as
PySpark assigns
/// them once per plan. The server caches resolved plans keyed by the whole
relation, so
/// re-executing a DataFrame, or building on one it already resolved, reuses
that work.
@@ -501,7 +507,6 @@ pub(crate) struct PlanIds {
/// The state of `PlanIds`, all guarded by its one lock.
#[derive(Debug)]
struct PlanIdNodes {
- next: i64,
ids: HashMap<usize, (Weak<LogicalPlan>, i64)>,
prune_at: usize,
}
@@ -512,7 +517,6 @@ impl PlanIds {
pub(crate) fn new() -> Self {
PlanIds {
nodes: Mutex::new(PlanIdNodes {
- next: 1,
ids: HashMap::new(),
prune_at: Self::MIN_PRUNE_AT,
}),
@@ -543,8 +547,7 @@ impl PlanIds {
nodes.ids.retain(|_, (node, _)| node.strong_count() > 0);
nodes.prune_at = (nodes.ids.len() * 2).max(Self::MIN_PRUNE_AT);
}
- let id = nodes.next;
- nodes.next += 1;
+ let id = NEXT_PLAN_ID.fetch_add(1, Ordering::Relaxed);
nodes.ids.insert(key, (Arc::downgrade(node), id));
id
}
@@ -552,8 +555,8 @@ impl PlanIds {
/// Give `node` the plan ID `id`, of a relation that this session sent: so
a DataFrame of the
/// relation, such as one that was pickled, keeps the plan ID of the
DataFrame it was of.
pub(crate) fn assign(&self, node: &Arc<LogicalPlan>, id: i64) {
+ NEXT_PLAN_ID.fetch_max(id.saturating_add(1), Ordering::Relaxed);
let mut nodes = self.nodes.lock().unwrap_or_else(|e| e.into_inner());
- nodes.next = nodes.next.max(id.saturating_add(1));
nodes
.ids
.insert(Arc::as_ptr(node) as usize, (Arc::downgrade(node), id));
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 4afd6911..cb17b2ef 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1314,6 +1314,22 @@ def
test_subqueries_and_table_arguments_return_the_rows_of_pyspark(session):
session.range(1).asTable().orderBy("id")
+def test_columns_of_another_session_do_not_resolve_in_this_one(session):
+ # As PySpark's, plan IDs come from one counter of the process, so a column
of a DataFrame of
+ # another session names no node of this one:
CANNOT_RESOLVE_DATAFRAME_COLUMN, not the
+ # column of the same name.
+ other = session.newSession()
+ a = session.createDataFrame([(1,)], "id int")
+ b = other.createDataFrame([(2,)], "id int")
+ with pytest.raises(AnalysisException) as e:
+ a.select(b.id).collect()
+ assert e.value.getCondition() == "CANNOT_RESOLVE_DATAFRAME_COLUMN"
+ with pytest.raises(AnalysisException) as e:
+ a.filter(b.id == 2).count()
+ assert e.value.getCondition() == "CANNOT_RESOLVE_DATAFRAME_COLUMN"
+ assert a.select(a.id).collect() == [Row(id=1)]
+
+
def test_tags_are_those_of_each_thread_as_pyspark(session):
# As PySpark's: the tags of a thread are on its operations only, so
interruptTag of the tag
# of one thread interrupts its operation, but not that of another thread.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]