This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 9abcb1c [SPARK-61041][CONNECT][PYTHON] Raise SESSION_NOT_SAME for the
DataFrames of other sessions in the Python client as PySpark does
9abcb1c is described below
commit 9abcb1c2b4e60bf98b33ce52ca31d192ba8ee96f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:50:42 2026 +0900
[SPARK-61041][CONNECT][PYTHON] Raise SESSION_NOT_SAME for the DataFrames of
other sessions in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61041](https://issues.apache.org/jira/browse/SPARK-61041): the
methods that take another DataFrame now check that it is of the same session,
as PySpark Connect's `_check_same_session` does.
These raise `SessionNotSameException` (`SESSION_NOT_SAME`) for a DataFrame
of another session, and for anything that is not a DataFrame:
- the DataFrame methods `join`, `crossJoin`, `lateralJoin`, `union`,
`unionAll`, `unionByName`, `intersect`, `intersectAll`, `subtract`,
`exceptAll`, `zip`, `nearestByJoin` and `sameSemantics`
- `GroupedData.cogroup`
- the `initialState` of `transformWithState` and
`transformWithStateInPandas`
`sameSemantics` first raises `NOT_EXPECTED_TYPE` for what is not a
DataFrame, as PySpark's does.
The core adds `GroupedData::spark_session`, the session of the grouped
DataFrame.
### Why are the changes needed?
The client took a DataFrame of another session and sent its plan to the
server under this session. There, the relations of the other session, such as
its temporary views, are not found. It also raised `TypeError`s for arguments
that are not DataFrames.
### Does this PR introduce _any_ user-facing change?
Yes. These methods raise the `SESSION_NOT_SAME` of PySpark 4.3.0.
### How was this patch tested?
- A new live test checks that a join and a union with a DataFrame of a
temporary view of another session raise `SessionNotSameException`. It passes
with official PySpark 4.3.0, and fails without this change.
- A new offline test checks the error of each of these methods, before any
request, and that DataFrames of the same session are taken.
- On a local server, 19 cases of these methods behave as in official
PySpark 4.3.0.
- The lib tests of both crates (571 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1811 passed, 8 skipped), and the
live drop-in and Arrow suites (109 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #243 from HyukjinKwon/SPARK-61041-same-session.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 102 +++++++++++++++++++++++++++---------
crates/pyspark-rs/src/group.rs | 32 +++++++++--
python/tests/test_dropin_e2e.py | 14 +++++
python/tests/test_dropin_offline.py | 30 +++++++++++
4 files changed, 149 insertions(+), 29 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index af7993f..3a1a041 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -49,6 +49,29 @@ pub struct PyDataFrame {
}
impl PyDataFrame {
+ /// `other`, a DataFrame of the session of this one, as PySpark's
`_check_same_session`
+ /// takes the DataFrame of a method: else `SESSION_NOT_SAME`, as for what
is not a
+ /// DataFrame.
+ pub(crate) fn same_session<'py>(
+ &self,
+ other: &Bound<'py, PyAny>,
+ ) -> PyResult<PyRef<'py, PyDataFrame>> {
+ match other.extract::<PyRef<'py, PyDataFrame>>() {
+ Ok(df)
+ if df.dataframe.spark_session().session_id()
+ == self.dataframe.spark_session().session_id() =>
+ {
+ Ok(df)
+ }
+ _ => Err(crate::column::raise_pyspark(
+ other.py(),
+ "SessionNotSameException",
+ "SESSION_NOT_SAME",
+ &[],
+ )),
+ }
+ }
+
/// The pandas DataFrame of the result, as `SparkConnectClient.to_pandas`
makes it: the
/// settings of the conversion first, in one request; then the Arrow
result with the schema
/// that the server sends, which `pyspark.sql.connect.client._to_pandas`
converts.
@@ -676,10 +699,11 @@ impl PyDataFrame {
#[pyo3(signature = (other, on=None, how=None))]
fn join(
&self,
- other: &PyDataFrame,
+ other: &Bound<'_, PyAny>,
on: Option<Bound<'_, PyAny>>,
how: Option<&str>,
) -> PyResult<PyDataFrame> {
+ let other = self.same_session(other)?;
let join_type = parse_join_type(how.unwrap_or("inner"))?;
// Resolve `on`: None | Column | str | list[str].
@@ -718,8 +742,11 @@ impl PyDataFrame {
}
/// Cross join.
- fn crossJoin(&self, other: &PyDataFrame) -> PyDataFrame {
- PyDataFrame::new(self.dataframe.cross_join(&other.dataframe))
+ fn crossJoin(&self, other: &Bound<'_, PyAny>) -> PyResult<PyDataFrame> {
+ let other = self.same_session(other)?;
+ Ok(PyDataFrame::new(
+ self.dataframe.cross_join(&other.dataframe),
+ ))
}
/// Lateral (correlated) join. `on` is an optional Column condition; `how`
@@ -727,10 +754,11 @@ impl PyDataFrame {
#[pyo3(signature = (other, on=None, how=None))]
fn lateralJoin(
&self,
- other: &PyDataFrame,
+ other: &Bound<'_, PyAny>,
on: Option<PyRef<PyColumn>>,
how: Option<&str>,
) -> PyResult<PyDataFrame> {
+ let other = self.same_session(other)?;
let join_type = parse_join_type(how.unwrap_or("inner"))?;
let on_col = on.map(|c| c.column.clone());
Ok(PyDataFrame::new(self.dataframe.lateral_join(
@@ -857,40 +885,52 @@ impl PyDataFrame {
/// Zip this DataFrame with another column-wise (side-by-side). Mirrors
/// `DataFrame.zip(other)`.
- fn zip(&self, other: &PyDataFrame) -> PyResult<PyDataFrame> {
+ fn zip(&self, other: &Bound<'_, PyAny>) -> PyResult<PyDataFrame> {
+ let other = self.same_session(other)?;
Ok(PyDataFrame::new(
self.dataframe.zip(&other.dataframe).to_pyerr()?,
))
}
/// Union with another DataFrame.
- fn union(&self, other: &PyDataFrame) -> PyDataFrame {
- PyDataFrame::new(self.dataframe.union(&other.dataframe))
+ fn union(&self, other: &Bound<'_, PyAny>) -> PyResult<PyDataFrame> {
+ let other = self.same_session(other)?;
+ Ok(PyDataFrame::new(self.dataframe.union(&other.dataframe)))
}
/// Union by name, optionally filling columns missing on one side with
null.
#[pyo3(signature = (other, allowMissingColumns=false))]
- fn unionByName(&self, other: &PyDataFrame, allowMissingColumns: bool) ->
PyDataFrame {
- PyDataFrame::new(
+ fn unionByName(
+ &self,
+ other: &Bound<'_, PyAny>,
+ allowMissingColumns: bool,
+ ) -> PyResult<PyDataFrame> {
+ let other = self.same_session(other)?;
+ Ok(PyDataFrame::new(
self.dataframe
.union_by_name_opt(&other.dataframe, allowMissingColumns),
- )
+ ))
}
/// Intersect with another DataFrame.
- fn intersect(&self, other: &PyDataFrame) -> PyDataFrame {
- PyDataFrame::new(self.dataframe.intersect(&other.dataframe))
+ fn intersect(&self, other: &Bound<'_, PyAny>) -> PyResult<PyDataFrame> {
+ let other = self.same_session(other)?;
+ Ok(PyDataFrame::new(self.dataframe.intersect(&other.dataframe)))
}
/// Intersect keeping duplicates (reference pyspark `intersectAll`).
#[pyo3(name = "intersectAll")]
- fn intersect_all(&self, other: &PyDataFrame) -> PyDataFrame {
- PyDataFrame::new(self.dataframe.intersect_all(&other.dataframe))
+ fn intersect_all(&self, other: &Bound<'_, PyAny>) -> PyResult<PyDataFrame>
{
+ let other = self.same_session(other)?;
+ Ok(PyDataFrame::new(
+ self.dataframe.intersect_all(&other.dataframe),
+ ))
}
/// Subtract another DataFrame.
- fn subtract(&self, other: &PyDataFrame) -> PyDataFrame {
- PyDataFrame::new(self.dataframe.subtract(&other.dataframe))
+ fn subtract(&self, other: &Bound<'_, PyAny>) -> PyResult<PyDataFrame> {
+ let other = self.same_session(other)?;
+ Ok(PyDataFrame::new(self.dataframe.subtract(&other.dataframe)))
}
/// Repartition. Mirrors reference `repartition(numPartitions, *cols)`
where the
@@ -1052,8 +1092,13 @@ impl PyDataFrame {
/// Whether this DataFrame is semantically equal to `other`.
#[pyo3(name = "sameSemantics")]
- fn same_semantics(&self, py: Python<'_>, other: &PyDataFrame) ->
PyResult<bool> {
- py.detach(|| self.dataframe.same_semantics(&other.dataframe))
+ fn same_semantics(&self, py: Python<'_>, other: &Bound<'_, PyAny>) ->
PyResult<bool> {
+ // As PySpark's: a DataFrame, else NOT_EXPECTED_TYPE, of the same
session.
+ if !other.is_instance_of::<PyDataFrame>() {
+ return Err(not_expected_type(other, "other", "DataFrame"));
+ }
+ let other = self.same_session(other)?.dataframe.clone();
+ py.detach(|| self.dataframe.same_semantics(&other))
.to_pyerr()
}
@@ -1079,8 +1124,11 @@ impl PyDataFrame {
/// Except all (keeps duplicates).
#[pyo3(name = "exceptAll")]
- fn except_all(&self, other: &PyDataFrame) -> PyDataFrame {
- PyDataFrame::new(self.dataframe.except_all(&other.dataframe))
+ fn except_all(&self, other: &Bound<'_, PyAny>) -> PyResult<PyDataFrame> {
+ let other = self.same_session(other)?;
+ Ok(PyDataFrame::new(
+ self.dataframe.except_all(&other.dataframe),
+ ))
}
/// Rename multiple columns from an {old: new} mapping.
@@ -2021,8 +2069,9 @@ impl PyDataFrame {
}
/// Additional set operation: unionAll (alias for union).
- fn unionAll(&self, other: &PyDataFrame) -> PyDataFrame {
- PyDataFrame::new(self.dataframe.union(&other.dataframe))
+ fn unionAll(&self, other: &Bound<'_, PyAny>) -> PyResult<PyDataFrame> {
+ let other = self.same_session(other)?;
+ Ok(PyDataFrame::new(self.dataframe.union(&other.dataframe)))
}
/// The V1 writer interface (`df.write`).
@@ -2228,21 +2277,22 @@ impl PyDataFrame {
#[pyo3(name = "nearestByJoin", signature = (other, rankingExpression,
numResults = 1, mode = "brute", direction = "asc", joinType = "inner"))]
fn nearest_by_join(
&self,
- other: &PyDataFrame,
+ other: &Bound<'_, PyAny>,
rankingExpression: &PyColumn,
numResults: i32,
mode: &str,
direction: &str,
joinType: &str,
- ) -> PyDataFrame {
- PyDataFrame::new(self.dataframe.nearest_by_join(
+ ) -> PyResult<PyDataFrame> {
+ let other = self.same_session(other)?;
+ Ok(PyDataFrame::new(self.dataframe.nearest_by_join(
&other.dataframe,
rankingExpression.column.clone(),
numResults,
mode,
direction,
joinType,
- ))
+ )))
}
/// `DataFrame.metadataColumn`.
diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index 971282a..30c19bd 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -9,6 +9,25 @@ use crate::dataframe::{
};
use crate::errors::ResultExt;
+/// As PySpark's `_check_same_session` of the DataFrames of grouped data:
`SESSION_NOT_SAME`
+/// unless `other` is of the session of `this`.
+fn check_same_session(
+ py: Python<'_>,
+ this: &RustGroupedData,
+ other: &RustGroupedData,
+) -> PyResult<()> {
+ if this.spark_session().session_id() == other.spark_session().session_id()
{
+ Ok(())
+ } else {
+ Err(crate::column::raise_pyspark(
+ py,
+ "SessionNotSameException",
+ "SESSION_NOT_SAME",
+ &[],
+ ))
+ }
+}
+
/// Python wrapper for Spark GroupedData.
#[pyclass(name = "GroupedData", module = "pyspark.sql.group")]
pub struct PyGroupedData {
@@ -236,10 +255,11 @@ impl PyGroupedData {
/// `GroupedData.cogroup`.
#[pyo3(name = "cogroup")]
- fn cogroup(&self, other: &PyGroupedData) -> PyCoGroupedData {
- PyCoGroupedData {
+ fn cogroup(&self, py: Python<'_>, other: &PyGroupedData) ->
PyResult<PyCoGroupedData> {
+ check_same_session(py, &self.grouped_data, &other.grouped_data)?;
+ Ok(PyCoGroupedData {
inner: self.grouped_data.cogroup(&other.grouped_data),
- }
+ })
}
/// `GroupedData.applyInPandasWithState` (stateful streaming).
@@ -293,6 +313,9 @@ impl PyGroupedData {
initialState: Option<PyRef<'_, PyGroupedData>>,
eventTimeColumnName: &str,
) -> PyResult<PyDataFrame> {
+ if let Some(state) = &initialState {
+ check_same_session(py, &self.grouped_data, &state.grouped_data)?;
+ }
let (func, et) = twus_func(
py,
&statefulProcessor,
@@ -340,6 +363,9 @@ impl PyGroupedData {
initialState: Option<PyRef<'_, PyGroupedData>>,
eventTimeColumnName: &str,
) -> PyResult<PyDataFrame> {
+ if let Some(state) = &initialState {
+ check_same_session(py, &self.grouped_data, &state.grouped_data)?;
+ }
let (func, et) = twus_func(
py,
&statefulProcessor,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 6042f4d..0189f28 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1215,6 +1215,20 @@ def
test_sample_sort_group_by_drop_and_select_expr_as_pyspark(session):
assert df.sample(0.5, 42).collect() == df.sample(fraction=0.5,
seed=42).collect()
+
+def test_dataframes_of_other_sessions_are_not_taken_as_pyspark(session):
+ # As PySpark's, a DataFrame of another session, whose temporary views this
session would
+ # not find, is not taken: SESSION_NOT_SAME.
+ from pyspark.errors import SessionNotSameException
+
+ other = session.newSession()
+ other.range(3).createOrReplaceTempView("view_of_another_session")
+ theirs = other.table("view_of_another_session")
+ for call in (lambda: session.range(3).join(theirs, "id"),
+ lambda: session.range(3).union(theirs)):
+ with pytest.raises(SessionNotSameException):
+ call()
+
def test_column_attributes_select_struct_fields_and_slices_substrings(session):
df = session.sql("SELECT named_struct('x', 1, 'y', named_struct('z', 'q'))
AS s, 'abcdef' AS t")
out = df.select(df.s.x, df.s.y.z, F.col("s").x.alias("x2"), df.t[2:3],
F.col("t")[F.lit(1):F.lit(2)])
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 40a3e62..f5f1f5e 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -7763,6 +7763,36 @@ def _offline_session():
return SparkSession.builder.remote("sc://localhost:1").getOrCreate()
+def test_dataframes_of_other_sessions_raise_session_not_same_as_pyspark():
+ # As PySpark's `_check_same_session`, of the DataFrames that the methods
take, before any
+ # request: SESSION_NOT_SAME for a DataFrame of another session, and for
what is not one.
+ from pyspark.errors import PySparkTypeError, SessionNotSameException
+ from pyspark.sql import functions as F
+
+ spark = _offline_session()
+ a, b = spark.range(1), spark.newSession().range(1)
+ for call in (
+ lambda: a.join(b), lambda: a.crossJoin(b), lambda: a.lateralJoin(b),
+ lambda: a.union(b), lambda: a.unionAll(b), lambda: a.unionByName(b),
+ lambda: a.intersect(b), lambda: a.intersectAll(b), lambda:
a.subtract(b),
+ lambda: a.exceptAll(b), lambda: a.zip(b), lambda: a.nearestByJoin(b,
F.lit(1)),
+ lambda: a.sameSemantics(b), lambda: a.union(1), lambda: a.join("id"),
+ lambda: a.groupBy("id").cogroup(b.groupBy("id")),
+ ):
+ with pytest.raises(SessionNotSameException) as error:
+ call()
+ assert error.value.getCondition() == "SESSION_NOT_SAME"
+ with pytest.raises(PySparkTypeError) as error:
+ a.sameSemantics(None)
+ assert error.value.getMessageParameters() == {
+ "expected_type": "DataFrame", "arg_name": "other", "arg_type":
"NoneType"}
+ # Of the same session, they are taken.
+ c = spark.range(1)
+ for result in (a.join(c), a.union(c), a.unionByName(c), a.exceptAll(c)):
+ assert type(result).__name__ == "DataFrame"
+ assert type(a.groupBy("id").cogroup(c.groupBy("id"))).__name__ ==
"PandasCogroupedOps"
+
+
def
test_explain_checks_its_arguments_and_prints_with_print_as_pyspark(mock_connect):
# As PySpark's DataFrame._explain_string, which checks its arguments and
the mode, and
# explain, which prints its plan with Python's print.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]