This is an automated email from the ASF dual-hosted git repository.
gaogaotiantian pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new d7563c254bf1 [SPARK-58159][PYTHON] Support `with` statement for
connect session
d7563c254bf1 is described below
commit d7563c254bf119f77c87b97e410f6ad924d701c8
Author: Tian Gao <[email protected]>
AuthorDate: Wed Jul 29 17:49:52 2026 -0700
[SPARK-58159][PYTHON] Support `with` statement for connect session
### What changes were proposed in this pull request?
Support `with` statement for connect session.
### Why are the changes needed?
It is supported for classic spark session. We should support this for
connect session too.
### Does this PR introduce _any_ user-facing change?
Yes, user can use
```python
with SparkSession.builder.remote("local[*]").getOrCreate() as spark:
...
```
Now
### How was this patch tested?
A feature test is added.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #57291 from gaogaotiantian/add-connect-session-context.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: Tian Gao <[email protected]>
(cherry picked from commit 6c37fe363429e7537a124b96f32fdb573bd3b896)
Signed-off-by: Tian Gao <[email protected]>
---
python/pyspark/sql/connect/session.py | 51 ++++++++++++++++++++++
.../sql/tests/connect/test_connect_session.py | 10 +++++
2 files changed, 61 insertions(+)
diff --git a/python/pyspark/sql/connect/session.py
b/python/pyspark/sql/connect/session.py
index 99da7308a1a0..1a3e1c1aebb3 100644
--- a/python/pyspark/sql/connect/session.py
+++ b/python/pyspark/sql/connect/session.py
@@ -24,6 +24,7 @@ import warnings
from collections.abc import Callable, Sized
import functools
from threading import RLock
+from types import TracebackType
from typing import (
Optional,
Any,
@@ -32,6 +33,7 @@ from typing import (
Dict,
List,
Tuple,
+ Type,
Set,
cast,
overload,
@@ -989,6 +991,55 @@ class SparkSession:
if "SPARK_REMOTE" in os.environ:
del os.environ["SPARK_REMOTE"]
+ def __enter__(self) -> "SparkSession":
+ """
+ Enable 'with SparkSession.builder.(...).getOrCreate() as session: app'
syntax.
+
+ .. versionadded:: 4.3.0
+
+ Examples
+ --------
+ >>> with SparkSession.builder.remote("local[*]").getOrCreate() as
session:
+ ... session.range(5).show() # doctest: +SKIP
+ +---+
+ | id|
+ +---+
+ | 0|
+ | 1|
+ | 2|
+ | 3|
+ | 4|
+ +---+
+ """
+ return self
+
+ def __exit__(
+ self,
+ exc_type: Optional[Type[BaseException]],
+ exc_value: Optional[BaseException],
+ traceback: Optional[TracebackType],
+ ) -> None:
+ """
+ Enable 'with SparkSession.builder.(...).getOrCreate() as session: app'
syntax.
+
+ .. versionadded:: 4.3.0
+
+ Examples
+ --------
+ >>> with SparkSession.builder.remote("local[*]").getOrCreate() as
session:
+ ... session.range(5).show() # doctest: +SKIP
+ +---+
+ | id|
+ +---+
+ | 0|
+ | 1|
+ | 2|
+ | 3|
+ | 4|
+ +---+
+ """
+ self.stop()
+
@property
def is_stopped(self) -> bool:
"""
diff --git a/python/pyspark/sql/tests/connect/test_connect_session.py
b/python/pyspark/sql/tests/connect/test_connect_session.py
index 92be5c83c048..3b5a816b7d24 100644
--- a/python/pyspark/sql/tests/connect/test_connect_session.py
+++ b/python/pyspark/sql/tests/connect/test_connect_session.py
@@ -115,6 +115,16 @@ class SparkConnectSessionTests(ReusedConnectTestCase):
self.spark.conf.get("some.conf")
self._check_no_active_session_error(e.exception)
+ def test_with_block(self):
+ with PySparkSession.builder.remote("local[*]").getOrCreate() as
session:
+ df = session.sql("select 1 as a, 2 as b")
+ self.assertEqual(df.collect(), [(1, 2)])
+
+ # Confirm that the session is stopped
+ with self.assertRaises(SparkConnectException) as e:
+ self.spark.sql("select 1")
+ self._check_no_active_session_error(e.exception)
+
def test_error_enrichment_message(self):
with self.sql_conf(
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]