This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 7b7ebbd [SPARK-61052][CONNECT][PYTHON] Release the GIL across the
requests of the writers, the catalog and the streaming queries in the Python
client as PySpark does
7b7ebbd is described below
commit 7b7ebbdb4fcdbc8ce539b2879d2d96ed08fef7f6
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 14:20:38 2026 +0900
[SPARK-61052][CONNECT][PYTHON] Release the GIL across the requests of the
writers, the catalog and the streaming queries in the Python client as PySpark
does
### What changes were proposed in this pull request?
[SPARK-61052](https://issues.apache.org/jira/browse/SPARK-61052): the
requests of the writers, the catalog and the streaming queries now release the
GIL while they wait for the server, as PySpark Connect's requests do through
grpcio. So other Python threads run meanwhile.
They are:
- `df.write.save`, `saveAsTable`, `insertInto` and the format writers, such
as `parquet`
- the writes of `df.writeTo` and `mergeInto`
- the methods of `spark.catalog`
- `df.isEmpty`
- the `start` and `toTable` of `df.writeStream`
- the methods of a `StreamingQuery`, such as `awaitTermination`, and of
`spark.streams`, such as `addListener`
The client already released it for `collect`, `count`, `sql` and the other
requests of a DataFrame. The new `detached` helpers of the catalog and of the
module run a request without the GIL.
### Why are the changes needed?
The client held the GIL across these requests, so every other Python thread
waited for them. `query.awaitTermination(3)` stopped every other thread for 3
seconds. A listener of `spark.streams` could not run its callbacks meanwhile:
none of the progress of a query reached it during `awaitTermination`.
### Does this PR introduce _any_ user-facing change?
Yes. Other Python threads, and the listeners of streaming queries, run
during these requests, as in PySpark 4.3.0.
### How was this patch tested?
- A new live test checks that another thread runs during
`awaitTermination`, a write and catalog requests, and that a listener receives
the progress of the query during `awaitTermination`. It passes with official
PySpark 4.3.0, and fails without this change.
- A new offline test checks that this thread runs while 12 of these
requests wait for a mock Connect server. Without this change, they hold the GIL
until the server gives up.
- The lib tests of both crates (589 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1989 passed, 8 skipped), and the
live drop-in and Arrow suites (134 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #273 from HyukjinKwon/SPARK-61052-gil.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/catalog.rs | 268 ++++++++++++++++++++++++------------
crates/pyspark-rs/src/dataframe.rs | 85 ++++++++----
crates/pyspark-rs/src/streaming.rs | 85 +++++++-----
python/tests/test_dropin_e2e.py | 66 +++++++++
python/tests/test_dropin_offline.py | 47 +++++++
5 files changed, 401 insertions(+), 150 deletions(-)
diff --git a/crates/pyspark-rs/src/catalog.rs b/crates/pyspark-rs/src/catalog.rs
index 18e13e7..a251a06 100644
--- a/crates/pyspark-rs/src/catalog.rs
+++ b/crates/pyspark-rs/src/catalog.rs
@@ -5,6 +5,8 @@ use spark_connect::catalog::{
Catalog, CatalogMetadata, Column, Database, Function, Table,
TablePartition,
};
+use spark_connect_core::error::SparkError;
+
use crate::dataframe::PyDataFrame;
use crate::errors::ResultExt;
@@ -279,74 +281,101 @@ impl PyCatalog {
pub fn new(catalog: Catalog) -> Self {
PyCatalog { catalog }
}
+
+ /// `f` of the catalog without the GIL, as PySpark's requests to the
server release it
+ /// while they wait, so that other Python threads run.
+ fn detached<T: Send>(
+ &self,
+ py: Python<'_>,
+ f: impl FnOnce(&Catalog) -> Result<T, Box<SparkError>> + Send,
+ ) -> PyResult<T> {
+ py.detach(|| f(&self.catalog)).to_pyerr()
+ }
}
#[pymethods]
impl PyCatalog {
/// Get the current catalog.
- fn currentCatalog(&self) -> PyResult<String> {
- self.catalog.current_catalog().to_pyerr()
+ fn currentCatalog(&self, py: Python<'_>) -> PyResult<String> {
+ self.detached(py, |catalog|
catalog.current_catalog().map_err(Box::new))
}
/// Set the current catalog.
- fn setCurrentCatalog(&self, catalogName: &str) -> PyResult<()> {
- self.catalog.set_current_catalog(catalogName).to_pyerr()
+ fn setCurrentCatalog(&self, py: Python<'_>, catalogName: &str) ->
PyResult<()> {
+ self.detached(py, |catalog| {
+ catalog.set_current_catalog(catalogName).map_err(Box::new)
+ })
}
/// List catalogs. Mirrors `Catalog.listCatalogs(pattern=None) ->
List[CatalogMetadata]`.
#[pyo3(signature = (pattern=None))]
- fn listCatalogs(&self, pattern: Option<&str>) ->
PyResult<Vec<PyCatalogMetadata>> {
+ fn listCatalogs(
+ &self,
+ py: Python<'_>,
+ pattern: Option<&str>,
+ ) -> PyResult<Vec<PyCatalogMetadata>> {
Ok(self
- .catalog
- .list_catalogs_typed(pattern)
- .to_pyerr()?
+ .detached(py, |catalog| {
+ catalog.list_catalogs_typed(pattern).map_err(Box::new)
+ })?
.into_iter()
.map(|inner| PyCatalogMetadata { inner })
.collect())
}
/// Get current database.
- fn currentDatabase(&self) -> PyResult<String> {
- self.catalog.current_database().to_pyerr()
+ fn currentDatabase(&self, py: Python<'_>) -> PyResult<String> {
+ self.detached(py, |catalog|
catalog.current_database().map_err(Box::new))
}
/// Set current database.
- fn setCurrentDatabase(&self, dbName: &str) -> PyResult<()> {
- self.catalog.set_current_database(dbName).to_pyerr()
+ fn setCurrentDatabase(&self, py: Python<'_>, dbName: &str) -> PyResult<()>
{
+ self.detached(py, |catalog| {
+ catalog.set_current_database(dbName).map_err(Box::new)
+ })
}
/// List databases. Mirrors `Catalog.listDatabases(pattern=None) ->
List[Database]`.
#[pyo3(signature = (pattern=None))]
- fn listDatabases(&self, pattern: Option<&str>) ->
PyResult<Vec<PyDatabase>> {
+ fn listDatabases(&self, py: Python<'_>, pattern: Option<&str>) ->
PyResult<Vec<PyDatabase>> {
Ok(self
- .catalog
- .list_databases_typed(pattern)
- .to_pyerr()?
+ .detached(py, |catalog| {
+ catalog.list_databases_typed(pattern).map_err(Box::new)
+ })?
.into_iter()
.map(|inner| PyDatabase { inner })
.collect())
}
/// Check if database exists.
- fn databaseExists(&self, dbName: &str) -> PyResult<bool> {
- self.catalog.database_exists(dbName).to_pyerr()
+ fn databaseExists(&self, py: Python<'_>, dbName: &str) -> PyResult<bool> {
+ self.detached(py, |catalog| {
+ catalog.database_exists(dbName).map_err(Box::new)
+ })
}
/// Get database info. Mirrors `Catalog.getDatabase(dbName) -> Database`.
- fn getDatabase(&self, dbName: &str) -> PyResult<PyDatabase> {
+ fn getDatabase(&self, py: Python<'_>, dbName: &str) ->
PyResult<PyDatabase> {
Ok(PyDatabase {
- inner: self.catalog.get_database_typed(dbName).to_pyerr()?,
+ inner: self.detached(py, |catalog| {
+ catalog.get_database_typed(dbName).map_err(Box::new)
+ })?,
})
}
/// List tables. Mirrors `Catalog.listTables(dbName=None, pattern=None) ->
List[Table]`.
#[pyo3(signature = (dbName=None, pattern=None))]
#[allow(non_snake_case)]
- fn listTables(&self, dbName: Option<&str>, pattern: Option<&str>) ->
PyResult<Vec<PyTable>> {
+ fn listTables(
+ &self,
+ py: Python<'_>,
+ dbName: Option<&str>,
+ pattern: Option<&str>,
+ ) -> PyResult<Vec<PyTable>> {
Ok(self
- .catalog
- .list_tables_typed(dbName, pattern)
- .to_pyerr()?
+ .detached(py, |catalog| {
+ catalog.list_tables_typed(dbName, pattern).map_err(Box::new)
+ })?
.into_iter()
.map(|inner| PyTable { inner })
.collect())
@@ -356,15 +385,17 @@ impl PyCatalog {
#[pyo3(signature = (tableName, dbName=None))]
fn tableExists(&self, py: Python<'_>, tableName: &str, dbName:
Option<&str>) -> PyResult<bool> {
warn_db_name(py, "tableExists", dbName)?;
- self.catalog
- .table_exists_with_database(tableName, dbName)
- .to_pyerr()
+ self.detached(py, |catalog| {
+ { catalog.table_exists_with_database(tableName, dbName)
}.map_err(Box::new)
+ })
}
/// Get table info. Mirrors `Catalog.getTable(tableName) -> Table`.
- fn getTable(&self, tableName: &str) -> PyResult<PyTable> {
+ fn getTable(&self, py: Python<'_>, tableName: &str) -> PyResult<PyTable> {
Ok(PyTable {
- inner: self.catalog.get_table_typed(tableName).to_pyerr()?,
+ inner: self.detached(py, |catalog| {
+ catalog.get_table_typed(tableName).map_err(Box::new)
+ })?,
})
}
@@ -373,13 +404,16 @@ impl PyCatalog {
#[allow(non_snake_case)]
fn listFunctions(
&self,
+ py: Python<'_>,
dbName: Option<&str>,
pattern: Option<&str>,
) -> PyResult<Vec<PyFunction>> {
Ok(self
- .catalog
- .list_functions_typed(dbName, pattern)
- .to_pyerr()?
+ .detached(py, |catalog| {
+ catalog
+ .list_functions_typed(dbName, pattern)
+ .map_err(Box::new)
+ })?
.into_iter()
.map(|inner| PyFunction { inner })
.collect())
@@ -394,15 +428,17 @@ impl PyCatalog {
dbName: Option<&str>,
) -> PyResult<bool> {
warn_db_name(py, "functionExists", dbName)?;
- self.catalog
- .function_exists_with_database(functionName, dbName)
- .to_pyerr()
+ self.detached(py, |catalog| {
+ { catalog.function_exists_with_database(functionName, dbName)
}.map_err(Box::new)
+ })
}
/// Get function info. Mirrors `Catalog.getFunction(functionName) ->
Function`.
- fn getFunction(&self, functionName: &str) -> PyResult<PyFunction> {
+ fn getFunction(&self, py: Python<'_>, functionName: &str) ->
PyResult<PyFunction> {
Ok(PyFunction {
- inner: self.catalog.get_function_typed(functionName).to_pyerr()?,
+ inner: self.detached(py, |catalog| {
+ catalog.get_function_typed(functionName).map_err(Box::new)
+ })?,
})
}
@@ -417,9 +453,11 @@ impl PyCatalog {
) -> PyResult<Vec<PyCatalogColumn>> {
warn_db_name(py, "listColumns", dbName)?;
Ok(self
- .catalog
- .list_columns_typed(tableName, dbName)
- .to_pyerr()?
+ .detached(py, |catalog| {
+ catalog
+ .list_columns_typed(tableName, dbName)
+ .map_err(Box::new)
+ })?
.into_iter()
.map(|inner| PyCatalogColumn { inner })
.collect())
@@ -431,53 +469,70 @@ impl PyCatalog {
/// `CacheTable.storage_level` field (the same field layout
`DataFrame.persist` uses);
/// `None` leaves it unset so the server default applies.
#[pyo3(signature = (tableName, storageLevel=None))]
- fn cacheTable(&self, tableName: &str, storageLevel: Option<&Bound<'_,
PyAny>>) -> PyResult<()> {
+ fn cacheTable(
+ &self,
+ py: Python<'_>,
+ tableName: &str,
+ storageLevel: Option<&Bound<'_, PyAny>>,
+ ) -> PyResult<()> {
let level = storageLevel
.map(crate::storagelevel::storage_level_proto)
.transpose()?;
- self.catalog
- .cache_table_with_storage_level(tableName, level)
- .to_pyerr()
+ self.detached(py, |catalog| {
+ { catalog.cache_table_with_storage_level(tableName, level)
}.map_err(Box::new)
+ })
}
/// Uncache a table.
- fn uncacheTable(&self, tableName: &str) -> PyResult<()> {
- self.catalog.uncache_table(tableName).to_pyerr()
+ fn uncacheTable(&self, py: Python<'_>, tableName: &str) -> PyResult<()> {
+ self.detached(py, |catalog| {
+ catalog.uncache_table(tableName).map_err(Box::new)
+ })
}
/// Drop temporary view.
- fn dropTempView(&self, viewName: &str) -> PyResult<bool> {
- self.catalog.drop_temp_view(viewName).to_pyerr()
+ fn dropTempView(&self, py: Python<'_>, viewName: &str) -> PyResult<bool> {
+ self.detached(py, |catalog| {
+ catalog.drop_temp_view(viewName).map_err(Box::new)
+ })
}
/// Drop a global temporary view.
- fn dropGlobalTempView(&self, viewName: &str) -> PyResult<bool> {
- self.catalog.drop_global_temp_view(viewName).to_pyerr()
+ fn dropGlobalTempView(&self, py: Python<'_>, viewName: &str) ->
PyResult<bool> {
+ self.detached(py, |catalog| {
+ catalog.drop_global_temp_view(viewName).map_err(Box::new)
+ })
}
/// Whether a table/view is cached.
- fn isCached(&self, tableName: &str) -> PyResult<bool> {
- self.catalog.is_cached(tableName).to_pyerr()
+ fn isCached(&self, py: Python<'_>, tableName: &str) -> PyResult<bool> {
+ self.detached(py, |catalog|
catalog.is_cached(tableName).map_err(Box::new))
}
/// Remove all cached tables from the in-memory cache.
- fn clearCache(&self) -> PyResult<()> {
- self.catalog.clear_cache().to_pyerr()
+ fn clearCache(&self, py: Python<'_>) -> PyResult<()> {
+ self.detached(py, |catalog| catalog.clear_cache().map_err(Box::new))
}
/// Invalidate and refresh cached metadata for a table.
- fn refreshTable(&self, tableName: &str) -> PyResult<()> {
- self.catalog.refresh_table(tableName).to_pyerr()
+ fn refreshTable(&self, py: Python<'_>, tableName: &str) -> PyResult<()> {
+ self.detached(py, |catalog| {
+ catalog.refresh_table(tableName).map_err(Box::new)
+ })
}
/// Invalidate and refresh cached data for any table at a path.
- fn refreshByPath(&self, path: &str) -> PyResult<()> {
- self.catalog.refresh_by_path(path).to_pyerr()
+ fn refreshByPath(&self, py: Python<'_>, path: &str) -> PyResult<()> {
+ self.detached(py, |catalog| {
+ catalog.refresh_by_path(path).map_err(Box::new)
+ })
}
/// Recover all the partitions of a table.
- fn recoverPartitions(&self, tableName: &str) -> PyResult<()> {
- self.catalog.recover_partitions(tableName).to_pyerr()
+ fn recoverPartitions(&self, py: Python<'_>, tableName: &str) ->
PyResult<()> {
+ self.detached(py, |catalog| {
+ catalog.recover_partitions(tableName).map_err(Box::new)
+ })
}
/// Create a table from a data source, as PySpark's: of a StructType
`schema` and string
@@ -566,6 +621,7 @@ impl PyCatalog {
#[allow(non_snake_case)]
fn createDatabase(
&self,
+ py: Python<'_>,
dbName: &str,
ifNotExists: bool,
properties: Option<&Bound<'_, pyo3::types::PyDict>>,
@@ -576,56 +632,81 @@ impl PyCatalog {
props.insert(k.extract::<String>()?, v.extract::<String>()?);
}
}
- self.catalog
- .create_database(dbName, ifNotExists, props)
- .to_pyerr()
+ self.detached(py, |catalog| {
+ { catalog.create_database(dbName, ifNotExists, props)
}.map_err(Box::new)
+ })
}
/// Drop a database.
#[pyo3(signature = (dbName, ifExists=false, cascade=false))]
#[allow(non_snake_case)]
- fn dropDatabase(&self, dbName: &str, ifExists: bool, cascade: bool) ->
PyResult<()> {
- self.catalog
- .drop_database(dbName, ifExists, cascade)
- .to_pyerr()
+ fn dropDatabase(
+ &self,
+ py: Python<'_>,
+ dbName: &str,
+ ifExists: bool,
+ cascade: bool,
+ ) -> PyResult<()> {
+ self.detached(py, |catalog| {
+ { catalog.drop_database(dbName, ifExists, cascade)
}.map_err(Box::new)
+ })
}
/// Drop a table.
#[pyo3(signature = (tableName, ifExists=false, purge=false))]
#[allow(non_snake_case)]
- fn dropTable(&self, tableName: &str, ifExists: bool, purge: bool) ->
PyResult<()> {
- self.catalog
- .drop_table(tableName, ifExists, purge)
- .to_pyerr()
+ fn dropTable(
+ &self,
+ py: Python<'_>,
+ tableName: &str,
+ ifExists: bool,
+ purge: bool,
+ ) -> PyResult<()> {
+ self.detached(py, |catalog| {
+ catalog
+ .drop_table(tableName, ifExists, purge)
+ .map_err(Box::new)
+ })
}
/// Drop a view.
#[pyo3(signature = (viewName, ifExists=false))]
#[allow(non_snake_case)]
- fn dropView(&self, viewName: &str, ifExists: bool) -> PyResult<()> {
- self.catalog.drop_view(viewName, ifExists).to_pyerr()
+ fn dropView(&self, py: Python<'_>, viewName: &str, ifExists: bool) ->
PyResult<()> {
+ self.detached(py, |catalog| {
+ catalog.drop_view(viewName, ifExists).map_err(Box::new)
+ })
}
/// Truncate a table.
#[allow(non_snake_case)]
- fn truncateTable(&self, tableName: &str) -> PyResult<()> {
- self.catalog.truncate_table(tableName).to_pyerr()
+ fn truncateTable(&self, py: Python<'_>, tableName: &str) -> PyResult<()> {
+ self.detached(py, |catalog| {
+ catalog.truncate_table(tableName).map_err(Box::new)
+ })
}
/// Recover the statistics of a table.
#[pyo3(signature = (tableName, noScan=false))]
#[allow(non_snake_case)]
- fn analyzeTable(&self, tableName: &str, noScan: bool) -> PyResult<()> {
- self.catalog.analyze_table(tableName, noScan).to_pyerr()
+ fn analyzeTable(&self, py: Python<'_>, tableName: &str, noScan: bool) ->
PyResult<()> {
+ self.detached(py, |catalog| {
+ catalog.analyze_table(tableName, noScan).map_err(Box::new)
+ })
}
/// Get the `CREATE TABLE` string of a table.
#[pyo3(signature = (tableName, asSerde=false))]
#[allow(non_snake_case)]
- fn getCreateTableString(&self, tableName: &str, asSerde: bool) ->
PyResult<String> {
- self.catalog
- .get_create_table_string(tableName, asSerde)
- .to_pyerr()
+ fn getCreateTableString(
+ &self,
+ py: Python<'_>,
+ tableName: &str,
+ asSerde: bool,
+ ) -> PyResult<String> {
+ self.detached(py, |catalog| {
+ { catalog.get_create_table_string(tableName, asSerde)
}.map_err(Box::new)
+ })
}
/// Get the properties of a table as a dict.
@@ -635,7 +716,9 @@ impl PyCatalog {
py: Python<'py>,
tableName: &str,
) -> PyResult<Bound<'py, pyo3::types::PyDict>> {
- let props = self.catalog.get_table_properties(tableName).to_pyerr()?;
+ let props = self.detached(py, |catalog| {
+ catalog.get_table_properties(tableName).map_err(Box::new)
+ })?;
let dict = pyo3::types::PyDict::new(py);
for (k, v) in props {
dict.set_item(k, v)?;
@@ -645,11 +728,11 @@ impl PyCatalog {
/// List the partitions of a table. Mirrors `Catalog.listPartitions ->
List[TablePartition]`.
#[allow(non_snake_case)]
- fn listPartitions(&self, tableName: &str) ->
PyResult<Vec<PyTablePartition>> {
+ fn listPartitions(&self, py: Python<'_>, tableName: &str) ->
PyResult<Vec<PyTablePartition>> {
Ok(self
- .catalog
- .list_partitions_typed(tableName)
- .to_pyerr()?
+ .detached(py, |catalog| {
+ catalog.list_partitions_typed(tableName).map_err(Box::new)
+ })?
.into_iter()
.map(|inner| PyTablePartition { inner })
.collect())
@@ -658,11 +741,16 @@ impl PyCatalog {
/// List the views in a database. Mirrors `Catalog.listViews ->
List[Table]`.
#[pyo3(signature = (dbName=None, pattern=None))]
#[allow(non_snake_case)]
- fn listViews(&self, dbName: Option<&str>, pattern: Option<&str>) ->
PyResult<Vec<PyTable>> {
+ fn listViews(
+ &self,
+ py: Python<'_>,
+ dbName: Option<&str>,
+ pattern: Option<&str>,
+ ) -> PyResult<Vec<PyTable>> {
Ok(self
- .catalog
- .list_views_typed(dbName, pattern)
- .to_pyerr()?
+ .detached(py, |catalog| {
+ catalog.list_views_typed(dbName, pattern).map_err(Box::new)
+ })?
.into_iter()
.map(|inner| PyTable { inner })
.collect())
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 77bbe83..1de5ccc 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -2108,8 +2108,8 @@ impl PyDataFrame {
}
/// Check if the DataFrame is empty.
- fn isEmpty(&self) -> PyResult<bool> {
- self.dataframe.is_empty().to_pyerr()
+ fn isEmpty(&self, py: Python<'_>) -> PyResult<bool> {
+ detached(py, || self.dataframe.is_empty().map_err(Box::new))
}
/// Print the schema through Python stdout, honoring the maximum nesting
depth.
@@ -2748,8 +2748,9 @@ impl PyMergeIntoWriter {
slf
}
- fn merge(&self) -> PyResult<()> {
- self.inner.clone().merge().to_pyerr()
+ fn merge(&self, py: Python<'_>) -> PyResult<()> {
+ let writer = self.inner.clone();
+ detached(py, || writer.merge().map_err(Box::new))
}
}
@@ -3015,6 +3016,7 @@ impl PyDataFrameWriter {
#[pyo3(signature = (url, table, mode=None, properties=None))]
fn jdbc(
&mut self,
+ py: Python<'_>,
url: &str,
table: &str,
mode: Option<&str>,
@@ -3031,13 +3033,14 @@ impl PyDataFrameWriter {
}
w = w.option("url", url).option("dbtable", table);
self.inner = w.clone();
- w.save(None).to_pyerr()
+ detached(py, || w.save(None).map_err(Box::new))
}
#[pyo3(signature = (path=None, format=None, mode=None, partitionBy=None,
**options))]
#[allow(non_snake_case)]
fn save(
&mut self,
+ py: Python<'_>,
path: Option<String>,
format: Option<&str>,
mode: Option<&str>,
@@ -3047,13 +3050,14 @@ impl PyDataFrameWriter {
let w = apply_write_common(self.inner.clone(), format, mode,
partitionBy, options)?;
// PySpark sets the inline format/mode/partitionBy/options on the
writer itself.
self.inner = w.clone();
- w.save(path.as_deref()).to_pyerr()
+ detached(py, || w.save(path.as_deref()).map_err(Box::new))
}
#[pyo3(name = "saveAsTable", signature = (name, format=None, mode=None,
partitionBy=None, **options))]
#[allow(non_snake_case)]
fn save_as_table(
&mut self,
+ py: Python<'_>,
name: &str,
format: Option<&str>,
mode: Option<&str>,
@@ -3062,20 +3066,25 @@ impl PyDataFrameWriter {
) -> PyResult<()> {
let w = apply_write_common(self.inner.clone(), format, mode,
partitionBy, options)?;
self.inner = w.clone();
- w.save_as_table(name).to_pyerr()
+ detached(py, || w.save_as_table(name).map_err(Box::new))
}
/// Insert into an existing table. `overwrite` maps to overwrite/append
mode,
/// mirroring reference `DataFrameWriter.insertInto(tableName,
overwrite=None)`.
#[pyo3(name = "insertInto", signature = (tableName, overwrite=None))]
- fn insert_into(&mut self, tableName: &str, overwrite: Option<bool>) ->
PyResult<()> {
+ fn insert_into(
+ &mut self,
+ py: Python<'_>,
+ tableName: &str,
+ overwrite: Option<bool>,
+ ) -> PyResult<()> {
let mut w = self.inner.clone();
if let Some(ow) = overwrite {
w = w.mode(if ow { "overwrite" } else { "append" });
}
// PySpark sets the overwrite/append mode on the writer itself.
self.inner = w.clone();
- w.insert_into(tableName).to_pyerr()
+ detached(py, || w.insert_into(tableName).map_err(Box::new))
}
/// Write as PARQUET - full pyspark signature; each named option is
@@ -3084,6 +3093,7 @@ impl PyDataFrameWriter {
#[allow(non_snake_case, clippy::too_many_arguments)]
fn parquet(
&mut self,
+ py: Python<'_>,
path: &str,
mode: Option<&Bound<'_, PyAny>>,
partitionBy: Option<&Bound<'_, PyAny>>,
@@ -3104,7 +3114,7 @@ impl PyDataFrameWriter {
w = wset_opt(w, "compression", compression)?;
// PySpark sets the mode/options/format on the writer itself, then
saves.
self.inner = w.clone().format("parquet");
- w.parquet(path).to_pyerr()
+ detached(py, || w.parquet(path).map_err(Box::new))
}
/// Write as JSON - full pyspark signature; each named option is
@@ -3113,6 +3123,7 @@ impl PyDataFrameWriter {
#[allow(non_snake_case, clippy::too_many_arguments)]
fn json(
&mut self,
+ py: Python<'_>,
path: &str,
mode: Option<&Bound<'_, PyAny>>,
compression: Option<&Bound<'_, PyAny>>,
@@ -3134,7 +3145,7 @@ impl PyDataFrameWriter {
w = wset_opt(w, "ignoreNullFields", ignoreNullFields)?;
// PySpark sets the mode/options/format on the writer itself, then
saves.
self.inner = w.clone().format("json");
- w.json(path).to_pyerr()
+ detached(py, || w.json(path).map_err(Box::new))
}
/// Write as CSV - full pyspark signature; each named option is
@@ -3143,6 +3154,7 @@ impl PyDataFrameWriter {
#[allow(non_snake_case, clippy::too_many_arguments)]
fn csv(
&mut self,
+ py: Python<'_>,
path: &str,
mode: Option<&Bound<'_, PyAny>>,
compression: Option<&Bound<'_, PyAny>>,
@@ -3184,7 +3196,7 @@ impl PyDataFrameWriter {
w = wset_opt(w, "lineSep", lineSep)?;
// PySpark sets the mode/options/format on the writer itself, then
saves.
self.inner = w.clone().format("csv");
- w.csv(path).to_pyerr()
+ detached(py, || w.csv(path).map_err(Box::new))
}
/// Write as ORC - full pyspark signature; each named option is
@@ -3193,6 +3205,7 @@ impl PyDataFrameWriter {
#[allow(non_snake_case, clippy::too_many_arguments)]
fn orc(
&mut self,
+ py: Python<'_>,
path: &str,
mode: Option<&Bound<'_, PyAny>>,
partitionBy: Option<&Bound<'_, PyAny>>,
@@ -3213,7 +3226,7 @@ impl PyDataFrameWriter {
w = wset_opt(w, "compression", compression)?;
// PySpark sets the mode/options/format on the writer itself, then
saves.
self.inner = w.clone().format("orc");
- w.orc(path).to_pyerr()
+ detached(py, || w.orc(path).map_err(Box::new))
}
/// Write as TEXT - full pyspark signature; each named option is
@@ -3222,6 +3235,7 @@ impl PyDataFrameWriter {
#[allow(non_snake_case, clippy::too_many_arguments)]
fn text(
&mut self,
+ py: Python<'_>,
path: &str,
compression: Option<&Bound<'_, PyAny>>,
lineSep: Option<&Bound<'_, PyAny>>,
@@ -3231,7 +3245,7 @@ impl PyDataFrameWriter {
w = wset_opt(w, "lineSep", lineSep)?;
// PySpark sets the mode/options/format on the writer itself, then
saves.
self.inner = w.clone().format("text");
- w.text(path).to_pyerr()
+ detached(py, || w.text(path).map_err(Box::new))
}
/// Write as XML - full pyspark signature; each named option is
@@ -3240,6 +3254,7 @@ impl PyDataFrameWriter {
#[allow(non_snake_case, clippy::too_many_arguments)]
fn xml(
&mut self,
+ py: Python<'_>,
path: &str,
rowTag: Option<&Bound<'_, PyAny>>,
mode: Option<&Bound<'_, PyAny>>,
@@ -3273,7 +3288,7 @@ impl PyDataFrameWriter {
w = wset_opt(w, "validateName", validateName)?;
// PySpark sets the mode/options/format on the writer itself, then
saves.
self.inner = w.clone().format("xml");
- w.xml(path).to_pyerr()
+ detached(py, || w.xml(path).map_err(Box::new))
}
}
@@ -3368,36 +3383,50 @@ impl PyDataFrameWriterV2 {
Ok(slf)
}
- fn create(&mut self) -> PyResult<()> {
- self.inner.clone().create().to_pyerr()
+ fn create(&mut self, py: Python<'_>) -> PyResult<()> {
+ let writer = self.inner.clone();
+ detached(py, || writer.create().map_err(Box::new))
}
- fn replace(&mut self) -> PyResult<()> {
- self.inner.clone().replace().to_pyerr()
+ fn replace(&mut self, py: Python<'_>) -> PyResult<()> {
+ let writer = self.inner.clone();
+ detached(py, || writer.replace().map_err(Box::new))
}
#[pyo3(name = "createOrReplace")]
- fn create_or_replace(&mut self) -> PyResult<()> {
- self.inner.clone().create_or_replace().to_pyerr()
+ fn create_or_replace(&mut self, py: Python<'_>) -> PyResult<()> {
+ let writer = self.inner.clone();
+ detached(py, || writer.create_or_replace().map_err(Box::new))
}
- fn append(&mut self) -> PyResult<()> {
- self.inner.clone().append().to_pyerr()
+ fn append(&mut self, py: Python<'_>) -> PyResult<()> {
+ let writer = self.inner.clone();
+ detached(py, || writer.append().map_err(Box::new))
}
/// Overwrite the rows of `condition`, a Column or a column name, as
PySpark's `F._to_col`,
/// else `NOT_EXPECTED_TYPE` of `col`.
- fn overwrite(&mut self, condition: &Bound<'_, PyAny>) -> PyResult<()> {
- let condition = column_or_name(condition)?;
- self.inner.clone().overwrite(condition).to_pyerr()
+ fn overwrite(&mut self, py: Python<'_>, condition: &Bound<'_, PyAny>) ->
PyResult<()> {
+ let (writer, condition) = (self.inner.clone(),
column_or_name(condition)?);
+ detached(py, || writer.overwrite(condition).map_err(Box::new))
}
#[pyo3(name = "overwritePartitions")]
- fn overwrite_partitions(&mut self) -> PyResult<()> {
- self.inner.clone().overwrite_partitions().to_pyerr()
+ fn overwrite_partitions(&mut self, py: Python<'_>) -> PyResult<()> {
+ let writer = self.inner.clone();
+ detached(py, || writer.overwrite_partitions().map_err(Box::new))
}
}
+/// `f` without the GIL, as PySpark's requests to the server release it while
they wait, so
+/// that other Python threads run. `f` boxes its error, which keeps its result
small.
+pub(crate) fn detached<T: Send>(
+ py: Python<'_>,
+ f: impl FnOnce() -> Result<T, Box<spark_connect_core::error::SparkError>>
+ Send,
+) -> PyResult<T> {
+ py.detach(f).to_pyerr()
+}
+
// ---- Shared helpers for closure-based (UDF) DataFrame/GroupedData methods
----
/// cloudpickle an arbitrary Python object to bytes (mirrors the scalar UDF
path,
diff --git a/crates/pyspark-rs/src/streaming.rs
b/crates/pyspark-rs/src/streaming.rs
index 8b24673..583d15e 100644
--- a/crates/pyspark-rs/src/streaming.rs
+++ b/crates/pyspark-rs/src/streaming.rs
@@ -823,9 +823,10 @@ impl PyDataStreamWriter {
}
#[pyo3(signature = (path=None, format=None, outputMode=None,
partitionBy=None, queryName=None, **options))]
- #[allow(non_snake_case)]
+ #[allow(non_snake_case, clippy::too_many_arguments)]
fn start(
&mut self,
+ py: Python<'_>,
path: Option<&str>,
format: Option<&str>,
outputMode: Option<&str>,
@@ -835,14 +836,16 @@ impl PyDataStreamWriter {
) -> PyResult<PyStreamingQuery> {
let writer = self.apply_write_opts(format, outputMode, partitionBy,
queryName, options)?;
// Memory/console/foreach sinks take no path; the core treats "" as
unset.
- let query = writer.start(path.unwrap_or("")).to_pyerr()?;
+ let query =
+ crate::dataframe::detached(py, ||
writer.start(path.unwrap_or("")).map_err(Box::new))?;
Ok(PyStreamingQuery::new(query))
}
#[pyo3(signature = (tableName, format=None, outputMode=None,
partitionBy=None, queryName=None, **options))]
- #[allow(non_snake_case)]
+ #[allow(non_snake_case, clippy::too_many_arguments)]
fn toTable(
&mut self,
+ py: Python<'_>,
tableName: &str,
format: Option<&str>,
outputMode: Option<&str>,
@@ -851,7 +854,8 @@ impl PyDataStreamWriter {
options: Option<&Bound<'_, PyDict>>,
) -> PyResult<PyStreamingQuery> {
let writer = self.apply_write_opts(format, outputMode, partitionBy,
queryName, options)?;
- let query = writer.to_table(tableName).to_pyerr()?;
+ let query =
+ crate::dataframe::detached(py, ||
writer.to_table(tableName).map_err(Box::new))?;
Ok(PyStreamingQuery::new(query))
}
@@ -1028,15 +1032,15 @@ impl PyStreamingQuery {
}
#[getter]
- fn isActive(&self) -> PyResult<bool> {
- self.inner.is_active().to_pyerr()
+ fn isActive(&self, py: Python<'_>) -> PyResult<bool> {
+ crate::dataframe::detached(py, ||
self.inner.is_active().map_err(Box::new))
}
/// The status of the query, as PySpark's: a dict of its `message`,
`isDataAvailable` and
/// `isTriggerActive`.
#[getter]
fn status<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
- let status = self.inner.status().to_pyerr()?;
+ let status = crate::dataframe::detached(py, ||
self.inner.status().map_err(Box::new))?;
let dict = PyDict::new(py);
dict.set_item("message", status.status_message)?;
dict.set_item("isDataAvailable", status.is_data_available)?;
@@ -1044,20 +1048,28 @@ impl PyStreamingQuery {
Ok(dict)
}
- fn stop(&self) -> PyResult<()> {
- self.inner.stop().to_pyerr()
+ fn stop(&self, py: Python<'_>) -> PyResult<()> {
+ crate::dataframe::detached(py, || self.inner.stop().map_err(Box::new))
}
#[pyo3(signature = (timeout=None))]
- fn awaitTermination(&self, timeout: Option<&Bound<'_, PyAny>>) ->
PyResult<Option<bool>> {
+ fn awaitTermination(
+ &self,
+ py: Python<'_>,
+ timeout: Option<&Bound<'_, PyAny>>,
+ ) -> PyResult<Option<bool>> {
let timeout = positive_timeout(timeout)?;
- self.inner.await_termination(timeout).to_pyerr()
+ crate::dataframe::detached(py, || {
+ self.inner.await_termination(timeout).map_err(Box::new)
+ })
}
/// The last progress of the query, as PySpark's `StreamingQueryProgress`,
or None.
#[getter]
fn lastProgress<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py,
PyAny>>> {
- match self.inner.last_progress().to_pyerr()? {
+ let progress =
+ crate::dataframe::detached(py, ||
self.inner.last_progress().map_err(Box::new))?;
+ match progress {
Some(json) => progress_from_json(py, &json).map(Some),
None => Ok(None),
}
@@ -1066,16 +1078,14 @@ impl PyStreamingQuery {
/// The recent progress of the query, each as PySpark's
`StreamingQueryProgress`.
#[getter]
fn recentProgress<'py>(&self, py: Python<'py>) -> PyResult<Vec<Bound<'py,
PyAny>>> {
- self.inner
- .recent_progress()
- .to_pyerr()?
+ crate::dataframe::detached(py, ||
self.inner.recent_progress().map_err(Box::new))?
.iter()
.map(|json| progress_from_json(py, json))
.collect()
}
- fn processAllAvailable(&self) -> PyResult<()> {
- self.inner.process_all_available().to_pyerr()
+ fn processAllAvailable(&self, py: Python<'_>) -> PyResult<()> {
+ crate::dataframe::detached(py, ||
self.inner.process_all_available().map_err(Box::new))
}
/// Print the plan of the query with Python's `print`, as PySpark's
@@ -1093,7 +1103,9 @@ impl PyStreamingQuery {
/// `pyspark.errors.exceptions.connect`, of the message of the server
without the type of
/// its Java exception, and with its JVM stack trace, and of the error
class as its reason.
fn exception(&self, py: Python<'_>) -> PyResult<Option<Py<PyAny>>> {
- let Some(exception) = self.inner.exception().to_pyerr()? else {
+ let exception =
+ crate::dataframe::detached(py, ||
self.inner.exception().map_err(Box::new))?;
+ let Some(exception) = exception else {
return Ok(None);
};
// The message is the `toString` of the Java StreamingQueryException:
drop its type.
@@ -1128,24 +1140,30 @@ impl PyStreamingQueryManager {
#[pymethods]
impl PyStreamingQueryManager {
#[getter]
- fn active(&self) -> PyResult<Vec<PyStreamingQuery>> {
- let queries = self.inner.active().to_pyerr()?;
+ fn active(&self, py: Python<'_>) -> PyResult<Vec<PyStreamingQuery>> {
+ let queries = crate::dataframe::detached(py, ||
self.inner.active().map_err(Box::new))?;
Ok(queries.into_iter().map(PyStreamingQuery::new).collect())
}
- fn get(&self, id: &str) -> PyResult<Option<PyStreamingQuery>> {
- let query = self.inner.get(id).to_pyerr()?;
+ fn get(&self, py: Python<'_>, id: &str) ->
PyResult<Option<PyStreamingQuery>> {
+ let query = crate::dataframe::detached(py, ||
self.inner.get(id).map_err(Box::new))?;
Ok(query.map(PyStreamingQuery::new))
}
#[pyo3(signature = (timeout=None))]
- fn awaitAnyTermination(&self, timeout: Option<&Bound<'_, PyAny>>) ->
PyResult<Option<bool>> {
+ fn awaitAnyTermination(
+ &self,
+ py: Python<'_>,
+ timeout: Option<&Bound<'_, PyAny>>,
+ ) -> PyResult<Option<bool>> {
let timeout = positive_timeout(timeout)?;
- self.inner.await_any_termination(timeout).to_pyerr()
+ crate::dataframe::detached(py, || {
+ self.inner.await_any_termination(timeout).map_err(Box::new)
+ })
}
- fn resetTerminated(&self) -> PyResult<()> {
- self.inner.reset_terminated().to_pyerr()
+ fn resetTerminated(&self, py: Python<'_>) -> PyResult<()> {
+ crate::dataframe::detached(py, ||
self.inner.reset_terminated().map_err(Box::new))
}
/// Register a client-side listener object (with
onQueryStarted/Progress/Idle/
@@ -1156,7 +1174,8 @@ impl PyStreamingQueryManager {
let adapter = Arc::new(PyListenerAdapter {
listener: listener.clone_ref(py),
});
- let id = self.inner.add_listener(adapter).to_pyerr()?;
+ let id =
+ crate::dataframe::detached(py, ||
self.inner.add_listener(adapter).map_err(Box::new))?;
listener.bind(py).setattr("_rust_listener_id", id)?;
Ok(())
}
@@ -1167,7 +1186,7 @@ impl PyStreamingQueryManager {
let bound = listener.bind(py);
if let Ok(id_obj) = bound.getattr("_rust_listener_id") {
let id: String = id_obj.extract()?;
- self.inner.remove_listener(&id).to_pyerr()?;
+ crate::dataframe::detached(py, ||
self.inner.remove_listener(&id).map_err(Box::new))?;
let _ = bound.delattr("_rust_listener_id");
}
Ok(())
@@ -1175,12 +1194,14 @@ impl PyStreamingQueryManager {
/// Remove all client-side listeners and stop the dispatch thread. Mirrors
/// `StreamingQueryManager.close`.
- fn close(&self) -> PyResult<()> {
- self.inner.close().to_pyerr()
+ fn close(&self, py: Python<'_>) -> PyResult<()> {
+ crate::dataframe::detached(py, || self.inner.close().map_err(Box::new))
}
- fn streamListenerEvents(&self) -> PyResult<PyListenerEventStream> {
- let stream = self.inner.listener_event_stream().to_pyerr()?;
+ fn streamListenerEvents(&self, py: Python<'_>) ->
PyResult<PyListenerEventStream> {
+ let stream = crate::dataframe::detached(py, || {
+ self.inner.listener_event_stream().map_err(Box::new)
+ })?;
Ok(PyListenerEventStream::new(stream))
}
}
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index ca18cd7..de7e129 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -839,6 +839,72 @@ def
test_registered_udtfs_are_called_from_sql_as_pyspark(session):
assert session.sql("SELECT * FROM doubled_udtf(21L)").collect() ==
[Row(y=42)]
+def
test_writes_the_catalog_and_streaming_queries_release_the_gil_as_pyspark(session,
tmp_path):
+ # As PySpark's requests to the server, which release the GIL while they
wait: another
+ # Python thread runs during them.
+ import threading
+ import time
+
+ def ticks_beside(call):
+ ticks, stop = [0], threading.Event()
+
+ def tick():
+ while not stop.is_set():
+ ticks[0] += 1
+ time.sleep(0.001)
+
+ thread = threading.Thread(target=tick)
+ thread.start()
+ try:
+ time.sleep(0.05)
+ before = ticks[0]
+ call()
+ return ticks[0] - before
+ finally:
+ stop.set()
+ thread.join()
+
+ from pyspark.sql.streaming import StreamingQueryListener
+
+ class Listener(StreamingQueryListener):
+ # The times of the progress of the query, which the listener runs in a
thread.
+ progress = []
+
+ def onQueryStarted(self, event):
+ pass
+
+ def onQueryProgress(self, event):
+ self.progress.append(time.time())
+
+ def onQueryIdle(self, event):
+ pass
+
+ def onQueryTerminated(self, event):
+ pass
+
+ listener = Listener()
+ session.streams.addListener(listener)
+ stream =
session.readStream.format("rate").load().writeStream.format("noop")
+ query = stream.trigger(processingTime="200 milliseconds").start()
+ try:
+ # The query makes progress first, which the listener receives between
waits.
+ for _ in range(120):
+ if Listener.progress:
+ break
+ time.sleep(0.25)
+ start = time.time()
+ assert ticks_beside(lambda: query.awaitTermination(3)) > 100
+ end = time.time()
+ finally:
+ query.stop()
+ session.streams.removeListener(listener)
+ # The listener runs during awaitTermination too.
+ assert any(start < at < end for at in Listener.progress)
+ rows = session.range(2_000_000).selectExpr("id", "id * 2 AS v")
+ assert ticks_beside(lambda: rows.write.parquet(str(tmp_path / "rows"))) >
10
+ assert ticks_beside(lambda: [session.catalog.currentDatabase() for _ in
range(20)]) > 10
+
+
def test_catalog_and_stream_start_errors_carry_their_operation_id(session):
for run in (
lambda: session.catalog.setCurrentDatabase("no_such_database_60960"),
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index b489554..254d928 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9307,6 +9307,53 @@ def
test_connect_registry_defines_a_flow_of_spark_sql_without_running_it(mock_co
assert 5 in
parse_message(parse_message(parse_message(service.execute_requests[-1])[3][0])[2][0])
+_REQUESTS_WITHOUT_THE_GIL = {
+ "save": lambda spark:
spark.range(1).write.format("parquet").save("/tmp/no_such_dir"),
+ "saveAsTable": lambda spark: spark.range(1).write.saveAsTable("t"),
+ "insertInto": lambda spark: spark.range(1).write.insertInto("t"),
+ "parquet": lambda spark: spark.range(1).write.parquet("/tmp/no_such_dir"),
+ "writeTo.create": lambda spark: spark.range(1).writeTo("t").create(),
+ "writeTo.append": lambda spark: spark.range(1).writeTo("t").append(),
+ "mergeInto.merge": lambda spark: spark.range(1)
+ .mergeInto("t", spark.range(1).id == 0)
+ .whenMatched()
+ .delete()
+ .merge(),
+ "isEmpty": lambda spark: spark.range(1).isEmpty(),
+ "catalog.currentDatabase": lambda spark: spark.catalog.currentDatabase(),
+ "catalog.tableExists": lambda spark: spark.catalog.tableExists("t"),
+ "catalog.refreshTable": lambda spark: spark.catalog.refreshTable("t"),
+ "writeStream.start": lambda spark: spark.readStream.format("rate")
+ .load()
+ .writeStream.format("noop")
+ .start(),
+}
+
+
[email protected]("name", sorted(_REQUESTS_WITHOUT_THE_GIL))
+def
test_requests_of_writes_the_catalog_and_streams_release_the_gil_as_pyspark(mock_connect,
name):
+ # As PySpark's requests to the server, which release the GIL while they
wait: this
+ # thread runs while the request waits for the server.
+ spark, service = mock_connect
+ service.block_execute = True
+ with ThreadPoolExecutor(max_workers=1) as pool:
+ request = pool.submit(_REQUESTS_WITHOUT_THE_GIL[name], spark)
+ try:
+ for _ in range(500):
+ if service.execute_calls:
+ break
+ time.sleep(0.01)
+ else:
+ raise AssertionError("The request never reached the server")
+ assert not request.done()
+ finally:
+ service.release_response()
+ error = request.exception(timeout=10)
+ # Whatever the request makes of the empty response of the mock server, it
did not hold
+ # the GIL, which the server would have waited for.
+ assert "could not run" not in str(error or "")
+
+
def test_sql_runs_a_statement_without_holding_the_gil(mock_connect):
spark, service = mock_connect
service.block_execute = True
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]