This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 7b7ebbd  [SPARK-61052][CONNECT][PYTHON] Release the GIL across the 
requests of the writers, the catalog and the streaming queries in the Python 
client as PySpark does
7b7ebbd is described below

commit 7b7ebbdb4fcdbc8ce539b2879d2d96ed08fef7f6
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 14:20:38 2026 +0900

    [SPARK-61052][CONNECT][PYTHON] Release the GIL across the requests of the 
writers, the catalog and the streaming queries in the Python client as PySpark 
does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61052](https://issues.apache.org/jira/browse/SPARK-61052): the 
requests of the writers, the catalog and the streaming queries now release the 
GIL while they wait for the server, as PySpark Connect's requests do through 
grpcio. So other Python threads run meanwhile.
    
    They are:
    - `df.write.save`, `saveAsTable`, `insertInto` and the format writers, such 
as `parquet`
    - the writes of `df.writeTo` and `mergeInto`
    - the methods of `spark.catalog`
    - `df.isEmpty`
    - the `start` and `toTable` of `df.writeStream`
    - the methods of a `StreamingQuery`, such as `awaitTermination`, and of 
`spark.streams`, such as `addListener`
    
    The client already released it for `collect`, `count`, `sql` and the other 
requests of a DataFrame. The new `detached` helpers of the catalog and of the 
module run a request without the GIL.
    
    ### Why are the changes needed?
    
    The client held the GIL across these requests, so every other Python thread 
waited for them. `query.awaitTermination(3)` stopped every other thread for 3 
seconds. A listener of `spark.streams` could not run its callbacks meanwhile: 
none of the progress of a query reached it during `awaitTermination`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Other Python threads, and the listeners of streaming queries, run 
during these requests, as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks that another thread runs during 
`awaitTermination`, a write and catalog requests, and that a listener receives 
the progress of the query during `awaitTermination`. It passes with official 
PySpark 4.3.0, and fails without this change.
    - A new offline test checks that this thread runs while 12 of these 
requests wait for a mock Connect server. Without this change, they hold the GIL 
until the server gives up.
    - The lib tests of both crates (589 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1989 passed, 8 skipped), and the 
live drop-in and Arrow suites (134 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #273 from HyukjinKwon/SPARK-61052-gil.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/catalog.rs    | 268 ++++++++++++++++++++++++------------
 crates/pyspark-rs/src/dataframe.rs  |  85 ++++++++----
 crates/pyspark-rs/src/streaming.rs  |  85 +++++++-----
 python/tests/test_dropin_e2e.py     |  66 +++++++++
 python/tests/test_dropin_offline.py |  47 +++++++
 5 files changed, 401 insertions(+), 150 deletions(-)

diff --git a/crates/pyspark-rs/src/catalog.rs b/crates/pyspark-rs/src/catalog.rs
index 18e13e7..a251a06 100644
--- a/crates/pyspark-rs/src/catalog.rs
+++ b/crates/pyspark-rs/src/catalog.rs
@@ -5,6 +5,8 @@ use spark_connect::catalog::{
     Catalog, CatalogMetadata, Column, Database, Function, Table, 
TablePartition,
 };
 
+use spark_connect_core::error::SparkError;
+
 use crate::dataframe::PyDataFrame;
 use crate::errors::ResultExt;
 
@@ -279,74 +281,101 @@ impl PyCatalog {
     pub fn new(catalog: Catalog) -> Self {
         PyCatalog { catalog }
     }
+
+    /// `f` of the catalog without the GIL, as PySpark's requests to the 
server release it
+    /// while they wait, so that other Python threads run.
+    fn detached<T: Send>(
+        &self,
+        py: Python<'_>,
+        f: impl FnOnce(&Catalog) -> Result<T, Box<SparkError>> + Send,
+    ) -> PyResult<T> {
+        py.detach(|| f(&self.catalog)).to_pyerr()
+    }
 }
 
 #[pymethods]
 impl PyCatalog {
     /// Get the current catalog.
-    fn currentCatalog(&self) -> PyResult<String> {
-        self.catalog.current_catalog().to_pyerr()
+    fn currentCatalog(&self, py: Python<'_>) -> PyResult<String> {
+        self.detached(py, |catalog| 
catalog.current_catalog().map_err(Box::new))
     }
 
     /// Set the current catalog.
-    fn setCurrentCatalog(&self, catalogName: &str) -> PyResult<()> {
-        self.catalog.set_current_catalog(catalogName).to_pyerr()
+    fn setCurrentCatalog(&self, py: Python<'_>, catalogName: &str) -> 
PyResult<()> {
+        self.detached(py, |catalog| {
+            catalog.set_current_catalog(catalogName).map_err(Box::new)
+        })
     }
 
     /// List catalogs. Mirrors `Catalog.listCatalogs(pattern=None) -> 
List[CatalogMetadata]`.
     #[pyo3(signature = (pattern=None))]
-    fn listCatalogs(&self, pattern: Option<&str>) -> 
PyResult<Vec<PyCatalogMetadata>> {
+    fn listCatalogs(
+        &self,
+        py: Python<'_>,
+        pattern: Option<&str>,
+    ) -> PyResult<Vec<PyCatalogMetadata>> {
         Ok(self
-            .catalog
-            .list_catalogs_typed(pattern)
-            .to_pyerr()?
+            .detached(py, |catalog| {
+                catalog.list_catalogs_typed(pattern).map_err(Box::new)
+            })?
             .into_iter()
             .map(|inner| PyCatalogMetadata { inner })
             .collect())
     }
 
     /// Get current database.
-    fn currentDatabase(&self) -> PyResult<String> {
-        self.catalog.current_database().to_pyerr()
+    fn currentDatabase(&self, py: Python<'_>) -> PyResult<String> {
+        self.detached(py, |catalog| 
catalog.current_database().map_err(Box::new))
     }
 
     /// Set current database.
-    fn setCurrentDatabase(&self, dbName: &str) -> PyResult<()> {
-        self.catalog.set_current_database(dbName).to_pyerr()
+    fn setCurrentDatabase(&self, py: Python<'_>, dbName: &str) -> PyResult<()> 
{
+        self.detached(py, |catalog| {
+            catalog.set_current_database(dbName).map_err(Box::new)
+        })
     }
 
     /// List databases. Mirrors `Catalog.listDatabases(pattern=None) -> 
List[Database]`.
     #[pyo3(signature = (pattern=None))]
-    fn listDatabases(&self, pattern: Option<&str>) -> 
PyResult<Vec<PyDatabase>> {
+    fn listDatabases(&self, py: Python<'_>, pattern: Option<&str>) -> 
PyResult<Vec<PyDatabase>> {
         Ok(self
-            .catalog
-            .list_databases_typed(pattern)
-            .to_pyerr()?
+            .detached(py, |catalog| {
+                catalog.list_databases_typed(pattern).map_err(Box::new)
+            })?
             .into_iter()
             .map(|inner| PyDatabase { inner })
             .collect())
     }
 
     /// Check if database exists.
-    fn databaseExists(&self, dbName: &str) -> PyResult<bool> {
-        self.catalog.database_exists(dbName).to_pyerr()
+    fn databaseExists(&self, py: Python<'_>, dbName: &str) -> PyResult<bool> {
+        self.detached(py, |catalog| {
+            catalog.database_exists(dbName).map_err(Box::new)
+        })
     }
 
     /// Get database info. Mirrors `Catalog.getDatabase(dbName) -> Database`.
-    fn getDatabase(&self, dbName: &str) -> PyResult<PyDatabase> {
+    fn getDatabase(&self, py: Python<'_>, dbName: &str) -> 
PyResult<PyDatabase> {
         Ok(PyDatabase {
-            inner: self.catalog.get_database_typed(dbName).to_pyerr()?,
+            inner: self.detached(py, |catalog| {
+                catalog.get_database_typed(dbName).map_err(Box::new)
+            })?,
         })
     }
 
     /// List tables. Mirrors `Catalog.listTables(dbName=None, pattern=None) -> 
List[Table]`.
     #[pyo3(signature = (dbName=None, pattern=None))]
     #[allow(non_snake_case)]
-    fn listTables(&self, dbName: Option<&str>, pattern: Option<&str>) -> 
PyResult<Vec<PyTable>> {
+    fn listTables(
+        &self,
+        py: Python<'_>,
+        dbName: Option<&str>,
+        pattern: Option<&str>,
+    ) -> PyResult<Vec<PyTable>> {
         Ok(self
-            .catalog
-            .list_tables_typed(dbName, pattern)
-            .to_pyerr()?
+            .detached(py, |catalog| {
+                catalog.list_tables_typed(dbName, pattern).map_err(Box::new)
+            })?
             .into_iter()
             .map(|inner| PyTable { inner })
             .collect())
@@ -356,15 +385,17 @@ impl PyCatalog {
     #[pyo3(signature = (tableName, dbName=None))]
     fn tableExists(&self, py: Python<'_>, tableName: &str, dbName: 
Option<&str>) -> PyResult<bool> {
         warn_db_name(py, "tableExists", dbName)?;
-        self.catalog
-            .table_exists_with_database(tableName, dbName)
-            .to_pyerr()
+        self.detached(py, |catalog| {
+            { catalog.table_exists_with_database(tableName, dbName) 
}.map_err(Box::new)
+        })
     }
 
     /// Get table info. Mirrors `Catalog.getTable(tableName) -> Table`.
-    fn getTable(&self, tableName: &str) -> PyResult<PyTable> {
+    fn getTable(&self, py: Python<'_>, tableName: &str) -> PyResult<PyTable> {
         Ok(PyTable {
-            inner: self.catalog.get_table_typed(tableName).to_pyerr()?,
+            inner: self.detached(py, |catalog| {
+                catalog.get_table_typed(tableName).map_err(Box::new)
+            })?,
         })
     }
 
@@ -373,13 +404,16 @@ impl PyCatalog {
     #[allow(non_snake_case)]
     fn listFunctions(
         &self,
+        py: Python<'_>,
         dbName: Option<&str>,
         pattern: Option<&str>,
     ) -> PyResult<Vec<PyFunction>> {
         Ok(self
-            .catalog
-            .list_functions_typed(dbName, pattern)
-            .to_pyerr()?
+            .detached(py, |catalog| {
+                catalog
+                    .list_functions_typed(dbName, pattern)
+                    .map_err(Box::new)
+            })?
             .into_iter()
             .map(|inner| PyFunction { inner })
             .collect())
@@ -394,15 +428,17 @@ impl PyCatalog {
         dbName: Option<&str>,
     ) -> PyResult<bool> {
         warn_db_name(py, "functionExists", dbName)?;
-        self.catalog
-            .function_exists_with_database(functionName, dbName)
-            .to_pyerr()
+        self.detached(py, |catalog| {
+            { catalog.function_exists_with_database(functionName, dbName) 
}.map_err(Box::new)
+        })
     }
 
     /// Get function info. Mirrors `Catalog.getFunction(functionName) -> 
Function`.
-    fn getFunction(&self, functionName: &str) -> PyResult<PyFunction> {
+    fn getFunction(&self, py: Python<'_>, functionName: &str) -> 
PyResult<PyFunction> {
         Ok(PyFunction {
-            inner: self.catalog.get_function_typed(functionName).to_pyerr()?,
+            inner: self.detached(py, |catalog| {
+                catalog.get_function_typed(functionName).map_err(Box::new)
+            })?,
         })
     }
 
@@ -417,9 +453,11 @@ impl PyCatalog {
     ) -> PyResult<Vec<PyCatalogColumn>> {
         warn_db_name(py, "listColumns", dbName)?;
         Ok(self
-            .catalog
-            .list_columns_typed(tableName, dbName)
-            .to_pyerr()?
+            .detached(py, |catalog| {
+                catalog
+                    .list_columns_typed(tableName, dbName)
+                    .map_err(Box::new)
+            })?
             .into_iter()
             .map(|inner| PyCatalogColumn { inner })
             .collect())
@@ -431,53 +469,70 @@ impl PyCatalog {
     /// `CacheTable.storage_level` field (the same field layout 
`DataFrame.persist` uses);
     /// `None` leaves it unset so the server default applies.
     #[pyo3(signature = (tableName, storageLevel=None))]
-    fn cacheTable(&self, tableName: &str, storageLevel: Option<&Bound<'_, 
PyAny>>) -> PyResult<()> {
+    fn cacheTable(
+        &self,
+        py: Python<'_>,
+        tableName: &str,
+        storageLevel: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<()> {
         let level = storageLevel
             .map(crate::storagelevel::storage_level_proto)
             .transpose()?;
-        self.catalog
-            .cache_table_with_storage_level(tableName, level)
-            .to_pyerr()
+        self.detached(py, |catalog| {
+            { catalog.cache_table_with_storage_level(tableName, level) 
}.map_err(Box::new)
+        })
     }
 
     /// Uncache a table.
-    fn uncacheTable(&self, tableName: &str) -> PyResult<()> {
-        self.catalog.uncache_table(tableName).to_pyerr()
+    fn uncacheTable(&self, py: Python<'_>, tableName: &str) -> PyResult<()> {
+        self.detached(py, |catalog| {
+            catalog.uncache_table(tableName).map_err(Box::new)
+        })
     }
 
     /// Drop temporary view.
-    fn dropTempView(&self, viewName: &str) -> PyResult<bool> {
-        self.catalog.drop_temp_view(viewName).to_pyerr()
+    fn dropTempView(&self, py: Python<'_>, viewName: &str) -> PyResult<bool> {
+        self.detached(py, |catalog| {
+            catalog.drop_temp_view(viewName).map_err(Box::new)
+        })
     }
 
     /// Drop a global temporary view.
-    fn dropGlobalTempView(&self, viewName: &str) -> PyResult<bool> {
-        self.catalog.drop_global_temp_view(viewName).to_pyerr()
+    fn dropGlobalTempView(&self, py: Python<'_>, viewName: &str) -> 
PyResult<bool> {
+        self.detached(py, |catalog| {
+            catalog.drop_global_temp_view(viewName).map_err(Box::new)
+        })
     }
 
     /// Whether a table/view is cached.
-    fn isCached(&self, tableName: &str) -> PyResult<bool> {
-        self.catalog.is_cached(tableName).to_pyerr()
+    fn isCached(&self, py: Python<'_>, tableName: &str) -> PyResult<bool> {
+        self.detached(py, |catalog| 
catalog.is_cached(tableName).map_err(Box::new))
     }
 
     /// Remove all cached tables from the in-memory cache.
-    fn clearCache(&self) -> PyResult<()> {
-        self.catalog.clear_cache().to_pyerr()
+    fn clearCache(&self, py: Python<'_>) -> PyResult<()> {
+        self.detached(py, |catalog| catalog.clear_cache().map_err(Box::new))
     }
 
     /// Invalidate and refresh cached metadata for a table.
-    fn refreshTable(&self, tableName: &str) -> PyResult<()> {
-        self.catalog.refresh_table(tableName).to_pyerr()
+    fn refreshTable(&self, py: Python<'_>, tableName: &str) -> PyResult<()> {
+        self.detached(py, |catalog| {
+            catalog.refresh_table(tableName).map_err(Box::new)
+        })
     }
 
     /// Invalidate and refresh cached data for any table at a path.
-    fn refreshByPath(&self, path: &str) -> PyResult<()> {
-        self.catalog.refresh_by_path(path).to_pyerr()
+    fn refreshByPath(&self, py: Python<'_>, path: &str) -> PyResult<()> {
+        self.detached(py, |catalog| {
+            catalog.refresh_by_path(path).map_err(Box::new)
+        })
     }
 
     /// Recover all the partitions of a table.
-    fn recoverPartitions(&self, tableName: &str) -> PyResult<()> {
-        self.catalog.recover_partitions(tableName).to_pyerr()
+    fn recoverPartitions(&self, py: Python<'_>, tableName: &str) -> 
PyResult<()> {
+        self.detached(py, |catalog| {
+            catalog.recover_partitions(tableName).map_err(Box::new)
+        })
     }
 
     /// Create a table from a data source, as PySpark's: of a StructType 
`schema` and string
@@ -566,6 +621,7 @@ impl PyCatalog {
     #[allow(non_snake_case)]
     fn createDatabase(
         &self,
+        py: Python<'_>,
         dbName: &str,
         ifNotExists: bool,
         properties: Option<&Bound<'_, pyo3::types::PyDict>>,
@@ -576,56 +632,81 @@ impl PyCatalog {
                 props.insert(k.extract::<String>()?, v.extract::<String>()?);
             }
         }
-        self.catalog
-            .create_database(dbName, ifNotExists, props)
-            .to_pyerr()
+        self.detached(py, |catalog| {
+            { catalog.create_database(dbName, ifNotExists, props) 
}.map_err(Box::new)
+        })
     }
 
     /// Drop a database.
     #[pyo3(signature = (dbName, ifExists=false, cascade=false))]
     #[allow(non_snake_case)]
-    fn dropDatabase(&self, dbName: &str, ifExists: bool, cascade: bool) -> 
PyResult<()> {
-        self.catalog
-            .drop_database(dbName, ifExists, cascade)
-            .to_pyerr()
+    fn dropDatabase(
+        &self,
+        py: Python<'_>,
+        dbName: &str,
+        ifExists: bool,
+        cascade: bool,
+    ) -> PyResult<()> {
+        self.detached(py, |catalog| {
+            { catalog.drop_database(dbName, ifExists, cascade) 
}.map_err(Box::new)
+        })
     }
 
     /// Drop a table.
     #[pyo3(signature = (tableName, ifExists=false, purge=false))]
     #[allow(non_snake_case)]
-    fn dropTable(&self, tableName: &str, ifExists: bool, purge: bool) -> 
PyResult<()> {
-        self.catalog
-            .drop_table(tableName, ifExists, purge)
-            .to_pyerr()
+    fn dropTable(
+        &self,
+        py: Python<'_>,
+        tableName: &str,
+        ifExists: bool,
+        purge: bool,
+    ) -> PyResult<()> {
+        self.detached(py, |catalog| {
+            catalog
+                .drop_table(tableName, ifExists, purge)
+                .map_err(Box::new)
+        })
     }
 
     /// Drop a view.
     #[pyo3(signature = (viewName, ifExists=false))]
     #[allow(non_snake_case)]
-    fn dropView(&self, viewName: &str, ifExists: bool) -> PyResult<()> {
-        self.catalog.drop_view(viewName, ifExists).to_pyerr()
+    fn dropView(&self, py: Python<'_>, viewName: &str, ifExists: bool) -> 
PyResult<()> {
+        self.detached(py, |catalog| {
+            catalog.drop_view(viewName, ifExists).map_err(Box::new)
+        })
     }
 
     /// Truncate a table.
     #[allow(non_snake_case)]
-    fn truncateTable(&self, tableName: &str) -> PyResult<()> {
-        self.catalog.truncate_table(tableName).to_pyerr()
+    fn truncateTable(&self, py: Python<'_>, tableName: &str) -> PyResult<()> {
+        self.detached(py, |catalog| {
+            catalog.truncate_table(tableName).map_err(Box::new)
+        })
     }
 
     /// Recover the statistics of a table.
     #[pyo3(signature = (tableName, noScan=false))]
     #[allow(non_snake_case)]
-    fn analyzeTable(&self, tableName: &str, noScan: bool) -> PyResult<()> {
-        self.catalog.analyze_table(tableName, noScan).to_pyerr()
+    fn analyzeTable(&self, py: Python<'_>, tableName: &str, noScan: bool) -> 
PyResult<()> {
+        self.detached(py, |catalog| {
+            catalog.analyze_table(tableName, noScan).map_err(Box::new)
+        })
     }
 
     /// Get the `CREATE TABLE` string of a table.
     #[pyo3(signature = (tableName, asSerde=false))]
     #[allow(non_snake_case)]
-    fn getCreateTableString(&self, tableName: &str, asSerde: bool) -> 
PyResult<String> {
-        self.catalog
-            .get_create_table_string(tableName, asSerde)
-            .to_pyerr()
+    fn getCreateTableString(
+        &self,
+        py: Python<'_>,
+        tableName: &str,
+        asSerde: bool,
+    ) -> PyResult<String> {
+        self.detached(py, |catalog| {
+            { catalog.get_create_table_string(tableName, asSerde) 
}.map_err(Box::new)
+        })
     }
 
     /// Get the properties of a table as a dict.
@@ -635,7 +716,9 @@ impl PyCatalog {
         py: Python<'py>,
         tableName: &str,
     ) -> PyResult<Bound<'py, pyo3::types::PyDict>> {
-        let props = self.catalog.get_table_properties(tableName).to_pyerr()?;
+        let props = self.detached(py, |catalog| {
+            catalog.get_table_properties(tableName).map_err(Box::new)
+        })?;
         let dict = pyo3::types::PyDict::new(py);
         for (k, v) in props {
             dict.set_item(k, v)?;
@@ -645,11 +728,11 @@ impl PyCatalog {
 
     /// List the partitions of a table. Mirrors `Catalog.listPartitions -> 
List[TablePartition]`.
     #[allow(non_snake_case)]
-    fn listPartitions(&self, tableName: &str) -> 
PyResult<Vec<PyTablePartition>> {
+    fn listPartitions(&self, py: Python<'_>, tableName: &str) -> 
PyResult<Vec<PyTablePartition>> {
         Ok(self
-            .catalog
-            .list_partitions_typed(tableName)
-            .to_pyerr()?
+            .detached(py, |catalog| {
+                catalog.list_partitions_typed(tableName).map_err(Box::new)
+            })?
             .into_iter()
             .map(|inner| PyTablePartition { inner })
             .collect())
@@ -658,11 +741,16 @@ impl PyCatalog {
     /// List the views in a database. Mirrors `Catalog.listViews -> 
List[Table]`.
     #[pyo3(signature = (dbName=None, pattern=None))]
     #[allow(non_snake_case)]
-    fn listViews(&self, dbName: Option<&str>, pattern: Option<&str>) -> 
PyResult<Vec<PyTable>> {
+    fn listViews(
+        &self,
+        py: Python<'_>,
+        dbName: Option<&str>,
+        pattern: Option<&str>,
+    ) -> PyResult<Vec<PyTable>> {
         Ok(self
-            .catalog
-            .list_views_typed(dbName, pattern)
-            .to_pyerr()?
+            .detached(py, |catalog| {
+                catalog.list_views_typed(dbName, pattern).map_err(Box::new)
+            })?
             .into_iter()
             .map(|inner| PyTable { inner })
             .collect())
diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 77bbe83..1de5ccc 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -2108,8 +2108,8 @@ impl PyDataFrame {
     }
 
     /// Check if the DataFrame is empty.
-    fn isEmpty(&self) -> PyResult<bool> {
-        self.dataframe.is_empty().to_pyerr()
+    fn isEmpty(&self, py: Python<'_>) -> PyResult<bool> {
+        detached(py, || self.dataframe.is_empty().map_err(Box::new))
     }
 
     /// Print the schema through Python stdout, honoring the maximum nesting 
depth.
@@ -2748,8 +2748,9 @@ impl PyMergeIntoWriter {
         slf
     }
 
-    fn merge(&self) -> PyResult<()> {
-        self.inner.clone().merge().to_pyerr()
+    fn merge(&self, py: Python<'_>) -> PyResult<()> {
+        let writer = self.inner.clone();
+        detached(py, || writer.merge().map_err(Box::new))
     }
 }
 
@@ -3015,6 +3016,7 @@ impl PyDataFrameWriter {
     #[pyo3(signature = (url, table, mode=None, properties=None))]
     fn jdbc(
         &mut self,
+        py: Python<'_>,
         url: &str,
         table: &str,
         mode: Option<&str>,
@@ -3031,13 +3033,14 @@ impl PyDataFrameWriter {
         }
         w = w.option("url", url).option("dbtable", table);
         self.inner = w.clone();
-        w.save(None).to_pyerr()
+        detached(py, || w.save(None).map_err(Box::new))
     }
 
     #[pyo3(signature = (path=None, format=None, mode=None, partitionBy=None, 
**options))]
     #[allow(non_snake_case)]
     fn save(
         &mut self,
+        py: Python<'_>,
         path: Option<String>,
         format: Option<&str>,
         mode: Option<&str>,
@@ -3047,13 +3050,14 @@ impl PyDataFrameWriter {
         let w = apply_write_common(self.inner.clone(), format, mode, 
partitionBy, options)?;
         // PySpark sets the inline format/mode/partitionBy/options on the 
writer itself.
         self.inner = w.clone();
-        w.save(path.as_deref()).to_pyerr()
+        detached(py, || w.save(path.as_deref()).map_err(Box::new))
     }
 
     #[pyo3(name = "saveAsTable", signature = (name, format=None, mode=None, 
partitionBy=None, **options))]
     #[allow(non_snake_case)]
     fn save_as_table(
         &mut self,
+        py: Python<'_>,
         name: &str,
         format: Option<&str>,
         mode: Option<&str>,
@@ -3062,20 +3066,25 @@ impl PyDataFrameWriter {
     ) -> PyResult<()> {
         let w = apply_write_common(self.inner.clone(), format, mode, 
partitionBy, options)?;
         self.inner = w.clone();
-        w.save_as_table(name).to_pyerr()
+        detached(py, || w.save_as_table(name).map_err(Box::new))
     }
 
     /// Insert into an existing table. `overwrite` maps to overwrite/append 
mode,
     /// mirroring reference `DataFrameWriter.insertInto(tableName, 
overwrite=None)`.
     #[pyo3(name = "insertInto", signature = (tableName, overwrite=None))]
-    fn insert_into(&mut self, tableName: &str, overwrite: Option<bool>) -> 
PyResult<()> {
+    fn insert_into(
+        &mut self,
+        py: Python<'_>,
+        tableName: &str,
+        overwrite: Option<bool>,
+    ) -> PyResult<()> {
         let mut w = self.inner.clone();
         if let Some(ow) = overwrite {
             w = w.mode(if ow { "overwrite" } else { "append" });
         }
         // PySpark sets the overwrite/append mode on the writer itself.
         self.inner = w.clone();
-        w.insert_into(tableName).to_pyerr()
+        detached(py, || w.insert_into(tableName).map_err(Box::new))
     }
 
     /// Write as PARQUET - full pyspark signature; each named option is
@@ -3084,6 +3093,7 @@ impl PyDataFrameWriter {
     #[allow(non_snake_case, clippy::too_many_arguments)]
     fn parquet(
         &mut self,
+        py: Python<'_>,
         path: &str,
         mode: Option<&Bound<'_, PyAny>>,
         partitionBy: Option<&Bound<'_, PyAny>>,
@@ -3104,7 +3114,7 @@ impl PyDataFrameWriter {
         w = wset_opt(w, "compression", compression)?;
         // PySpark sets the mode/options/format on the writer itself, then 
saves.
         self.inner = w.clone().format("parquet");
-        w.parquet(path).to_pyerr()
+        detached(py, || w.parquet(path).map_err(Box::new))
     }
 
     /// Write as JSON - full pyspark signature; each named option is
@@ -3113,6 +3123,7 @@ impl PyDataFrameWriter {
     #[allow(non_snake_case, clippy::too_many_arguments)]
     fn json(
         &mut self,
+        py: Python<'_>,
         path: &str,
         mode: Option<&Bound<'_, PyAny>>,
         compression: Option<&Bound<'_, PyAny>>,
@@ -3134,7 +3145,7 @@ impl PyDataFrameWriter {
         w = wset_opt(w, "ignoreNullFields", ignoreNullFields)?;
         // PySpark sets the mode/options/format on the writer itself, then 
saves.
         self.inner = w.clone().format("json");
-        w.json(path).to_pyerr()
+        detached(py, || w.json(path).map_err(Box::new))
     }
 
     /// Write as CSV - full pyspark signature; each named option is
@@ -3143,6 +3154,7 @@ impl PyDataFrameWriter {
     #[allow(non_snake_case, clippy::too_many_arguments)]
     fn csv(
         &mut self,
+        py: Python<'_>,
         path: &str,
         mode: Option<&Bound<'_, PyAny>>,
         compression: Option<&Bound<'_, PyAny>>,
@@ -3184,7 +3196,7 @@ impl PyDataFrameWriter {
         w = wset_opt(w, "lineSep", lineSep)?;
         // PySpark sets the mode/options/format on the writer itself, then 
saves.
         self.inner = w.clone().format("csv");
-        w.csv(path).to_pyerr()
+        detached(py, || w.csv(path).map_err(Box::new))
     }
 
     /// Write as ORC - full pyspark signature; each named option is
@@ -3193,6 +3205,7 @@ impl PyDataFrameWriter {
     #[allow(non_snake_case, clippy::too_many_arguments)]
     fn orc(
         &mut self,
+        py: Python<'_>,
         path: &str,
         mode: Option<&Bound<'_, PyAny>>,
         partitionBy: Option<&Bound<'_, PyAny>>,
@@ -3213,7 +3226,7 @@ impl PyDataFrameWriter {
         w = wset_opt(w, "compression", compression)?;
         // PySpark sets the mode/options/format on the writer itself, then 
saves.
         self.inner = w.clone().format("orc");
-        w.orc(path).to_pyerr()
+        detached(py, || w.orc(path).map_err(Box::new))
     }
 
     /// Write as TEXT - full pyspark signature; each named option is
@@ -3222,6 +3235,7 @@ impl PyDataFrameWriter {
     #[allow(non_snake_case, clippy::too_many_arguments)]
     fn text(
         &mut self,
+        py: Python<'_>,
         path: &str,
         compression: Option<&Bound<'_, PyAny>>,
         lineSep: Option<&Bound<'_, PyAny>>,
@@ -3231,7 +3245,7 @@ impl PyDataFrameWriter {
         w = wset_opt(w, "lineSep", lineSep)?;
         // PySpark sets the mode/options/format on the writer itself, then 
saves.
         self.inner = w.clone().format("text");
-        w.text(path).to_pyerr()
+        detached(py, || w.text(path).map_err(Box::new))
     }
 
     /// Write as XML - full pyspark signature; each named option is
@@ -3240,6 +3254,7 @@ impl PyDataFrameWriter {
     #[allow(non_snake_case, clippy::too_many_arguments)]
     fn xml(
         &mut self,
+        py: Python<'_>,
         path: &str,
         rowTag: Option<&Bound<'_, PyAny>>,
         mode: Option<&Bound<'_, PyAny>>,
@@ -3273,7 +3288,7 @@ impl PyDataFrameWriter {
         w = wset_opt(w, "validateName", validateName)?;
         // PySpark sets the mode/options/format on the writer itself, then 
saves.
         self.inner = w.clone().format("xml");
-        w.xml(path).to_pyerr()
+        detached(py, || w.xml(path).map_err(Box::new))
     }
 }
 
@@ -3368,36 +3383,50 @@ impl PyDataFrameWriterV2 {
         Ok(slf)
     }
 
-    fn create(&mut self) -> PyResult<()> {
-        self.inner.clone().create().to_pyerr()
+    fn create(&mut self, py: Python<'_>) -> PyResult<()> {
+        let writer = self.inner.clone();
+        detached(py, || writer.create().map_err(Box::new))
     }
 
-    fn replace(&mut self) -> PyResult<()> {
-        self.inner.clone().replace().to_pyerr()
+    fn replace(&mut self, py: Python<'_>) -> PyResult<()> {
+        let writer = self.inner.clone();
+        detached(py, || writer.replace().map_err(Box::new))
     }
 
     #[pyo3(name = "createOrReplace")]
-    fn create_or_replace(&mut self) -> PyResult<()> {
-        self.inner.clone().create_or_replace().to_pyerr()
+    fn create_or_replace(&mut self, py: Python<'_>) -> PyResult<()> {
+        let writer = self.inner.clone();
+        detached(py, || writer.create_or_replace().map_err(Box::new))
     }
 
-    fn append(&mut self) -> PyResult<()> {
-        self.inner.clone().append().to_pyerr()
+    fn append(&mut self, py: Python<'_>) -> PyResult<()> {
+        let writer = self.inner.clone();
+        detached(py, || writer.append().map_err(Box::new))
     }
 
     /// Overwrite the rows of `condition`, a Column or a column name, as 
PySpark's `F._to_col`,
     /// else `NOT_EXPECTED_TYPE` of `col`.
-    fn overwrite(&mut self, condition: &Bound<'_, PyAny>) -> PyResult<()> {
-        let condition = column_or_name(condition)?;
-        self.inner.clone().overwrite(condition).to_pyerr()
+    fn overwrite(&mut self, py: Python<'_>, condition: &Bound<'_, PyAny>) -> 
PyResult<()> {
+        let (writer, condition) = (self.inner.clone(), 
column_or_name(condition)?);
+        detached(py, || writer.overwrite(condition).map_err(Box::new))
     }
 
     #[pyo3(name = "overwritePartitions")]
-    fn overwrite_partitions(&mut self) -> PyResult<()> {
-        self.inner.clone().overwrite_partitions().to_pyerr()
+    fn overwrite_partitions(&mut self, py: Python<'_>) -> PyResult<()> {
+        let writer = self.inner.clone();
+        detached(py, || writer.overwrite_partitions().map_err(Box::new))
     }
 }
 
+/// `f` without the GIL, as PySpark's requests to the server release it while 
they wait, so
+/// that other Python threads run. `f` boxes its error, which keeps its result 
small.
+pub(crate) fn detached<T: Send>(
+    py: Python<'_>,
+    f: impl FnOnce() -> Result<T, Box<spark_connect_core::error::SparkError>> 
+ Send,
+) -> PyResult<T> {
+    py.detach(f).to_pyerr()
+}
+
 // ---- Shared helpers for closure-based (UDF) DataFrame/GroupedData methods 
----
 
 /// cloudpickle an arbitrary Python object to bytes (mirrors the scalar UDF 
path,
diff --git a/crates/pyspark-rs/src/streaming.rs 
b/crates/pyspark-rs/src/streaming.rs
index 8b24673..583d15e 100644
--- a/crates/pyspark-rs/src/streaming.rs
+++ b/crates/pyspark-rs/src/streaming.rs
@@ -823,9 +823,10 @@ impl PyDataStreamWriter {
     }
 
     #[pyo3(signature = (path=None, format=None, outputMode=None, 
partitionBy=None, queryName=None, **options))]
-    #[allow(non_snake_case)]
+    #[allow(non_snake_case, clippy::too_many_arguments)]
     fn start(
         &mut self,
+        py: Python<'_>,
         path: Option<&str>,
         format: Option<&str>,
         outputMode: Option<&str>,
@@ -835,14 +836,16 @@ impl PyDataStreamWriter {
     ) -> PyResult<PyStreamingQuery> {
         let writer = self.apply_write_opts(format, outputMode, partitionBy, 
queryName, options)?;
         // Memory/console/foreach sinks take no path; the core treats "" as 
unset.
-        let query = writer.start(path.unwrap_or("")).to_pyerr()?;
+        let query =
+            crate::dataframe::detached(py, || 
writer.start(path.unwrap_or("")).map_err(Box::new))?;
         Ok(PyStreamingQuery::new(query))
     }
 
     #[pyo3(signature = (tableName, format=None, outputMode=None, 
partitionBy=None, queryName=None, **options))]
-    #[allow(non_snake_case)]
+    #[allow(non_snake_case, clippy::too_many_arguments)]
     fn toTable(
         &mut self,
+        py: Python<'_>,
         tableName: &str,
         format: Option<&str>,
         outputMode: Option<&str>,
@@ -851,7 +854,8 @@ impl PyDataStreamWriter {
         options: Option<&Bound<'_, PyDict>>,
     ) -> PyResult<PyStreamingQuery> {
         let writer = self.apply_write_opts(format, outputMode, partitionBy, 
queryName, options)?;
-        let query = writer.to_table(tableName).to_pyerr()?;
+        let query =
+            crate::dataframe::detached(py, || 
writer.to_table(tableName).map_err(Box::new))?;
         Ok(PyStreamingQuery::new(query))
     }
 
@@ -1028,15 +1032,15 @@ impl PyStreamingQuery {
     }
 
     #[getter]
-    fn isActive(&self) -> PyResult<bool> {
-        self.inner.is_active().to_pyerr()
+    fn isActive(&self, py: Python<'_>) -> PyResult<bool> {
+        crate::dataframe::detached(py, || 
self.inner.is_active().map_err(Box::new))
     }
 
     /// The status of the query, as PySpark's: a dict of its `message`, 
`isDataAvailable` and
     /// `isTriggerActive`.
     #[getter]
     fn status<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
-        let status = self.inner.status().to_pyerr()?;
+        let status = crate::dataframe::detached(py, || 
self.inner.status().map_err(Box::new))?;
         let dict = PyDict::new(py);
         dict.set_item("message", status.status_message)?;
         dict.set_item("isDataAvailable", status.is_data_available)?;
@@ -1044,20 +1048,28 @@ impl PyStreamingQuery {
         Ok(dict)
     }
 
-    fn stop(&self) -> PyResult<()> {
-        self.inner.stop().to_pyerr()
+    fn stop(&self, py: Python<'_>) -> PyResult<()> {
+        crate::dataframe::detached(py, || self.inner.stop().map_err(Box::new))
     }
 
     #[pyo3(signature = (timeout=None))]
-    fn awaitTermination(&self, timeout: Option<&Bound<'_, PyAny>>) -> 
PyResult<Option<bool>> {
+    fn awaitTermination(
+        &self,
+        py: Python<'_>,
+        timeout: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<Option<bool>> {
         let timeout = positive_timeout(timeout)?;
-        self.inner.await_termination(timeout).to_pyerr()
+        crate::dataframe::detached(py, || {
+            self.inner.await_termination(timeout).map_err(Box::new)
+        })
     }
 
     /// The last progress of the query, as PySpark's `StreamingQueryProgress`, 
or None.
     #[getter]
     fn lastProgress<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, 
PyAny>>> {
-        match self.inner.last_progress().to_pyerr()? {
+        let progress =
+            crate::dataframe::detached(py, || 
self.inner.last_progress().map_err(Box::new))?;
+        match progress {
             Some(json) => progress_from_json(py, &json).map(Some),
             None => Ok(None),
         }
@@ -1066,16 +1078,14 @@ impl PyStreamingQuery {
     /// The recent progress of the query, each as PySpark's 
`StreamingQueryProgress`.
     #[getter]
     fn recentProgress<'py>(&self, py: Python<'py>) -> PyResult<Vec<Bound<'py, 
PyAny>>> {
-        self.inner
-            .recent_progress()
-            .to_pyerr()?
+        crate::dataframe::detached(py, || 
self.inner.recent_progress().map_err(Box::new))?
             .iter()
             .map(|json| progress_from_json(py, json))
             .collect()
     }
 
-    fn processAllAvailable(&self) -> PyResult<()> {
-        self.inner.process_all_available().to_pyerr()
+    fn processAllAvailable(&self, py: Python<'_>) -> PyResult<()> {
+        crate::dataframe::detached(py, || 
self.inner.process_all_available().map_err(Box::new))
     }
 
     /// Print the plan of the query with Python's `print`, as PySpark's
@@ -1093,7 +1103,9 @@ impl PyStreamingQuery {
     /// `pyspark.errors.exceptions.connect`, of the message of the server 
without the type of
     /// its Java exception, and with its JVM stack trace, and of the error 
class as its reason.
     fn exception(&self, py: Python<'_>) -> PyResult<Option<Py<PyAny>>> {
-        let Some(exception) = self.inner.exception().to_pyerr()? else {
+        let exception =
+            crate::dataframe::detached(py, || 
self.inner.exception().map_err(Box::new))?;
+        let Some(exception) = exception else {
             return Ok(None);
         };
         // The message is the `toString` of the Java StreamingQueryException: 
drop its type.
@@ -1128,24 +1140,30 @@ impl PyStreamingQueryManager {
 #[pymethods]
 impl PyStreamingQueryManager {
     #[getter]
-    fn active(&self) -> PyResult<Vec<PyStreamingQuery>> {
-        let queries = self.inner.active().to_pyerr()?;
+    fn active(&self, py: Python<'_>) -> PyResult<Vec<PyStreamingQuery>> {
+        let queries = crate::dataframe::detached(py, || 
self.inner.active().map_err(Box::new))?;
         Ok(queries.into_iter().map(PyStreamingQuery::new).collect())
     }
 
-    fn get(&self, id: &str) -> PyResult<Option<PyStreamingQuery>> {
-        let query = self.inner.get(id).to_pyerr()?;
+    fn get(&self, py: Python<'_>, id: &str) -> 
PyResult<Option<PyStreamingQuery>> {
+        let query = crate::dataframe::detached(py, || 
self.inner.get(id).map_err(Box::new))?;
         Ok(query.map(PyStreamingQuery::new))
     }
 
     #[pyo3(signature = (timeout=None))]
-    fn awaitAnyTermination(&self, timeout: Option<&Bound<'_, PyAny>>) -> 
PyResult<Option<bool>> {
+    fn awaitAnyTermination(
+        &self,
+        py: Python<'_>,
+        timeout: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<Option<bool>> {
         let timeout = positive_timeout(timeout)?;
-        self.inner.await_any_termination(timeout).to_pyerr()
+        crate::dataframe::detached(py, || {
+            self.inner.await_any_termination(timeout).map_err(Box::new)
+        })
     }
 
-    fn resetTerminated(&self) -> PyResult<()> {
-        self.inner.reset_terminated().to_pyerr()
+    fn resetTerminated(&self, py: Python<'_>) -> PyResult<()> {
+        crate::dataframe::detached(py, || 
self.inner.reset_terminated().map_err(Box::new))
     }
 
     /// Register a client-side listener object (with 
onQueryStarted/Progress/Idle/
@@ -1156,7 +1174,8 @@ impl PyStreamingQueryManager {
         let adapter = Arc::new(PyListenerAdapter {
             listener: listener.clone_ref(py),
         });
-        let id = self.inner.add_listener(adapter).to_pyerr()?;
+        let id =
+            crate::dataframe::detached(py, || 
self.inner.add_listener(adapter).map_err(Box::new))?;
         listener.bind(py).setattr("_rust_listener_id", id)?;
         Ok(())
     }
@@ -1167,7 +1186,7 @@ impl PyStreamingQueryManager {
         let bound = listener.bind(py);
         if let Ok(id_obj) = bound.getattr("_rust_listener_id") {
             let id: String = id_obj.extract()?;
-            self.inner.remove_listener(&id).to_pyerr()?;
+            crate::dataframe::detached(py, || 
self.inner.remove_listener(&id).map_err(Box::new))?;
             let _ = bound.delattr("_rust_listener_id");
         }
         Ok(())
@@ -1175,12 +1194,14 @@ impl PyStreamingQueryManager {
 
     /// Remove all client-side listeners and stop the dispatch thread. Mirrors
     /// `StreamingQueryManager.close`.
-    fn close(&self) -> PyResult<()> {
-        self.inner.close().to_pyerr()
+    fn close(&self, py: Python<'_>) -> PyResult<()> {
+        crate::dataframe::detached(py, || self.inner.close().map_err(Box::new))
     }
 
-    fn streamListenerEvents(&self) -> PyResult<PyListenerEventStream> {
-        let stream = self.inner.listener_event_stream().to_pyerr()?;
+    fn streamListenerEvents(&self, py: Python<'_>) -> 
PyResult<PyListenerEventStream> {
+        let stream = crate::dataframe::detached(py, || {
+            self.inner.listener_event_stream().map_err(Box::new)
+        })?;
         Ok(PyListenerEventStream::new(stream))
     }
 }
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index ca18cd7..de7e129 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -839,6 +839,72 @@ def 
test_registered_udtfs_are_called_from_sql_as_pyspark(session):
     assert session.sql("SELECT * FROM doubled_udtf(21L)").collect() == 
[Row(y=42)]
 
 
+def 
test_writes_the_catalog_and_streaming_queries_release_the_gil_as_pyspark(session,
 tmp_path):
+    # As PySpark's requests to the server, which release the GIL while they 
wait: another
+    # Python thread runs during them.
+    import threading
+    import time
+
+    def ticks_beside(call):
+        ticks, stop = [0], threading.Event()
+
+        def tick():
+            while not stop.is_set():
+                ticks[0] += 1
+                time.sleep(0.001)
+
+        thread = threading.Thread(target=tick)
+        thread.start()
+        try:
+            time.sleep(0.05)
+            before = ticks[0]
+            call()
+            return ticks[0] - before
+        finally:
+            stop.set()
+            thread.join()
+
+    from pyspark.sql.streaming import StreamingQueryListener
+
+    class Listener(StreamingQueryListener):
+        # The times of the progress of the query, which the listener runs in a 
thread.
+        progress = []
+
+        def onQueryStarted(self, event):
+            pass
+
+        def onQueryProgress(self, event):
+            self.progress.append(time.time())
+
+        def onQueryIdle(self, event):
+            pass
+
+        def onQueryTerminated(self, event):
+            pass
+
+    listener = Listener()
+    session.streams.addListener(listener)
+    stream = 
session.readStream.format("rate").load().writeStream.format("noop")
+    query = stream.trigger(processingTime="200 milliseconds").start()
+    try:
+        # The query makes progress first, which the listener receives between 
waits.
+        for _ in range(120):
+            if Listener.progress:
+                break
+            time.sleep(0.25)
+        start = time.time()
+        assert ticks_beside(lambda: query.awaitTermination(3)) > 100
+        end = time.time()
+    finally:
+        query.stop()
+        session.streams.removeListener(listener)
+    # The listener runs during awaitTermination too.
+    assert any(start < at < end for at in Listener.progress)
+    rows = session.range(2_000_000).selectExpr("id", "id * 2 AS v")
+    assert ticks_beside(lambda: rows.write.parquet(str(tmp_path / "rows"))) > 
10
+    assert ticks_beside(lambda: [session.catalog.currentDatabase() for _ in 
range(20)]) > 10
+
+
 def test_catalog_and_stream_start_errors_carry_their_operation_id(session):
     for run in (
         lambda: session.catalog.setCurrentDatabase("no_such_database_60960"),
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index b489554..254d928 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9307,6 +9307,53 @@ def 
test_connect_registry_defines_a_flow_of_spark_sql_without_running_it(mock_co
     assert 5 in 
parse_message(parse_message(parse_message(service.execute_requests[-1])[3][0])[2][0])
 
 
+_REQUESTS_WITHOUT_THE_GIL = {
+    "save": lambda spark: 
spark.range(1).write.format("parquet").save("/tmp/no_such_dir"),
+    "saveAsTable": lambda spark: spark.range(1).write.saveAsTable("t"),
+    "insertInto": lambda spark: spark.range(1).write.insertInto("t"),
+    "parquet": lambda spark: spark.range(1).write.parquet("/tmp/no_such_dir"),
+    "writeTo.create": lambda spark: spark.range(1).writeTo("t").create(),
+    "writeTo.append": lambda spark: spark.range(1).writeTo("t").append(),
+    "mergeInto.merge": lambda spark: spark.range(1)
+    .mergeInto("t", spark.range(1).id == 0)
+    .whenMatched()
+    .delete()
+    .merge(),
+    "isEmpty": lambda spark: spark.range(1).isEmpty(),
+    "catalog.currentDatabase": lambda spark: spark.catalog.currentDatabase(),
+    "catalog.tableExists": lambda spark: spark.catalog.tableExists("t"),
+    "catalog.refreshTable": lambda spark: spark.catalog.refreshTable("t"),
+    "writeStream.start": lambda spark: spark.readStream.format("rate")
+    .load()
+    .writeStream.format("noop")
+    .start(),
+}
+
+
[email protected]("name", sorted(_REQUESTS_WITHOUT_THE_GIL))
+def 
test_requests_of_writes_the_catalog_and_streams_release_the_gil_as_pyspark(mock_connect,
 name):
+    # As PySpark's requests to the server, which release the GIL while they 
wait: this
+    # thread runs while the request waits for the server.
+    spark, service = mock_connect
+    service.block_execute = True
+    with ThreadPoolExecutor(max_workers=1) as pool:
+        request = pool.submit(_REQUESTS_WITHOUT_THE_GIL[name], spark)
+        try:
+            for _ in range(500):
+                if service.execute_calls:
+                    break
+                time.sleep(0.01)
+            else:
+                raise AssertionError("The request never reached the server")
+            assert not request.done()
+        finally:
+            service.release_response()
+        error = request.exception(timeout=10)
+    # Whatever the request makes of the empty response of the mock server, it 
did not hold
+    # the GIL, which the server would have waited for.
+    assert "could not run" not in str(error or "")
+
+
 def test_sql_runs_a_statement_without_holding_the_gil(mock_connect):
     spark, service = mock_connect
     service.block_execute = True


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to