This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new a019837  [SPARK-61022][CONNECT][PYTHON] Ask the server whether a 
DataFrame is streaming in isStreaming as PySpark does
a019837 is described below

commit a0198373dfa08766a8ff388dbd2e658fef65293b
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 02:35:05 2026 +0900

    [SPARK-61022][CONNECT][PYTHON] Ask the server whether a DataFrame is 
streaming in isStreaming as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61022](https://issues.apache.org/jira/browse/SPARK-61022): 
`df.isStreaming` now asks the server, by an `IsStreaming` AnalyzePlan request 
of the plan, as PySpark Connect's does. It is true for a DataFrame with a 
streaming source anywhere in its plan.
    
    In the Rust API, `DataFrame::is_streaming` returns the server's answer, or 
the error of the request, as a `Result<bool>`.
    
    ### Why are the changes needed?
    
    `isStreaming` checked only whether the root of the plan was a streaming 
read. So it was False for 
`spark.readStream.format("rate").load().select("value")`, and for the filters, 
unions and joins of a streaming DataFrame, which PySpark 4.3.0 reports as 
streaming.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `isStreaming` is True for every DataFrame of a stream, as in PySpark 
4.3.0. In the Rust API, `DataFrame::is_streaming` returns a `Result<bool>`.
    
    ### How was this patch tested?
    
    - A new live test checks `isStreaming` of a stream, of a select, a filter 
and a union of it, and of a join with it. It passes with official PySpark 
4.3.0, and fails without this change.
    - A new offline test checks the `IsStreaming` requests sent to a mock 
Connect server, which now answers them.
    - The e2e coverage test checks a select of a stream. A rate source with a 
schema, and a named source, are now errors of `is_streaming`, as the server 
rejects them when it analyzes the plan, as for PySpark. The test creates the 
directories of its file sources.
    - The lib tests, every integration test against a local Spark 4.3.0 server, 
the offline Python suites, and the live drop-in and Arrow suites and live cases.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #223 from HyukjinKwon/SPARK-61022-is-streaming.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs          |  5 +++--
 crates/spark-connect/src/dataframe.rs       | 32 +++++++++++++++++++---------
 crates/spark-connect/tests/e2e_coverage.rs  | 33 +++++++++++++++++++++++------
 crates/spark-connect/tests/e2e_dataframe.rs |  2 +-
 python/tests/_mock_connect_server.py        |  5 +++++
 python/tests/test_dropin_e2e.py             | 14 ++++++++++++
 python/tests/test_dropin_offline.py         | 14 ++++++++++++
 7 files changed, 85 insertions(+), 20 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index eab80f0..99ec54b 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1080,8 +1080,9 @@ impl PyDataFrame {
     /// Whether this DataFrame contains one or more sources that continuously 
return data.
     #[pyo3(name = "isStreaming")]
     #[getter]
-    fn is_streaming(&self) -> bool {
-        self.dataframe.is_streaming()
+    fn is_streaming(&self, py: Python<'_>) -> PyResult<bool> {
+        py.detach(|| self.dataframe.is_streaming().map_err(Box::new))
+            .to_pyerr()
     }
 
     /// The storage level currently in use for this DataFrame.
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index cc335a4..68d048f 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1875,16 +1875,28 @@ impl DataFrame {
         }
     }
 
-    /// Check if this DataFrame is streaming.
-    pub fn is_streaming(&self) -> bool {
-        // Check if the plan has streaming-related operations
-        matches!(
-            self.plan.as_ref(),
-            LogicalPlan::Read {
-                is_streaming: true,
-                ..
-            }
-        )
+    /// Whether this DataFrame is streaming, mirroring 
`pyspark.sql.DataFrame.isStreaming`,
+    /// which asks the server: whether a source of its plan continuously 
returns data.
+    pub fn is_streaming(&self) -> Result<bool> {
+        let request = proto::AnalyzePlanRequest {
+            session_id: self.session.client().session_id().to_string(),
+            user_context: Some(proto::UserContext::default()),
+            analyze: Some(proto::analyze_plan_request::Analyze::IsStreaming(
+                proto::analyze_plan_request::IsStreaming {
+                    plan: Some(proto::Plan {
+                        op_type: 
Some(proto::plan::OpType::Root(self.relation())),
+                    }),
+                },
+            )),
+            ..Default::default()
+        };
+        let response = block_on(self.session.client().analyze_plan(request))?;
+        match response.result {
+            Some(proto::analyze_plan_response::Result::IsStreaming(r)) => 
Ok(r.is_streaming),
+            _ => Err(SparkError::connect_msg(
+                "AnalyzePlan response did not contain an isStreaming result",
+            )),
+        }
     }
 
     /// Collect the DataFrame and serialize it to Arrow IPC (file format) 
bytes.
diff --git a/crates/spark-connect/tests/e2e_coverage.rs 
b/crates/spark-connect/tests/e2e_coverage.rs
index d1b949a..5dd59af 100644
--- a/crates/spark-connect/tests/e2e_coverage.rs
+++ b/crates/spark-connect/tests/e2e_coverage.rs
@@ -492,7 +492,10 @@ fn streaming_surface() {
         .format("rate")
         .option("rowsPerSecond", "1")
         .load(None);
-    assert!(sdf.is_streaming());
+    assert!(sdf.is_streaming().unwrap());
+    // As PySpark's, which asks the server: a DataFrame of a stream is 
streaming too.
+    assert!(sdf.select([col("value")]).is_streaming().unwrap());
+    assert!(!s.range(1).unwrap().is_streaming().unwrap());
     let mut opts = Map::new();
     opts.insert("rowsPerSecond".to_string(), "1".to_string());
     assert!(s
@@ -500,13 +503,17 @@ fn streaming_surface() {
         .format("rate")
         .options(opts)
         .load(None)
-        .is_streaming());
+        .is_streaming()
+        .unwrap());
+    // The rate source takes no schema, which the server reports when it 
analyzes the plan,
+    // as for PySpark.
     assert!(s
         .read_stream()
         .format("rate")
         .schema("timestamp timestamp, value long")
         .load(None)
-        .is_streaming());
+        .is_streaming()
+        .is_err());
 
     // Start a query and exercise the full StreamingQuery read surface.
     let q = sdf
@@ -550,15 +557,27 @@ fn streaming_surface() {
         .cluster_by(vec!["value"])
         .output_mode("complete");
 
-    // Streaming file-source readers + name(): all build lazy streaming plans
-    // (no execution until start), so is_streaming() is a client-side check.
+    // Streaming file-source readers + name(): all build lazy streaming plans 
(no execution
+    // until start), which the server analyzes for is_streaming(), of 
directories that exist.
     let schema = "value long";
+    for dir in [
+        "/tmp/cov_s_json",
+        "/tmp/cov_s_pq",
+        "/tmp/cov_s_orc",
+        "/tmp/cov_s_csv",
+        "/tmp/cov_s_txt",
+    ] {
+        std::fs::create_dir_all(dir).unwrap();
+    }
+    // A named source needs 
spark.sql.streaming.queryEvolution.enableSourceEvolution, which the
+    // server reports when it analyzes the plan, as for PySpark.
     assert!(s
         .read_stream()
         .name("rate")
         .format("rate")
         .load(None)
-        .is_streaming());
+        .is_streaming()
+        .is_err());
     for df in [
         s.read_stream().schema(schema).json("/tmp/cov_s_json"),
         s.read_stream().schema(schema).parquet("/tmp/cov_s_pq"),
@@ -571,7 +590,7 @@ fn streaming_surface() {
             .schema("value string")
             .text("/tmp/cov_s_txt"),
     ] {
-        assert!(df.is_streaming());
+        assert!(df.is_streaming().unwrap());
     }
 
     // process_all_available drains a rate stream feeding a memory sink, then 
stop.
diff --git a/crates/spark-connect/tests/e2e_dataframe.rs 
b/crates/spark-connect/tests/e2e_dataframe.rs
index 78ed569..d242ffe 100644
--- a/crates/spark-connect/tests/e2e_dataframe.rs
+++ b/crates/spark-connect/tests/e2e_dataframe.rs
@@ -114,7 +114,7 @@ fn dataframe_metadata_json_semantics() {
     assert!(df.same_semantics(&df4(&s)).unwrap());
     let _ = df.semantic_hash().unwrap();
     // flags
-    assert!(!df.is_streaming());
+    assert!(!df.is_streaming().unwrap());
     let _ = df.is_local();
     // input_files (a sql df reads no files -> empty, but the call path runs)
     let _ = df.input_files().unwrap();
diff --git a/python/tests/_mock_connect_server.py 
b/python/tests/_mock_connect_server.py
index 847504b..2a7400a 100644
--- a/python/tests/_mock_connect_server.py
+++ b/python/tests/_mock_connect_server.py
@@ -132,6 +132,11 @@ def serve(connection):
         def analyze(self, request, context):
             self.calls += 1
             self.analyze_requests.append(request)
+            # base.proto: AnalyzePlanRequest.is_streaming=8; 
AnalyzePlanResponse.is_streaming=6,
+            # IsStreaming.is_streaming=1, of `streaming`, which a test sets.
+            if 8 in parse_message(request):
+                streaming = getattr(self, "streaming", False)
+                return message_field(6, varint_field(1, 1) if streaming else 
b"")
             if self.block_response and not self.response_gate.wait(5):
                 context.abort(
                     grpc.StatusCode.DEADLINE_EXCEEDED,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 6905009..3d2066a 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1375,6 +1375,20 @@ def 
test_create_dataframe_merges_the_types_of_every_row_as_pyspark(session):
         assert [tuple(r) for r in df.collect()] == [(text,), ("5",)]
 
 
+def test_dataframes_of_a_stream_are_streaming_as_pyspark(session):
+    # As PySpark's, which asks the server: a DataFrame with a streaming source 
is streaming.
+    stream = session.readStream.format("rate").load()
+    for df in (
+        stream,
+        stream.select("value"),
+        stream.where("value > 1"),
+        stream.union(stream),
+        session.range(1).join(stream.select(F.col("value").alias("id")), "id"),
+    ):
+        assert df.isStreaming is True
+    assert session.range(1).isStreaming is False
+
+
 def test_create_dataframe_raises_merge_errors_as_pyspark(session):
     from pyspark.errors import PySparkException
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 8f00a42..47a4ac4 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3736,6 +3736,20 @@ def test_struct_field_equality():
     assert f1 != f3
 
 
+def test_is_streaming_asks_the_server_as_pyspark(mock_connect):
+    # As PySpark's: isStreaming is the server's answer to an IsStreaming 
request of the plan.
+    spark, service = mock_connect
+    df = spark.range(1).select("id")
+    try:
+        assert df.isStreaming is False
+        service.streaming = True
+        assert df.isStreaming is True
+    finally:
+        service.streaming = False
+    # base.proto: AnalyzePlanRequest.is_streaming=8.
+    assert sum(8 in parse_message(request) for request in 
service.analyze_requests) == 2
+
+
 def test_struct_type_field_access():
     """Test accessing StructType fields by index and name."""
     st = T.StructType([


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to