This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new a019837 [SPARK-61022][CONNECT][PYTHON] Ask the server whether a
DataFrame is streaming in isStreaming as PySpark does
a019837 is described below
commit a0198373dfa08766a8ff388dbd2e658fef65293b
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 02:35:05 2026 +0900
[SPARK-61022][CONNECT][PYTHON] Ask the server whether a DataFrame is
streaming in isStreaming as PySpark does
### What changes were proposed in this pull request?
[SPARK-61022](https://issues.apache.org/jira/browse/SPARK-61022):
`df.isStreaming` now asks the server, by an `IsStreaming` AnalyzePlan request
of the plan, as PySpark Connect's does. It is true for a DataFrame with a
streaming source anywhere in its plan.
In the Rust API, `DataFrame::is_streaming` returns the server's answer, or
the error of the request, as a `Result<bool>`.
### Why are the changes needed?
`isStreaming` checked only whether the root of the plan was a streaming
read. So it was False for
`spark.readStream.format("rate").load().select("value")`, and for the filters,
unions and joins of a streaming DataFrame, which PySpark 4.3.0 reports as
streaming.
### Does this PR introduce _any_ user-facing change?
Yes. `isStreaming` is True for every DataFrame of a stream, as in PySpark
4.3.0. In the Rust API, `DataFrame::is_streaming` returns a `Result<bool>`.
### How was this patch tested?
- A new live test checks `isStreaming` of a stream, of a select, a filter
and a union of it, and of a join with it. It passes with official PySpark
4.3.0, and fails without this change.
- A new offline test checks the `IsStreaming` requests sent to a mock
Connect server, which now answers them.
- The e2e coverage test checks a select of a stream. A rate source with a
schema, and a named source, are now errors of `is_streaming`, as the server
rejects them when it analyzes the plan, as for PySpark. The test creates the
directories of its file sources.
- The lib tests, every integration test against a local Spark 4.3.0 server,
the offline Python suites, and the live drop-in and Arrow suites and live cases.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #223 from HyukjinKwon/SPARK-61022-is-streaming.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 5 +++--
crates/spark-connect/src/dataframe.rs | 32 +++++++++++++++++++---------
crates/spark-connect/tests/e2e_coverage.rs | 33 +++++++++++++++++++++++------
crates/spark-connect/tests/e2e_dataframe.rs | 2 +-
python/tests/_mock_connect_server.py | 5 +++++
python/tests/test_dropin_e2e.py | 14 ++++++++++++
python/tests/test_dropin_offline.py | 14 ++++++++++++
7 files changed, 85 insertions(+), 20 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index eab80f0..99ec54b 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1080,8 +1080,9 @@ impl PyDataFrame {
/// Whether this DataFrame contains one or more sources that continuously
return data.
#[pyo3(name = "isStreaming")]
#[getter]
- fn is_streaming(&self) -> bool {
- self.dataframe.is_streaming()
+ fn is_streaming(&self, py: Python<'_>) -> PyResult<bool> {
+ py.detach(|| self.dataframe.is_streaming().map_err(Box::new))
+ .to_pyerr()
}
/// The storage level currently in use for this DataFrame.
diff --git a/crates/spark-connect/src/dataframe.rs
b/crates/spark-connect/src/dataframe.rs
index cc335a4..68d048f 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1875,16 +1875,28 @@ impl DataFrame {
}
}
- /// Check if this DataFrame is streaming.
- pub fn is_streaming(&self) -> bool {
- // Check if the plan has streaming-related operations
- matches!(
- self.plan.as_ref(),
- LogicalPlan::Read {
- is_streaming: true,
- ..
- }
- )
+ /// Whether this DataFrame is streaming, mirroring
`pyspark.sql.DataFrame.isStreaming`,
+ /// which asks the server: whether a source of its plan continuously
returns data.
+ pub fn is_streaming(&self) -> Result<bool> {
+ let request = proto::AnalyzePlanRequest {
+ session_id: self.session.client().session_id().to_string(),
+ user_context: Some(proto::UserContext::default()),
+ analyze: Some(proto::analyze_plan_request::Analyze::IsStreaming(
+ proto::analyze_plan_request::IsStreaming {
+ plan: Some(proto::Plan {
+ op_type:
Some(proto::plan::OpType::Root(self.relation())),
+ }),
+ },
+ )),
+ ..Default::default()
+ };
+ let response = block_on(self.session.client().analyze_plan(request))?;
+ match response.result {
+ Some(proto::analyze_plan_response::Result::IsStreaming(r)) =>
Ok(r.is_streaming),
+ _ => Err(SparkError::connect_msg(
+ "AnalyzePlan response did not contain an isStreaming result",
+ )),
+ }
}
/// Collect the DataFrame and serialize it to Arrow IPC (file format)
bytes.
diff --git a/crates/spark-connect/tests/e2e_coverage.rs
b/crates/spark-connect/tests/e2e_coverage.rs
index d1b949a..5dd59af 100644
--- a/crates/spark-connect/tests/e2e_coverage.rs
+++ b/crates/spark-connect/tests/e2e_coverage.rs
@@ -492,7 +492,10 @@ fn streaming_surface() {
.format("rate")
.option("rowsPerSecond", "1")
.load(None);
- assert!(sdf.is_streaming());
+ assert!(sdf.is_streaming().unwrap());
+ // As PySpark's, which asks the server: a DataFrame of a stream is
streaming too.
+ assert!(sdf.select([col("value")]).is_streaming().unwrap());
+ assert!(!s.range(1).unwrap().is_streaming().unwrap());
let mut opts = Map::new();
opts.insert("rowsPerSecond".to_string(), "1".to_string());
assert!(s
@@ -500,13 +503,17 @@ fn streaming_surface() {
.format("rate")
.options(opts)
.load(None)
- .is_streaming());
+ .is_streaming()
+ .unwrap());
+ // The rate source takes no schema, which the server reports when it
analyzes the plan,
+ // as for PySpark.
assert!(s
.read_stream()
.format("rate")
.schema("timestamp timestamp, value long")
.load(None)
- .is_streaming());
+ .is_streaming()
+ .is_err());
// Start a query and exercise the full StreamingQuery read surface.
let q = sdf
@@ -550,15 +557,27 @@ fn streaming_surface() {
.cluster_by(vec!["value"])
.output_mode("complete");
- // Streaming file-source readers + name(): all build lazy streaming plans
- // (no execution until start), so is_streaming() is a client-side check.
+ // Streaming file-source readers + name(): all build lazy streaming plans
(no execution
+ // until start), which the server analyzes for is_streaming(), of
directories that exist.
let schema = "value long";
+ for dir in [
+ "/tmp/cov_s_json",
+ "/tmp/cov_s_pq",
+ "/tmp/cov_s_orc",
+ "/tmp/cov_s_csv",
+ "/tmp/cov_s_txt",
+ ] {
+ std::fs::create_dir_all(dir).unwrap();
+ }
+ // A named source needs
spark.sql.streaming.queryEvolution.enableSourceEvolution, which the
+ // server reports when it analyzes the plan, as for PySpark.
assert!(s
.read_stream()
.name("rate")
.format("rate")
.load(None)
- .is_streaming());
+ .is_streaming()
+ .is_err());
for df in [
s.read_stream().schema(schema).json("/tmp/cov_s_json"),
s.read_stream().schema(schema).parquet("/tmp/cov_s_pq"),
@@ -571,7 +590,7 @@ fn streaming_surface() {
.schema("value string")
.text("/tmp/cov_s_txt"),
] {
- assert!(df.is_streaming());
+ assert!(df.is_streaming().unwrap());
}
// process_all_available drains a rate stream feeding a memory sink, then
stop.
diff --git a/crates/spark-connect/tests/e2e_dataframe.rs
b/crates/spark-connect/tests/e2e_dataframe.rs
index 78ed569..d242ffe 100644
--- a/crates/spark-connect/tests/e2e_dataframe.rs
+++ b/crates/spark-connect/tests/e2e_dataframe.rs
@@ -114,7 +114,7 @@ fn dataframe_metadata_json_semantics() {
assert!(df.same_semantics(&df4(&s)).unwrap());
let _ = df.semantic_hash().unwrap();
// flags
- assert!(!df.is_streaming());
+ assert!(!df.is_streaming().unwrap());
let _ = df.is_local();
// input_files (a sql df reads no files -> empty, but the call path runs)
let _ = df.input_files().unwrap();
diff --git a/python/tests/_mock_connect_server.py
b/python/tests/_mock_connect_server.py
index 847504b..2a7400a 100644
--- a/python/tests/_mock_connect_server.py
+++ b/python/tests/_mock_connect_server.py
@@ -132,6 +132,11 @@ def serve(connection):
def analyze(self, request, context):
self.calls += 1
self.analyze_requests.append(request)
+ # base.proto: AnalyzePlanRequest.is_streaming=8;
AnalyzePlanResponse.is_streaming=6,
+ # IsStreaming.is_streaming=1, of `streaming`, which a test sets.
+ if 8 in parse_message(request):
+ streaming = getattr(self, "streaming", False)
+ return message_field(6, varint_field(1, 1) if streaming else
b"")
if self.block_response and not self.response_gate.wait(5):
context.abort(
grpc.StatusCode.DEADLINE_EXCEEDED,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 6905009..3d2066a 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1375,6 +1375,20 @@ def
test_create_dataframe_merges_the_types_of_every_row_as_pyspark(session):
assert [tuple(r) for r in df.collect()] == [(text,), ("5",)]
+def test_dataframes_of_a_stream_are_streaming_as_pyspark(session):
+ # As PySpark's, which asks the server: a DataFrame with a streaming source
is streaming.
+ stream = session.readStream.format("rate").load()
+ for df in (
+ stream,
+ stream.select("value"),
+ stream.where("value > 1"),
+ stream.union(stream),
+ session.range(1).join(stream.select(F.col("value").alias("id")), "id"),
+ ):
+ assert df.isStreaming is True
+ assert session.range(1).isStreaming is False
+
+
def test_create_dataframe_raises_merge_errors_as_pyspark(session):
from pyspark.errors import PySparkException
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 8f00a42..47a4ac4 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3736,6 +3736,20 @@ def test_struct_field_equality():
assert f1 != f3
+def test_is_streaming_asks_the_server_as_pyspark(mock_connect):
+ # As PySpark's: isStreaming is the server's answer to an IsStreaming
request of the plan.
+ spark, service = mock_connect
+ df = spark.range(1).select("id")
+ try:
+ assert df.isStreaming is False
+ service.streaming = True
+ assert df.isStreaming is True
+ finally:
+ service.streaming = False
+ # base.proto: AnalyzePlanRequest.is_streaming=8.
+ assert sum(8 in parse_message(request) for request in
service.analyze_requests) == 2
+
+
def test_struct_type_field_access():
"""Test accessing StructType fields by index and name."""
st = T.StructType([
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]