This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 88ac7b1  [SPARK-61038][CONNECT][PYTHON] Take the triggers of 
DataStreamWriter.trigger, including realTime, in the Python client as PySpark 
does
88ac7b1 is described below

commit 88ac7b130947c975cb155e8d722310b741f10066
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 06:18:38 2026 +0900

    [SPARK-61038][CONNECT][PYTHON] Take the triggers of 
DataStreamWriter.trigger, including realTime, in the Python client as PySpark 
does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61038](https://issues.apache.org/jira/browse/SPARK-61038): 
`DataStreamWriter.trigger` now takes its triggers, including `realTime`, as 
PySpark Connect does.
    
    `df.writeStream.trigger(...)`:
    - takes exactly one of `processingTime`, `once`, `continuous`, 
`availableNow` and `realTime`, and raises `ONLY_ALLOW_SINGLE_TRIGGER` for none 
or more than one
    - takes an interval or duration as a non-empty string, which it strips, 
else `VALUE_NOT_NON_EMPTY_STR`
    - takes `once` and `availableNow` as `True`, else `VALUE_NOT_ALLOWED`
    - sets the `real_time_batch_duration` of the `WriteStreamOperationStart` 
for `trigger(realTime="5 seconds")`, the real-time mode of the query
    
    The core `Trigger` has a `RealTime` variant, and `Trigger::to_proto` 
serializes each trigger.
    
    ### Why are the changes needed?
    
    The client:
    - raised `NotImplementedError: the realTime trigger is not yet supported by 
this client`
    - took the first trigger of several
    - raised plain `ValueError`s for no trigger and for `once=False`, and 
`TypeError`s for intervals that are not strings
    - sent empty and unstripped intervals
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `trigger` takes the triggers, and raises the errors, of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test starts a query of the real-time trigger, which the server 
rejects for the memory sink with 
`STREAMING_REAL_TIME_MODE.OPERATOR_OR_SINK_NOT_IN_ALLOWLIST`, as with official 
PySpark 4.3.0. It fails without this change.
    - New offline tests check the errors of `trigger`, and the triggers that 
`start` sends to a mock Connect server.
    - A new lib test checks the proto of each trigger.
    - On a local server, 16 cases of `trigger` behave as in official PySpark 
4.3.0.
    - The lib tests of both crates (569 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1807 passed, 8 skipped), and the 
live drop-in and Arrow suites (106 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #232 from HyukjinKwon/SPARK-61038-real-time-trigger.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/streaming.rs    | 93 ++++++++++++++++++++++++++---------
 crates/spark-connect/src/streaming.rs | 70 ++++++++++++++++----------
 python/tests/test_dropin_e2e.py       | 16 ++++++
 python/tests/test_dropin_offline.py   | 50 +++++++++++++++++++
 4 files changed, 179 insertions(+), 50 deletions(-)

diff --git a/crates/pyspark-rs/src/streaming.rs 
b/crates/pyspark-rs/src/streaming.rs
index 8f47836..c1f6130 100644
--- a/crates/pyspark-rs/src/streaming.rs
+++ b/crates/pyspark-rs/src/streaming.rs
@@ -640,35 +640,82 @@ impl PyDataStreamWriter {
         Ok(slf)
     }
 
-    /// `DataStreamWriter.trigger(...)`: mirrors the reference keyword API — 
exactly one
-    /// of `processingTime` / `once` / `availableNow` / `continuous` is given.
+    /// `DataStreamWriter.trigger(...)`, as PySpark's: exactly one of 
`processingTime`, `once`,
+    /// `continuous`, `availableNow` and `realTime`, else 
`ONLY_ALLOW_SINGLE_TRIGGER`; an
+    /// interval or duration is a non-empty string, which is stripped, else
+    /// `VALUE_NOT_NON_EMPTY_STR`, and `once` and `availableNow` are True, else
+    /// `VALUE_NOT_ALLOWED`.
     #[pyo3(signature = (*, processingTime=None, once=None, continuous=None, 
availableNow=None, realTime=None))]
     fn trigger<'py>(
         mut slf: PyRefMut<'py, Self>,
-        processingTime: Option<&str>,
-        once: Option<bool>,
-        continuous: Option<&str>,
-        availableNow: Option<bool>,
-        realTime: Option<&str>,
+        processingTime: Option<Bound<'py, PyAny>>,
+        once: Option<Bound<'py, PyAny>>,
+        continuous: Option<Bound<'py, PyAny>>,
+        availableNow: Option<Bound<'py, PyAny>>,
+        realTime: Option<Bound<'py, PyAny>>,
     ) -> PyResult<PyRefMut<'py, Self>> {
-        let trigger = if let Some(interval) = processingTime {
-            Trigger::ProcessingTime(interval.to_string())
-        } else if once == Some(true) {
-            Trigger::Once
-        } else if let Some(interval) = continuous {
-            Trigger::Continuous(interval.to_string())
-        } else if availableNow == Some(true) {
-            Trigger::AvailableNow
-        } else if realTime.is_some() {
-            // `realTime` is accepted for signature parity with reference 
pyspark; the
-            // connect core's Trigger enum does not yet model a real-time 
trigger.
-            return Err(PyErr::new::<pyo3::exceptions::PyNotImplementedError, 
_>(
-                "the realTime trigger is not yet supported by this client",
+        use crate::column::raise_pyspark;
+        let py = slf.py();
+        let given = [
+            &processingTime,
+            &once,
+            &continuous,
+            &availableNow,
+            &realTime,
+        ]
+        .iter()
+        .filter(|value| value.is_some())
+        .count();
+        if given != 1 {
+            return Err(raise_pyspark(
+                py,
+                "PySparkValueError",
+                "ONLY_ALLOW_SINGLE_TRIGGER",
+                &[],
             ));
+        }
+        let interval = |value: &Bound<'py, PyAny>, name: &str| -> 
PyResult<String> {
+            let text = match value.cast::<pyo3::types::PyString>() {
+                Ok(text) => text.to_cow()?.trim().to_string(),
+                Err(_) => String::new(),
+            };
+            if text.is_empty() {
+                return Err(raise_pyspark(
+                    py,
+                    "PySparkValueError",
+                    "VALUE_NOT_NON_EMPTY_STR",
+                    &[("arg_name", name), ("arg_value", 
&value.str()?.to_string())],
+                ));
+            }
+            Ok(text)
+        };
+        // As PySpark's `once is not True`.
+        let true_flag = |value: &Bound<'py, PyAny>, name: &str| -> 
PyResult<()> {
+            if value.is(pyo3::types::PyBool::new(py, true)) {
+                Ok(())
+            } else {
+                Err(raise_pyspark(
+                    py,
+                    "PySparkValueError",
+                    "VALUE_NOT_ALLOWED",
+                    &[("arg_name", name), ("allowed_values", "[True]")],
+                ))
+            }
+        };
+        let trigger = if let Some(value) = &processingTime {
+            Trigger::ProcessingTime(interval(value, "processingTime")?)
+        } else if let Some(value) = &once {
+            true_flag(value, "once")?;
+            Trigger::Once
+        } else if let Some(value) = &continuous {
+            Trigger::Continuous(interval(value, "continuous")?)
+        } else if let Some(value) = &realTime {
+            Trigger::RealTime(interval(value, "realTime")?)
         } else {
-            return Err(PyErr::new::<pyo3::exceptions::PyValueError, _>(
-                "trigger() requires exactly one of processingTime, once, 
continuous, availableNow, realTime",
-            ));
+            if let Some(value) = &availableNow {
+                true_flag(value, "availableNow")?;
+            }
+            Trigger::AvailableNow
         };
         slf.inner = slf.inner.clone().trigger(trigger);
         Ok(slf)
diff --git a/crates/spark-connect/src/streaming.rs 
b/crates/spark-connect/src/streaming.rs
index a5b4b79..e1648bb 100644
--- a/crates/spark-connect/src/streaming.rs
+++ b/crates/spark-connect/src/streaming.rs
@@ -253,6 +253,23 @@ pub enum Trigger {
     AvailableNow,
     /// Continuous processing with checkpoint interval.
     Continuous(String),
+    /// Real-time processing, of batches of the duration (e.g., "5 seconds"), 
as PySpark's
+    /// `trigger(realTime=...)`.
+    RealTime(String),
+}
+
+impl Trigger {
+    /// The trigger of a `WriteStreamOperationStart`.
+    pub(crate) fn to_proto(&self) -> 
proto::write_stream_operation_start::Trigger {
+        use proto::write_stream_operation_start::Trigger as Proto;
+        match self {
+            Trigger::ProcessingTime(interval) => 
Proto::ProcessingTimeInterval(interval.clone()),
+            Trigger::Once => Proto::Once(true),
+            Trigger::AvailableNow => Proto::AvailableNow(true),
+            Trigger::Continuous(interval) => 
Proto::ContinuousCheckpointInterval(interval.clone()),
+            Trigger::RealTime(duration) => 
Proto::RealTimeBatchDuration(duration.clone()),
+        }
+    }
 }
 
 /// DataStreamWriter for writing streaming data to various sinks.
@@ -396,33 +413,7 @@ impl DataStreamWriter {
             write_op.query_name = name.clone();
         }
 
-        if let Some(trigger) = &self.trigger {
-            match trigger {
-                Trigger::ProcessingTime(interval) => {
-                    write_op.trigger = Some(
-                        
proto::write_stream_operation_start::Trigger::ProcessingTimeInterval(
-                            interval.clone(),
-                        ),
-                    );
-                }
-                Trigger::Once => {
-                    write_op.trigger =
-                        
Some(proto::write_stream_operation_start::Trigger::Once(true));
-                }
-                Trigger::AvailableNow => {
-                    write_op.trigger = Some(
-                        
proto::write_stream_operation_start::Trigger::AvailableNow(true),
-                    );
-                }
-                Trigger::Continuous(interval) => {
-                    write_op.trigger = Some(
-                        
proto::write_stream_operation_start::Trigger::ContinuousCheckpointInterval(
-                            interval.clone(),
-                        ),
-                    );
-                }
-            }
-        }
+        write_op.trigger = self.trigger.as_ref().map(Trigger::to_proto);
 
         if let Some(path) = &self.path {
             // A memory/console/foreach sink has no path; only set the 
destination for a
@@ -1296,6 +1287,31 @@ mod tests {
         assert!(writer.trigger.is_some());
     }
 
+    #[test]
+    fn triggers_are_sent_as_pyspark_sends_them() {
+        use proto::write_stream_operation_start::Trigger as Proto;
+        let cases = [
+            (
+                Trigger::ProcessingTime("1 second".to_string()),
+                Proto::ProcessingTimeInterval("1 second".to_string()),
+            ),
+            (Trigger::Once, Proto::Once(true)),
+            (Trigger::AvailableNow, Proto::AvailableNow(true)),
+            (
+                Trigger::Continuous("1 minute".to_string()),
+                Proto::ContinuousCheckpointInterval("1 minute".to_string()),
+            ),
+            // The real_time_batch_duration of PySpark's 
`trigger(realTime=...)`.
+            (
+                Trigger::RealTime("5 seconds".to_string()),
+                Proto::RealTimeBatchDuration("5 seconds".to_string()),
+            ),
+        ];
+        for (trigger, sent) in cases {
+            assert_eq!(trigger.to_proto(), sent);
+        }
+    }
+
     #[test]
     fn stream_writer_option_options() {
         let spark = session();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index ccf7b56..71052cc 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1541,6 +1541,22 @@ def 
test_dataframes_of_a_stream_are_streaming_as_pyspark(session):
     assert session.range(1).isStreaming is False
 
 
+def 
test_stream_writer_starts_a_query_of_its_real_time_trigger_as_pyspark(session):
+    # As PySpark's, the real-time trigger reaches the server, which runs 
real-time mode for
+    # some sinks only, not the memory sink.
+    from pyspark.errors import IllegalArgumentException
+
+    writer = 
session.readStream.format("rate").load().writeStream.format("memory")
+    with pytest.raises(IllegalArgumentException) as error:
+        query = writer.queryName("real_time_trigger").trigger(realTime="5 
seconds").start()
+        try:
+            query.awaitTermination(10)
+        finally:
+            query.stop()
+    assert error.value.getCondition() == (
+        "STREAMING_REAL_TIME_MODE.OPERATOR_OR_SINK_NOT_IN_ALLOWLIST")
+
+
 def test_create_dataframe_raises_merge_errors_as_pyspark(session):
     from pyspark.errors import PySparkException
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 685dea5..593892b 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -8256,6 +8256,56 @@ def 
test_stream_writer_setters_mutate_and_return_self(offline_spark):
     assert w.foreach(lambda row: None) is w
 
 
[email protected]("kwargs, condition, parameters", [
+    ({}, "ONLY_ALLOW_SINGLE_TRIGGER", {}),
+    ({"processingTime": "1 second", "once": True}, 
"ONLY_ALLOW_SINGLE_TRIGGER", {}),
+    ({"processingTime": "  "}, "VALUE_NOT_NON_EMPTY_STR",
+     {"arg_name": "processingTime", "arg_value": "  "}),
+    ({"continuous": 5}, "VALUE_NOT_NON_EMPTY_STR", {"arg_name": "continuous", 
"arg_value": "5"}),
+    ({"realTime": ""}, "VALUE_NOT_NON_EMPTY_STR", {"arg_name": "realTime", 
"arg_value": ""}),
+    ({"once": False}, "VALUE_NOT_ALLOWED", {"arg_name": "once", 
"allowed_values": "[True]"}),
+    ({"availableNow": 1}, "VALUE_NOT_ALLOWED",
+     {"arg_name": "availableNow", "allowed_values": "[True]"}),
+], ids=["none", "two", "empty", "not-a-string", "empty-real-time", 
"once-false",
+        "available-now-1"])
+def test_stream_writer_triggers_are_checked_as_pyspark(offline_spark, kwargs, 
condition,
+                                                        parameters):
+    from pyspark.errors import PySparkValueError
+
+    w = offline_spark.readStream.format("rate").load().writeStream
+    with pytest.raises(PySparkValueError) as error:
+        w.trigger(**kwargs)
+    assert error.value.getCondition() == condition
+    assert error.value.getMessageParameters() == parameters
+    # A trigger of None is not given, as in PySpark.
+    assert w.trigger(processingTime=None, once=True) is w
+
+
+def test_stream_writer_sends_its_trigger_as_pyspark(mock_connect):
+    # As PySpark's: the trigger of a WriteStreamOperationStart, an interval or 
duration
+    # stripped, of the real-time trigger too.
+    spark, service = mock_connect
+    # ExecutePlanResponse.write_stream_operation_start_result=8: query_id=1 
(id=1, run_id=2).
+    service.execute_payload = message_field(
+        8, message_field(1, message_field(1, "q") + message_field(2, "r")))
+
+    def sent(**trigger):
+        
spark.readStream.format("rate").load().writeStream.format("memory").queryName(
+            "q").trigger(**trigger).start()
+        # ExecutePlanRequest.plan=3 -> Plan.command=2 -> 
Command.write_stream_operation_start=6.
+        plan = parse_message(parse_message(service.execute_requests[-1])[3][0])
+        start = parse_message(parse_message(plan[2][0])[6][0])
+        # WriteStreamOperationStart: processing_time_interval=5, 
available_now=6, once=7,
+        # continuous_checkpoint_interval=8, real_time_batch_duration=100.
+        return {field: start[field] for field in (5, 6, 7, 8, 100) if field in 
start}
+
+    assert sent(processingTime=" 1 second ") == {5: [b"1 second"]}
+    assert sent(once=True) == {7: [1]}
+    assert sent(availableNow=True) == {6: [1]}
+    assert sent(continuous="1 minute") == {8: [b"1 minute"]}
+    assert sent(realTime=" 5 seconds ") == {100: [b"5 seconds"]}
+
+
 def test_merge_into_writer_mutates_and_returns_self(offline_spark):
     # PySpark: withSchemaEvolution and every clause action change the writer 
and
     # return that same writer; a clause object can be used more than once.


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to