This is an automated email from the ASF dual-hosted git repository.

LuciferYang pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.2 by this push:
     new bc1d9580f0d1 [SPARK-57635][SQL] Make Declarative Pipelines 
run-termination reason deterministic
bc1d9580f0d1 is described below

commit bc1d9580f0d17e0b5fa0619bc2b61edf4f79e3e6
Author: YangJie <[email protected]>
AuthorDate: Mon Jul 27 23:22:32 2026 +0800

    [SPARK-57635][SQL] Make Declarative Pipelines run-termination reason 
deterministic
    
    ### What changes were proposed in this pull request?
    `TriggeredGraphExecution.getRunTerminationReason` decided which failed 
flow's reason to report by calling `collectFirst` over `failureTracker`, a 
`ConcurrentHashMap` whose iteration order is unspecified. When more than one 
flow exhausts its retries (a non-retryable `StopFlowExecution`), the flow whose 
reason gets surfaced therefore varied from run to run.
    
    This extracts a small pure helper, `chooseRunTerminationReason`, that 
considers only the stopped flows and picks the earliest one by 
`(lastFailTimestamp, flowName)`, so the reported reason is stable across 
otherwise-identical runs. `getRunTerminationReason` now calls it and falls back 
to `UnexpectedRunFailure()`. The previous code also computed 
`graphForExecution.flow(...)` and `lastException` only to discard them; those 
are dropped.
    
    ### Why are the changes needed?
    Two runs that fail the same way could report different termination reasons 
(different flow name and cause), which is confusing in logs and events and 
makes the outcome non-reproducible.
    
    ### Does this PR introduce _any_ user-facing change?
    No. The reported reason was already one of the failing flows; it is now 
chosen deterministically.
    
    ### How was this patch tested?
    Added unit tests for `chooseRunTerminationReason` in 
`TriggeredGraphExecutionSuite`: the earliest failure wins regardless of 
iteration order, ties are broken by flow name, and retryable failures are 
ignored. They fail against the previous order-dependent selection and pass with 
this change.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    Generated-by: Claude Code (Claude Opus 4.8)
    
    Closes #56698 from LuciferYang/sdp-deterministic-termination-reason.
    
    Authored-by: YangJie <[email protected]>
    Signed-off-by: yangjie01 <[email protected]>
    (cherry picked from commit effd582afa3980a6b674d212a705c0d5212f2948)
    Signed-off-by: yangjie01 <[email protected]>
---
 .../pipelines/graph/TriggeredGraphExecution.scala  | 35 +++++++++-------
 .../graph/TriggeredGraphExecutionSuite.scala       | 49 ++++++++++++++++++++++
 2 files changed, 69 insertions(+), 15 deletions(-)

diff --git 
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecution.scala
 
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecution.scala
index 02871e64aa76..22b64798f598 100644
--- 
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecution.scala
+++ 
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecution.scala
@@ -418,21 +418,9 @@ class TriggeredGraphExecution(
       return RunCompletion()
     }
 
-    val executionFailureOpt = failureTracker.iterator
-      .map {
-        case (flowIdentifier, failureInfo) =>
-          (
-            graphForExecution.flow(flowIdentifier),
-            failureInfo.lastException,
-            failureInfo.lastExceptionAction
-          )
-      }
-      .collectFirst {
-        case (_, _, GraphExecution.StopFlowExecution(reason)) =>
-          reason.runTerminationReason
-      }
-
-    executionFailureOpt.getOrElse(UnexpectedRunFailure())
+    TriggeredGraphExecution
+      .chooseRunTerminationReason(failureTracker.iterator)
+      .getOrElse(UnexpectedRunFailure())
   }
 }
 
@@ -450,6 +438,23 @@ case class TriggeredFailureInfo(
 
 object TriggeredGraphExecution {
 
+  /**
+   * Picks the run-termination reason from the flows whose execution was 
stopped because they
+   * exhausted their retries. Several flows can stop a run and `failures` 
comes from an unordered
+   * map, so the earliest failure is chosen - ties broken by flow name - to 
keep the reported
+   * reason stable across otherwise-identical runs.
+   */
+  private[graph] def chooseRunTerminationReason(
+      failures: Iterator[(TableIdentifier, TriggeredFailureInfo)]): 
Option[RunTerminationReason] = {
+    failures
+      .collect {
+        case (id, TriggeredFailureInfo(ts, _, _, 
GraphExecution.StopFlowExecution(r))) =>
+          (ts, id.unquotedString, r.runTerminationReason)
+      }
+      .minByOption { case (ts, flowName, _) => (ts, flowName) }
+      .map { case (_, _, reason) => reason }
+  }
+
   // All possible states of a data stream for a flow
   sealed trait StreamState
   object StreamState {
diff --git 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecutionSuite.scala
 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecutionSuite.scala
index 57baf4c2d5b1..db8c368ca89d 100644
--- 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecutionSuite.scala
+++ 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/TriggeredGraphExecutionSuite.scala
@@ -1060,4 +1060,53 @@ class TriggeredGraphExecutionSuite extends ExecutionTest 
with SharedSparkSession
 
     assert(warnCount == 2 && errorCount == 1)
   }
+
+  /** A non-retryable (retries exhausted) failure, i.e. one that stops the 
run. */
+  private def stopFailure(ts: Long, flowName: String, cause: Throwable): 
TriggeredFailureInfo = {
+    // currentNumTries > maxAllowedRetries yields a StopFlowExecution.
+    val action = GraphExecution.determineFlowExecutionActionFromError(
+      ex = cause,
+      flowDisplayName = flowName,
+      currentNumTries = 2,
+      maxAllowedRetries = 1)
+    TriggeredFailureInfo(ts, numFailures = 2, lastException = cause, 
lastExceptionAction = action)
+  }
+
+  test("chooseRunTerminationReason surfaces the earliest non-retryable failure 
deterministically") {
+    val earliestCause = new RuntimeException("earliest")
+    // A retryable failure with an earlier timestamp must be ignored: it does 
not stop the run.
+    val retryable = {
+      val cause = new RuntimeException("retryable")
+      val action = GraphExecution.determineFlowExecutionActionFromError(
+        ex = cause, flowDisplayName = "flow_retry", currentNumTries = 1, 
maxAllowedRetries = 3)
+      TableIdentifier("flow_retry") -> TriggeredFailureInfo(
+        50, numFailures = 1, lastException = cause, lastExceptionAction = 
action)
+    }
+    val entries = Seq(
+      TableIdentifier("flow_c") -> stopFailure(300, "flow_c", new 
RuntimeException("c")),
+      TableIdentifier("flow_b") -> stopFailure(100, "flow_b", earliestCause),
+      TableIdentifier("flow_a") -> stopFailure(200, "flow_a", new 
RuntimeException("a")),
+      retryable)
+
+    // The earliest non-retryable failure (flow_b @ 100) wins regardless of 
iteration order.
+    Seq(entries, entries.reverse).foreach { ordered =>
+      assert(
+        TriggeredGraphExecution.chooseRunTerminationReason(ordered.iterator)
+          .contains(QueryExecutionFailure("flow_b", 1, Some(earliestCause))))
+    }
+  }
+
+  test("chooseRunTerminationReason breaks ties between equal timestamps by 
flow name") {
+    val aCause = new RuntimeException("a")
+    val entries = Seq(
+      TableIdentifier("flow_z") -> stopFailure(100, "flow_z", new 
RuntimeException("z")),
+      TableIdentifier("flow_a") -> stopFailure(100, "flow_a", aCause))
+    assert(
+      TriggeredGraphExecution.chooseRunTerminationReason(entries.iterator)
+        .contains(QueryExecutionFailure("flow_a", 1, Some(aCause))))
+  }
+
+  test("chooseRunTerminationReason has no reason when no flow stopped the 
run") {
+    
assert(TriggeredGraphExecution.chooseRunTerminationReason(Iterator.empty).isEmpty)
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to