This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new 55a31f1  [SPARK-58541] Support retry with exponential backoff in 
`SparkConnectClient`
55a31f1 is described below

commit 55a31f10877ea59f9812b69bd2f8420ea23e47ad
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Mon Aug 3 23:45:25 2026 -0700

    [SPARK-58541] Support retry with exponential backoff in `SparkConnectClient`
    
    ### What changes were proposed in this pull request?
    
    This PR supports automatic retries with exponential backoff for RPC calls 
in `SparkConnectClient`.
    
    1. A new `RetryPolicy` implements the same default policy as the Scala side
       `org.apache.spark.sql.connect.client.RetryPolicy` and the Python side
       `pyspark.sql.connect.client.retries.DefaultPolicy`: `maxRetries=15`, 
`initialBackoff=50ms`,
       `maxBackoff=60s`, `backoffMultiplier=4.0`, `jitter=500ms`, 
`minJitterThreshold=2s`, and
       `maxServerRetryDelay=10min`, guaranteeing a maximum tolerated wait of at 
least 10 minutes.
    2. Retriable errors are `UNAVAILABLE`, `INTERNAL` caused by 
`INVALID_CURSOR.DISCONNECTED`, and
       any error containing `google.rpc.RetryInfo` in its gRPC status details. 
A server-provided
       `RetryInfo.retry_delay` overrides the client's backoff, limited by 
`maxServerRetryDelay`.
    3. A `withRetry` helper wraps both `withGPRC` implementations, so every 
non-streaming RPC
       (`AnalyzePlan`, `Config`, `Interrupt`, `GetStatus`, artifact RPCs, etc.) 
is retried with a
       fresh channel per attempt. `Task` cancellation stops the retry loop 
immediately, and the last
       error is rethrown when the retries are exhausted.
    4. Since `ExecutePlan` can have side effects on the server, it is retried 
only until the first
       response arrives (the operation is not started before that). A complete 
solution is
       `ReattachExecute`, which is a follow-up.
    
    ### Why are the changes needed?
    
    Every RPC call failed immediately on transient network errors, server 
restarts, or `UNAVAILABLE`
    responses. The PySpark and Scala Spark Connect clients retry these errors 
by default, and this PR
    brings the Swift client to parity with their `DefaultPolicy`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, transient RPC failures are now retried automatically for up to 15 
attempts instead of
    failing immediately. Non-retriable errors are still thrown as before.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added `RetryPolicyTests` test suite covering 
the retriable-error
    classification, the exponential backoff sequence and its cap, the 
server-provided retry delay and
    its limit, the retry loop attempt counts, and the cancellation propagation. 
All existing test
    suites pass with a live Spark Connect server.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #475 from dongjoon-hyun/SPARK-58541.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 Sources/SparkConnect/DataFrame+Actions.swift   | 144 ++++++++++---------
 Sources/SparkConnect/DataFrame.swift           |  25 ++--
 Sources/SparkConnect/RetryPolicy.swift         | 148 +++++++++++++++++++
 Sources/SparkConnect/SparkConnectClient.swift  |  50 ++++---
 Tests/SparkConnectTests/RetryPolicyTests.swift | 190 +++++++++++++++++++++++++
 5 files changed, 460 insertions(+), 97 deletions(-)

diff --git a/Sources/SparkConnect/DataFrame+Actions.swift 
b/Sources/SparkConnect/DataFrame+Actions.swift
index b57d1ba..845685e 100644
--- a/Sources/SparkConnect/DataFrame+Actions.swift
+++ b/Sources/SparkConnect/DataFrame+Actions.swift
@@ -36,68 +36,74 @@ extension DataFrame {
 
   /// Execute the plan and try to fill `schema` and `batches`.
   private func execute() async throws {
-    // Clear all existing batches.
-    self.batches.removeAll()
-
-    try await withGPRC { client in
-      let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
-      try await service.executePlan(spark.client.getExecutePlanRequest(plan)) {
-        response in
-        for try await m in response.messages {
-          if m.hasSchema {
-            // The original schema should arrive before ArrowBatches
-            await self.setSchema(m.schema)
-          }
-          let ipcStreamBytes = m.arrowBatch.data
-          if !ipcStreamBytes.isEmpty && m.arrowBatch.rowCount > 0 {
-            let IPC_CONTINUATION_TOKEN = Int32(-1)
-            let totalSize = Int64(ipcStreamBytes.count)
-            // Schema
-            guard totalSize >= 8,
-              ipcStreamBytes[0..<4].int32 == IPC_CONTINUATION_TOKEN
-            else {
-              throw SparkConnectError.InvalidArrowData
-            }
-            let schemaSize = Int64(ipcStreamBytes[4..<8].int32)
-            guard schemaSize >= 0, 8 + schemaSize + 4 <= totalSize else {
-              throw SparkConnectError.InvalidArrowData
-            }
-            let schema = Data(ipcStreamBytes[8..<(8 + schemaSize)])
-
-            // Arrow IPC Data
-            guard ipcStreamBytes[(8 + schemaSize)..<(8 + schemaSize + 4)].int32
-              == IPC_CONTINUATION_TOKEN
-            else {
-              throw SparkConnectError.InvalidArrowData
-            }
-            var pos: Int64 = 8 + schemaSize + 4
-            guard pos + 4 <= totalSize else {
-              throw SparkConnectError.InvalidArrowData
-            }
-            let dataHeaderSize = Int64(ipcStreamBytes[pos..<(pos + 4)].int32)
-            pos += 4
-            guard dataHeaderSize >= 0, pos + dataHeaderSize + 8 <= totalSize 
else {
-              throw SparkConnectError.InvalidArrowData
-            }
-            let dataHeader = Data(ipcStreamBytes[pos..<(pos + dataHeaderSize)])
-            pos += dataHeaderSize
-            let dataBodySize = totalSize - pos - 8
-            let dataBody = Data(ipcStreamBytes[pos..<(pos + dataBodySize)])
-
-            // Read ArrowBatches
-            let reader = ArrowReader()
-            let arrowResult = ArrowReader.makeArrowReaderResult()
-            if case .failure(let error) = reader.fromMessage(
-              schema, dataBody: Data(), result: arrowResult)
-            {
-              throw error
+    // `ExecutePlan` can have side effects on the server. Retry only until the 
first
+    // response arrives because the operation is not started before that.
+    let received = Atomic(false)
+    try await withRetry(shouldRetry: { !received.load(ordering: .relaxed) && 
RetryPolicy.canRetry($0) }) {
+      // Clear all existing batches.
+      self.batches.removeAll()
+
+      try await withGPRC(retryable: false) { client in
+        let service = Spark_Connect_SparkConnectService.Client(wrapping: 
client)
+        try await 
service.executePlan(spark.client.getExecutePlanRequest(plan)) {
+          response in
+          for try await m in response.messages {
+            received.store(true, ordering: .relaxed)
+            if m.hasSchema {
+              // The original schema should arrive before ArrowBatches
+              await self.setSchema(m.schema)
             }
-            if case .failure(let error) = reader.fromMessage(
-              dataHeader, dataBody: dataBody, result: arrowResult)
-            {
-              throw error
+            let ipcStreamBytes = m.arrowBatch.data
+            if !ipcStreamBytes.isEmpty && m.arrowBatch.rowCount > 0 {
+              let IPC_CONTINUATION_TOKEN = Int32(-1)
+              let totalSize = Int64(ipcStreamBytes.count)
+              // Schema
+              guard totalSize >= 8,
+                ipcStreamBytes[0..<4].int32 == IPC_CONTINUATION_TOKEN
+              else {
+                throw SparkConnectError.InvalidArrowData
+              }
+              let schemaSize = Int64(ipcStreamBytes[4..<8].int32)
+              guard schemaSize >= 0, 8 + schemaSize + 4 <= totalSize else {
+                throw SparkConnectError.InvalidArrowData
+              }
+              let schema = Data(ipcStreamBytes[8..<(8 + schemaSize)])
+
+              // Arrow IPC Data
+              guard ipcStreamBytes[(8 + schemaSize)..<(8 + schemaSize + 
4)].int32
+                == IPC_CONTINUATION_TOKEN
+              else {
+                throw SparkConnectError.InvalidArrowData
+              }
+              var pos: Int64 = 8 + schemaSize + 4
+              guard pos + 4 <= totalSize else {
+                throw SparkConnectError.InvalidArrowData
+              }
+              let dataHeaderSize = Int64(ipcStreamBytes[pos..<(pos + 4)].int32)
+              pos += 4
+              guard dataHeaderSize >= 0, pos + dataHeaderSize + 8 <= totalSize 
else {
+                throw SparkConnectError.InvalidArrowData
+              }
+              let dataHeader = Data(ipcStreamBytes[pos..<(pos + 
dataHeaderSize)])
+              pos += dataHeaderSize
+              let dataBodySize = totalSize - pos - 8
+              let dataBody = Data(ipcStreamBytes[pos..<(pos + dataBodySize)])
+
+              // Read ArrowBatches
+              let reader = ArrowReader()
+              let arrowResult = ArrowReader.makeArrowReaderResult()
+              if case .failure(let error) = reader.fromMessage(
+                schema, dataBody: Data(), result: arrowResult)
+              {
+                throw error
+              }
+              if case .failure(let error) = reader.fromMessage(
+                dataHeader, dataBody: dataBody, result: arrowResult)
+              {
+                throw error
+              }
+              await self.addBatches(arrowResult.batches)
             }
-            await self.addBatches(arrowResult.batches)
           }
         }
       }
@@ -179,12 +185,18 @@ extension DataFrame {
   public func count() async throws -> Int64 {
     let counter = Atomic(Int64(0))
 
-    try await withGPRC { client in
-      let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
-      try await service.executePlan(spark.client.getExecutePlanRequest(plan)) {
-        response in
-        for try await m in response.messages {
-          counter.add(m.arrowBatch.rowCount, ordering: .relaxed)
+    // `ExecutePlan` can have side effects on the server. Retry only until the 
first
+    // response arrives because the operation is not started before that.
+    let received = Atomic(false)
+    try await withRetry(shouldRetry: { !received.load(ordering: .relaxed) && 
RetryPolicy.canRetry($0) }) {
+      try await withGPRC(retryable: false) { client in
+        let service = Spark_Connect_SparkConnectService.Client(wrapping: 
client)
+        try await 
service.executePlan(spark.client.getExecutePlanRequest(plan)) {
+          response in
+          for try await m in response.messages {
+            received.store(true, ordering: .relaxed)
+            counter.add(m.arrowBatch.rowCount, ordering: .relaxed)
+          }
         }
       }
     }
diff --git a/Sources/SparkConnect/DataFrame.swift 
b/Sources/SparkConnect/DataFrame.swift
index f174139..754f148 100644
--- a/Sources/SparkConnect/DataFrame.swift
+++ b/Sources/SparkConnect/DataFrame.swift
@@ -305,19 +305,22 @@ public actor DataFrame: Sendable {
   }
 
   func withGPRC<Result: Sendable>(
+    retryable: Bool = true,
     _ f: (GRPCClient<GRPCNIOTransportHTTP2.HTTP2ClientTransport.Posix>) async 
throws -> Result
   ) async throws -> Result {
-    try await withGRPCClient(
-      transport: .http2NIOPosix(
-        target: .dns(host: spark.client.host, port: spark.client.port),
-        transportSecurity: spark.client.transportSecurity
-      ),
-      interceptors: spark.client.getIntercepters()
-    ) { client in
-      do {
-        return try await f(client)
-      } catch let error as RPCError where error.code == .internalError {
-        throw GrpcErrorConverter.convert(error) ?? error
+    try await withRetry(shouldRetry: { retryable && RetryPolicy.canRetry($0) 
}) {
+      try await withGRPCClient(
+        transport: .http2NIOPosix(
+          target: .dns(host: spark.client.host, port: spark.client.port),
+          transportSecurity: spark.client.transportSecurity
+        ),
+        interceptors: spark.client.getIntercepters()
+      ) { client in
+        do {
+          return try await f(client)
+        } catch let error as RPCError where error.code == .internalError {
+          throw GrpcErrorConverter.convert(error) ?? error
+        }
       }
     }
   }
diff --git a/Sources/SparkConnect/RetryPolicy.swift 
b/Sources/SparkConnect/RetryPolicy.swift
new file mode 100644
index 0000000..afa29dd
--- /dev/null
+++ b/Sources/SparkConnect/RetryPolicy.swift
@@ -0,0 +1,148 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+import GRPCCore
+import GRPCProtobuf
+
+/// A retry policy with exponential backoff for RPC calls
+/// like `org.apache.spark.sql.connect.client.RetryPolicy`.
+struct RetryPolicy: Sendable {
+  let maxRetries: Int
+  let initialBackoff: Duration
+  let maxBackoff: Duration
+  let backoffMultiplier: Double
+  let jitter: Duration
+  let minJitterThreshold: Duration
+  let maxServerRetryDelay: Duration
+
+  /// The default policy whose constants are synchronized with the Scala side
+  /// `org.apache.spark.sql.connect.client.RetryPolicy` and the Python side
+  /// `pyspark.sql.connect.client.retries.DefaultPolicy`.
+  ///
+  /// Note: these constants are selected so that the maximum tolerated wait is 
guaranteed
+  /// to be at least 10 minutes.
+  static let defaultPolicy = RetryPolicy(
+    maxRetries: 15,
+    initialBackoff: .milliseconds(50),
+    maxBackoff: .seconds(60),
+    backoffMultiplier: 4.0,
+    jitter: .milliseconds(500),
+    minJitterThreshold: .seconds(2),
+    maxServerRetryDelay: .seconds(10 * 60))
+
+  /// Return true if the error can be retried under the default policy.
+  /// - Parameter error: The error thrown by an RPC call.
+  /// - Returns: True if the error is a ``RPCError`` with code `UNAVAILABLE`, 
an `INTERNAL`
+  /// error caused by another RPC preempting this RPC, or any error containing
+  /// `google.rpc.RetryInfo` in its status details.
+  static func canRetry(_ error: Error) -> Bool {
+    guard let error = error as? RPCError else {
+      return false
+    }
+    if error.code == .unavailable {
+      return true
+    }
+    // This error happens if another RPC preempts this RPC.
+    if error.code == .internalError && 
error.message.contains("INVALID_CURSOR.DISCONNECTED") {
+      return true
+    }
+    // All errors containing `RetryInfo` should be retried.
+    return serverRetryDelay(of: error) != nil
+  }
+
+  /// Extract `google.rpc.RetryInfo.retry_delay` from the gRPC status details.
+  /// - Parameter error: The error thrown by an RPC call.
+  /// - Returns: The server-provided retry delay, or nil if absent.
+  static func serverRetryDelay(of error: Error) -> Duration? {
+    guard let error = error as? RPCError,
+      let status = try? error.unpackGoogleRPCStatus()
+    else {
+      return nil
+    }
+    return status.details.compactMap { $0.retryInfo }.first?.delay
+  }
+}
+
+/// The stateful part of ``RetryPolicy`` tracking how many attempts have 
happened
+/// and how long to wait until the next one.
+struct RetryPolicyState {
+  private let policy: RetryPolicy
+  private var attempt: Int = 0
+  private var nextWait: Duration
+
+  init(_ policy: RetryPolicy) {
+    self.policy = policy
+    self.nextWait = policy.initialBackoff
+  }
+
+  /// Return the time to wait until the next attempt, or nil if the retries 
are exhausted.
+  /// - Parameter error: The error that caused this attempt, used to recognize 
the
+  /// server-provided `RetryInfo.retry_delay` which can override the client's 
`maxBackoff`.
+  mutating func nextAttempt(_ error: Error) -> Duration? {
+    guard attempt < policy.maxRetries else {
+      return nil
+    }
+    attempt += 1
+
+    var wait = nextWait
+    nextWait = min(multiply(nextWait, by: policy.backoffMultiplier), 
policy.maxBackoff)
+
+    if let retryDelay = RetryPolicy.serverRetryDelay(of: error) {
+      wait = max(wait, min(retryDelay, policy.maxServerRetryDelay))
+    }
+
+    if wait >= policy.minJitterThreshold {
+      wait += multiply(policy.jitter, by: Double.random(in: 0...1))
+    }
+    return wait
+  }
+
+  private func multiply(_ duration: Duration, by multiplier: Double) -> 
Duration {
+    let seconds =
+      Double(duration.components.seconds) + 
Double(duration.components.attoseconds) / 1e18
+    return .seconds(seconds * multiplier)
+  }
+}
+
+/// Run `body` and retry it with exponential backoff when `shouldRetry` 
accepts the thrown error.
+/// Task cancellation stops the retry loop immediately. When the retries are 
exhausted,
+/// the last error is rethrown.
+/// - Parameters:
+///   - policy: A ``RetryPolicy`` instance.
+///   - isolation: The actor isolation inherited from the caller.
+///   - shouldRetry: A function that determines whether an error can be 
retried.
+///   - body: A block to run.
+/// - Returns: The result of `body`.
+func withRetry<Result>(
+  _ policy: RetryPolicy = .defaultPolicy,
+  isolation: isolated (any Actor)? = #isolation,
+  shouldRetry: (Error) -> Bool = RetryPolicy.canRetry,
+  _ body: () async throws -> Result
+) async throws -> Result {
+  var state = RetryPolicyState(policy)
+  while true {
+    do {
+      return try await body()
+    } catch {
+      guard !Task.isCancelled, shouldRetry(error), let wait = 
state.nextAttempt(error) else {
+        throw error
+      }
+      try await Task.sleep(for: wait)
+    }
+  }
+}
diff --git a/Sources/SparkConnect/SparkConnectClient.swift 
b/Sources/SparkConnect/SparkConnectClient.swift
index 878180b..79411ef 100644
--- a/Sources/SparkConnect/SparkConnectClient.swift
+++ b/Sources/SparkConnect/SparkConnectClient.swift
@@ -24,6 +24,7 @@ import Foundation
 import GRPCCore
 import GRPCNIOTransportHTTP2
 import GRPCProtobuf
+import Synchronization
 
 /// Conceptually the remote spark session that communicates with the server
 public actor SparkConnectClient {
@@ -133,19 +134,22 @@ public actor SparkConnectClient {
   }
 
   func withGPRC<Result: Sendable>(
+    retryable: Bool = true,
     _ f: (GRPCClient<GRPCNIOTransportHTTP2.HTTP2ClientTransport.Posix>) async 
throws -> Result
   ) async throws -> Result {
-    try await withGRPCClient(
-      transport: .http2NIOPosix(
-        target: .dns(host: self.host, port: self.port),
-        transportSecurity: self.transportSecurity
-      ),
-      interceptors: self.intercepters
-    ) { client in
-      do {
-        return try await f(client)
-      } catch let error as RPCError where error.code == .internalError {
-        throw GrpcErrorConverter.convert(error) ?? error
+    try await withRetry(shouldRetry: { retryable && RetryPolicy.canRetry($0) 
}) {
+      try await withGRPCClient(
+        transport: .http2NIOPosix(
+          target: .dns(host: self.host, port: self.port),
+          transportSecurity: self.transportSecurity
+        ),
+        interceptors: self.intercepters
+      ) { client in
+        do {
+          return try await f(client)
+        } catch let error as RPCError where error.code == .internalError {
+          throw GrpcErrorConverter.convert(error) ?? error
+        }
       }
     }
   }
@@ -854,15 +858,21 @@ public actor SparkConnectClient {
 
   @discardableResult
   func execute(_ sessionID: String, _ command: Command) async throws -> 
[ExecutePlanResponse] {
-    self.result.removeAll()
-    try await withGPRC { client in
-      let service = SparkConnectService.Client(wrapping: client)
-      var plan = Plan()
-      plan.opType = .command(command)
-      try await service.executePlan(getExecutePlanRequest(plan)) {
-        response in
-        for try await m in response.messages {
-          await self.addResponse(m)
+    // `ExecutePlan` can have side effects on the server. Retry only until the 
first
+    // response arrives because the operation is not started before that.
+    let received = Atomic(false)
+    try await withRetry(shouldRetry: { !received.load(ordering: .relaxed) && 
RetryPolicy.canRetry($0) }) {
+      self.result.removeAll()
+      try await withGPRC(retryable: false) { client in
+        let service = SparkConnectService.Client(wrapping: client)
+        var plan = Plan()
+        plan.opType = .command(command)
+        try await service.executePlan(getExecutePlanRequest(plan)) {
+          response in
+          for try await m in response.messages {
+            received.store(true, ordering: .relaxed)
+            await self.addResponse(m)
+          }
         }
       }
     }
diff --git a/Tests/SparkConnectTests/RetryPolicyTests.swift 
b/Tests/SparkConnectTests/RetryPolicyTests.swift
new file mode 100644
index 0000000..d78fa9c
--- /dev/null
+++ b/Tests/SparkConnectTests/RetryPolicyTests.swift
@@ -0,0 +1,190 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+import GRPCCore
+import GRPCProtobuf
+import Testing
+
+@testable import SparkConnect
+
+/// A test suite for `RetryPolicy`
+@Suite(.serialized)
+struct RetryPolicyTests {
+  // Disambiguate from `GRPCCore.RetryPolicy`.
+  typealias RetryPolicy = SparkConnect.RetryPolicy
+
+  static let unavailable = RPCError(code: .unavailable, message: "Connection 
refused")
+  static let cursorDisconnected = RPCError(
+    code: .internalError, message: "INVALID_CURSOR.DISCONNECTED: The cursor 
has been disconnected.")
+
+  static func errorWithRetryInfo(
+    _ code: RPCError.Code, delay: Duration
+  ) -> RPCError {
+    let status = GoogleRPCStatus(code: code, message: "m", details: 
.retryInfo(delay: delay))
+    return RPCError(code: code, message: "m", metadata: 
status.rpcErrorMetadata)
+  }
+
+  static func testPolicy(
+    maxRetries: Int = 15,
+    initialBackoff: Duration = .milliseconds(50),
+    maxBackoff: Duration = .seconds(60),
+    backoffMultiplier: Double = 4.0,
+    jitter: Duration = .zero,
+    minJitterThreshold: Duration = .zero,
+    maxServerRetryDelay: Duration = .seconds(10 * 60)
+  ) -> RetryPolicy {
+    RetryPolicy(
+      maxRetries: maxRetries,
+      initialBackoff: initialBackoff,
+      maxBackoff: maxBackoff,
+      backoffMultiplier: backoffMultiplier,
+      jitter: jitter,
+      minJitterThreshold: minJitterThreshold,
+      maxServerRetryDelay: maxServerRetryDelay)
+  }
+
+  func expectApprox(_ wait: Duration?, _ expected: Duration) throws {
+    let wait = try #require(wait)
+    #expect(wait >= expected - .microseconds(1) && wait <= expected + 
.microseconds(1))
+  }
+
+  @Test
+  func defaultPolicy() {
+    let policy = RetryPolicy.defaultPolicy
+    #expect(policy.maxRetries == 15)
+    #expect(policy.initialBackoff == .milliseconds(50))
+    #expect(policy.maxBackoff == .seconds(60))
+    #expect(policy.backoffMultiplier == 4.0)
+    #expect(policy.jitter == .milliseconds(500))
+    #expect(policy.minJitterThreshold == .seconds(2))
+    #expect(policy.maxServerRetryDelay == .seconds(10 * 60))
+  }
+
+  @Test
+  func canRetry() {
+    #expect(RetryPolicy.canRetry(Self.unavailable))
+    #expect(RetryPolicy.canRetry(Self.cursorDisconnected))
+    #expect(
+      RetryPolicy.canRetry(Self.errorWithRetryInfo(.resourceExhausted, delay: 
.seconds(1))))
+    #expect(!RetryPolicy.canRetry(RPCError(code: .internalError, message: 
"Other error")))
+    #expect(!RetryPolicy.canRetry(RPCError(code: .deadlineExceeded, message: 
"Timeout")))
+    #expect(!RetryPolicy.canRetry(RPCError(code: .invalidArgument, message: 
"Invalid")))
+    #expect(!RetryPolicy.canRetry(SparkConnectError.InvalidArgument))
+    #expect(!RetryPolicy.canRetry(CancellationError()))
+  }
+
+  @Test
+  func serverRetryDelay() throws {
+    let delay = RetryPolicy.serverRetryDelay(
+      of: Self.errorWithRetryInfo(.unavailable, delay: .seconds(5)))
+    #expect(delay == .seconds(5))
+    #expect(RetryPolicy.serverRetryDelay(of: Self.unavailable) == nil)
+    #expect(RetryPolicy.serverRetryDelay(of: 
SparkConnectError.InvalidArgument) == nil)
+  }
+
+  @Test
+  func backoffSequence() throws {
+    var state = RetryPolicyState(Self.testPolicy(maxRetries: 8))
+    try expectApprox(state.nextAttempt(Self.unavailable), .milliseconds(50))
+    try expectApprox(state.nextAttempt(Self.unavailable), .milliseconds(200))
+    try expectApprox(state.nextAttempt(Self.unavailable), .milliseconds(800))
+    try expectApprox(state.nextAttempt(Self.unavailable), .milliseconds(3200))
+    try expectApprox(state.nextAttempt(Self.unavailable), .milliseconds(12800))
+    try expectApprox(state.nextAttempt(Self.unavailable), .milliseconds(51200))
+    try expectApprox(state.nextAttempt(Self.unavailable), .seconds(60))
+    try expectApprox(state.nextAttempt(Self.unavailable), .seconds(60))
+    #expect(state.nextAttempt(Self.unavailable) == nil)
+  }
+
+  @Test
+  func serverRetryDelayOverridesBackoff() throws {
+    var state = RetryPolicyState(Self.testPolicy())
+    let error = Self.errorWithRetryInfo(.unavailable, delay: .seconds(5))
+    try expectApprox(state.nextAttempt(error), .seconds(5))
+
+    // The server-provided delay is limited by `maxServerRetryDelay`.
+    var another = RetryPolicyState(Self.testPolicy())
+    let longDelay = Self.errorWithRetryInfo(.unavailable, delay: .seconds(20 * 
60))
+    try expectApprox(another.nextAttempt(longDelay), .seconds(10 * 60))
+  }
+
+  @Test
+  func withRetryExhaustsAndRethrowsLastError() async throws {
+    let policy = Self.testPolicy(maxRetries: 3, initialBackoff: 
Duration.milliseconds(1), maxBackoff: Duration.milliseconds(2))
+    var attempts = 0
+    let error = await #expect(throws: RPCError.self) {
+      try await withRetry(policy) {
+        attempts += 1
+        throw Self.unavailable
+      }
+    }
+    #expect(attempts == 4)
+    #expect(error?.code == .unavailable)
+  }
+
+  @Test
+  func withRetrySucceedsAfterFailures() async throws {
+    let policy = Self.testPolicy(maxRetries: 3, initialBackoff: 
Duration.milliseconds(1), maxBackoff: Duration.milliseconds(2))
+    var attempts = 0
+    let value = try await withRetry(policy) {
+      attempts += 1
+      if attempts < 3 {
+        throw Self.unavailable
+      }
+      return 42
+    }
+    #expect(value == 42)
+    #expect(attempts == 3)
+  }
+
+  @Test
+  func withRetryDoesNotRetryNonRetryableError() async throws {
+    let policy = Self.testPolicy(maxRetries: 3, initialBackoff: 
Duration.milliseconds(1), maxBackoff: Duration.milliseconds(2))
+    var attempts = 0
+    await #expect(throws: SparkConnectError.InvalidArgument) {
+      try await withRetry(policy) {
+        attempts += 1
+        throw SparkConnectError.InvalidArgument
+      }
+    }
+    #expect(attempts == 1)
+  }
+
+  @Test
+  func withRetryStopsOnCancellation() async throws {
+    let policy = Self.testPolicy(
+      maxRetries: 5, initialBackoff: Duration.seconds(60), backoffMultiplier: 
1.0)
+    let clock = ContinuousClock()
+    let start = clock.now
+    let task = Task {
+      try await withRetry(policy) {
+        throw Self.unavailable
+      }
+    }
+    task.cancel()
+    let result = await task.result
+    #expect(clock.now - start < .seconds(10))
+    switch result {
+    case .success:
+      Issue.record("Expected an error")
+    case .failure(let error):
+      #expect(error is CancellationError || (error as? RPCError)?.code == 
.unavailable)
+    }
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to