This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 35662ff  [SPARK-60963][CONNECT] Retry the cache and artifact uploads 
as PySpark does
35662ff is described below

commit 35662ff8b83e4eb17b85932a235f6ead3e1a3a0e
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 18:51:07 2026 +0900

    [SPARK-60963][CONNECT] Retry the cache and artifact uploads as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60963](https://issues.apache.org/jira/browse/SPARK-60963): the 
artifact uploads of the Rust transport now run under the retry policy, as 
PySpark's do.
    
    - **Cache uploads** of large local relations: the `ArtifactStatus` check 
and the `AddArtifacts` upload are one attempt, as in 
`SparkConnectClient.cache_artifacts`. So a transient error of either is retried 
from the status check.
    - **`addArtifacts` and `copyFromLocalToFs`** upload each path in an attempt 
of its own, as `SparkConnectClient.add_artifacts` does. That makes one 
`AddArtifacts` call per path, as PySpark makes.
    
    An attempt sends all the frames again, since a stream cannot be resumed. 
The checks of the response, its CRCs and the acknowledgement of each cached 
artifact, stay outside the retries.
    
    ### Why are the changes needed?
    
    Each of these requests was sent once, so a transient error, such as an 
`UNAVAILABLE` of a reset connection, failed `createDataFrame` of a large 
PyArrow table, or `addArtifacts`, at once. PySpark retries it.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, these calls survive transient errors, as with PySpark 4.3.0. 
`addArtifacts` of several paths makes one upload per path.
    
    ### How was this patch tested?
    
    - New tests against the mock Connect server of the Arrow handoff tests, 
which can now fail `ArtifactStatus` and `AddArtifacts` calls with given status 
codes:
      - A cache upload whose status check and upload each fail once with 
`UNAVAILABLE` succeeds, with three status checks and two upload attempts, and 
the server reads the cached data.
      - A `PERMISSION_DENIED` of the upload is not retried.
      - `addArtifacts` of two paths, with one transient failure, uploads both 
in three attempts.
    
      The retry tests fail without this change.
    - The test of the frames of `addArtifacts` now expects one upload per path.
    - The lib tests and every integration test, e2e included, against a local 
Spark 4.3.0 server (660 passed, and 123 of the transport crate), the build of 
all tests with `wasm-udf`, the offline Python suites (913 passed, 9 skipped), 
and the live drop-in and Arrow suites (82 passed) and live cases (10 passed). 
Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #202 from HyukjinKwon/SPARK-60963-cache-retries.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/spark-connect-core/src/artifact.rs |  28 +++++
 crates/spark-connect-core/src/client.rs   | 175 ++++++++++++++++--------------
 python/tests/test_arrow_handoff.py        |  58 +++++++++-
 3 files changed, 176 insertions(+), 85 deletions(-)

diff --git a/crates/spark-connect-core/src/artifact.rs 
b/crates/spark-connect-core/src/artifact.rs
index d254571..692d5a5 100644
--- a/crates/spark-connect-core/src/artifact.rs
+++ b/crates/spark-connect-core/src/artifact.rs
@@ -94,6 +94,34 @@ impl NamedArtifact {
     }
 }
 
+/// Checks the response of an artifact upload: every artifact passed its CRC 
check and, for
+/// a cache upload, every artifact of `expected_cache_names` was acknowledged.
+pub(crate) fn check_artifact_upload(
+    response: spark_connect_proto::AddArtifactsResponse,
+    expected_cache_names: Option<&std::collections::HashSet<String>>,
+) -> Result<()> {
+    let mut received = std::collections::HashSet::new();
+    for summary in response.artifacts {
+        if !summary.is_crc_successful {
+            return Err(SparkError::connect_msg(format!(
+                "CRC check failed for artifact: {}",
+                summary.name
+            )));
+        }
+        if expected_cache_names.is_some() {
+            received.insert(summary.name);
+        }
+    }
+    if let Some(expected) = expected_cache_names {
+        if !expected.is_subset(&received) {
+            return Err(SparkError::connect_msg(
+                "Missing cache artifact upload acknowledgement",
+            ));
+        }
+    }
+    Ok(())
+}
+
 /// Frame cached in-memory artifacts as they are consumed by the upload stream.
 /// Payload slices share the original allocation instead of copying each chunk.
 /// The allocation is freed when its last slice is dropped, not chunk by chunk.
diff --git a/crates/spark-connect-core/src/client.rs 
b/crates/spark-connect-core/src/client.rs
index 18df94b..9835846 100644
--- a/crates/spark-connect-core/src/client.rs
+++ b/crates/spark-connect-core/src/client.rs
@@ -19,7 +19,8 @@ use spark_connect_proto::{
 };
 
 use crate::artifact::{
-    build_artifact_request_stream, cache_artifact_requests, FileArtifact, 
NamedArtifact,
+    build_artifact_request_stream, cache_artifact_requests, 
check_artifact_upload, FileArtifact,
+    NamedArtifact,
 };
 use crate::channel::{ChannelBuilder, GRPC_MAX_MESSAGE_LENGTH_DEFAULT};
 use crate::error::{Result, SparkError};
@@ -815,8 +816,9 @@ impl SparkConnectClient {
 
     /// Add artifacts to this session.
     ///
-    /// Mirrors 
`pyspark.sql.connect.client.core.SparkConnectClient.add_artifacts`.
-    /// Handles chunking large artifacts and batching small ones according to 
CHUNK_SIZE.
+    /// Mirrors 
`pyspark.sql.connect.client.core.SparkConnectClient.add_artifacts`, which
+    /// uploads each path in an attempt of its own under the retry policy. 
Handles chunking
+    /// large artifacts and batching small ones according to CHUNK_SIZE.
     pub async fn add_artifacts(
         &self,
         paths: &[&str],
@@ -826,21 +828,14 @@ impl SparkConnectClient {
     ) -> Result<()> {
         // Each path is named as PySpark's ArtifactManager names it, by the 
flags and its file
         // extension (see `parse_artifact`).
-        let mut artifacts = Vec::new();
         for path in paths {
             let (name, local) = crate::artifact::parse_artifact(path, pyfile, 
archive, file)
                 .map_err(|e| SparkError::connect_msg(e.to_string()))?;
-            artifacts.push(NamedArtifact::new(name, 
Box::new(FileArtifact::new(local))));
+            let artifacts = vec![NamedArtifact::new(name, 
Box::new(FileArtifact::new(local)))];
+            let requests = 
build_artifact_request_stream(self.session_id.clone(), artifacts)?;
+            self.upload_artifact_requests(requests, None).await?;
         }
-
-        if artifacts.is_empty() {
-            return Ok(());
-        }
-
-        // Build request stream with proper chunking/batching
-        let requests = build_artifact_request_stream(self.session_id.clone(), 
artifacts)?;
-        self.upload_artifact_requests(futures::stream::iter(requests), None)
-            .await
+        Ok(())
     }
 
     /// Upload a single local file as an artifact under an explicit artifact 
name
@@ -854,12 +849,14 @@ impl SparkConnectClient {
             Box::new(FileArtifact::new(local_path)),
         )];
         let requests = build_artifact_request_stream(self.session_id.clone(), 
artifacts)?;
-        self.upload_artifact_requests(futures::stream::iter(requests), None)
-            .await
+        self.upload_artifact_requests(requests, None).await
     }
 
     /// Upload session-scoped cache blobs, returning SHA-256 hashes in input 
order.
     /// Existing artifacts and duplicate blobs are uploaded only once.
+    ///
+    /// As `SparkConnectClient.cache_artifacts`, the status check and the 
upload are one
+    /// attempt under the retry policy, so a transient failure of either 
retries both.
     pub async fn cache_artifacts(&self, blobs: Vec<Vec<u8>>) -> 
Result<Vec<String>> {
         use bytes::Bytes;
         use std::collections::HashSet;
@@ -869,24 +866,37 @@ impl SparkConnectClient {
         }
         let (blobs, hashes) = hash_artifact_blobs(blobs).await?;
         let names: Vec<String> = hashes.iter().map(|hash| 
format!("cache/{hash}")).collect();
-        let unique: HashSet<&str> = names.iter().map(String::as_str).collect();
-        let cached = self
-            .artifact_status(&unique.into_iter().collect::<Vec<_>>())
+        let blobs: Vec<Bytes> = blobs.into_iter().map(Bytes::from).collect();
+        let unique: Vec<String> = names
+            .iter()
+            .collect::<HashSet<_>>()
+            .into_iter()
+            .cloned()
+            .collect();
+        let (blobs, names, unique) = (&blobs, &names, &unique);
+        let uploaded = self
+            .with_retry(|| async move {
+                let cached = self.artifact_status_rpc(unique).await?;
+                let mut seen = HashSet::new();
+                let mut artifacts = Vec::new();
+                for (blob, name) in blobs.iter().zip(names) {
+                    if !cached.get(name).copied().unwrap_or(false) && 
seen.insert(name.clone()) {
+                        artifacts.push((name.clone(), blob.clone()));
+                    }
+                }
+                if artifacts.is_empty() {
+                    return Ok(None);
+                }
+                let expected: HashSet<String> =
+                    artifacts.iter().map(|(name, _)| name.clone()).collect();
+                let requests = 
cache_artifact_requests(self.session_id.clone(), artifacts);
+                let response = self.add_artifacts_rpc(requests).await?;
+                Ok(Some((response, expected)))
+            })
             .await?;
-        let mut seen = HashSet::new();
-        let mut artifacts = Vec::new();
-        for (blob, name) in blobs.into_iter().zip(names) {
-            if !cached.get(&name).copied().unwrap_or(false) && 
seen.insert(name.clone()) {
-                artifacts.push((name, Bytes::from(blob)));
-            }
-        }
-        if artifacts.is_empty() {
-            return Ok(hashes);
+        if let Some((response, expected)) = uploaded {
+            check_artifact_upload(response, Some(&expected))?;
         }
-        let expected: HashSet<String> = artifacts.iter().map(|(name, _)| 
name.clone()).collect();
-        let requests = cache_artifact_requests(self.session_id.clone(), 
artifacts);
-        self.upload_artifact_requests(futures::stream::iter(requests), 
Some(&expected))
-            .await?;
         Ok(hashes)
     }
 
@@ -899,52 +909,45 @@ impl SparkConnectClient {
         }
     }
 
-    /// Send artifact frames and validate the response.
-    /// Stamps user_context and client_type on every frame; all upload paths 
must
-    /// go through here.
-    /// Cache uploads also require an acknowledgement for every requested 
artifact.
-    /// The stream is consumed once; retries must rebuild the frames and 
restart
-    /// AddArtifacts rather than resume a partially consumed stream.
+    /// Send artifact frames, retrying a transient failure of the whole upload 
under the retry
+    /// policy, and validate the response. An attempt sends all the frames 
again: a stream is
+    /// not resumed. Cache uploads also require an acknowledgement for every 
requested
+    /// artifact.
     async fn upload_artifact_requests(
         &self,
-        requests: impl futures::Stream<Item = 
spark_connect_proto::AddArtifactsRequest> + Send + 'static,
+        requests: Vec<spark_connect_proto::AddArtifactsRequest>,
         expected_cache_names: Option<&std::collections::HashSet<String>>,
     ) -> Result<()> {
-        use futures::StreamExt;
+        let requests = &requests;
+        let response = self
+            .with_retry(|| self.add_artifacts_rpc(requests.clone()))
+            .await?;
+        check_artifact_upload(response, expected_cache_names)
+    }
 
+    /// The AddArtifacts call of `requests`, once. Stamps user_context and 
client_type on
+    /// every frame; all upload paths must go through here.
+    async fn add_artifacts_rpc(
+        &self,
+        requests: impl IntoIterator<Item = 
spark_connect_proto::AddArtifactsRequest>,
+    ) -> std::result::Result<spark_connect_proto::AddArtifactsResponse, 
tonic::Status> {
         let user_context = self.artifact_user_context();
         let client_type = self.user_agent.clone();
-        let requests = requests.map(move |mut request| {
-            request.user_context = Some(user_context.clone());
-            request.client_type = Some(client_type.clone());
-            request
-        });
-        let mut req = Request::new(requests);
+        let requests: Vec<_> = requests
+            .into_iter()
+            .map(|mut request| {
+                request.user_context = Some(user_context.clone());
+                request.client_type = Some(client_type.clone());
+                request
+            })
+            .collect();
+        let mut req = Request::new(futures::stream::iter(requests));
         self._attach_metadata(&mut req);
-        let response = match self.stub.clone().add_artifacts(req).await {
-            Ok(response) => response.into_inner(),
-            Err(status) => return Err(self.error_from_status(status).await),
-        };
-        let mut received = std::collections::HashSet::new();
-        for summary in response.artifacts {
-            if !summary.is_crc_successful {
-                return Err(SparkError::connect_msg(format!(
-                    "CRC check failed for artifact: {}",
-                    summary.name
-                )));
-            }
-            if expected_cache_names.is_some() {
-                received.insert(summary.name);
-            }
-        }
-        if let Some(expected) = expected_cache_names {
-            if !expected.is_subset(&received) {
-                return Err(SparkError::connect_msg(
-                    "Missing cache artifact upload acknowledgement",
-                ));
-            }
-        }
-        Ok(())
+        self.stub
+            .clone()
+            .add_artifacts(req)
+            .await
+            .map(Response::into_inner)
     }
 
     /// Check artifact status on the server.
@@ -954,26 +957,32 @@ impl SparkConnectClient {
         &self,
         names: &[&str],
     ) -> Result<std::collections::HashMap<String, bool>> {
+        let names: Vec<String> = names.iter().map(|n| n.to_string()).collect();
+        match self.artifact_status_rpc(&names).await {
+            Ok(statuses) => Ok(statuses),
+            Err(status) => Err(self.error_from_status(status).await),
+        }
+    }
+
+    /// The ArtifactStatus call of `names`, once: whether each artifact exists.
+    async fn artifact_status_rpc(
+        &self,
+        names: &[String],
+    ) -> std::result::Result<std::collections::HashMap<String, bool>, 
tonic::Status> {
         let mut request = ArtifactStatusesRequest::default();
         request.session_id = self.session_id.clone();
         request.user_context = Some(self.artifact_user_context());
         request.client_type = Some(self.user_agent.clone());
-        request.names = names.iter().map(|n| n.to_string()).collect();
+        request.names = names.to_vec();
 
         let mut req = Request::new(request);
         self._attach_metadata(&mut req);
-        let response = match self.stub.clone().artifact_status(req).await {
-            Ok(response) => response.into_inner(),
-            Err(status) => return Err(self.error_from_status(status).await),
-        };
-
-        // Build map of artifact name -> exists status
-        let mut result = std::collections::HashMap::new();
-        for (name, status) in response.statuses {
-            result.insert(name, status.exists);
-        }
-
-        Ok(result)
+        let response = 
self.stub.clone().artifact_status(req).await?.into_inner();
+        Ok(response
+            .statuses
+            .into_iter()
+            .map(|(name, status)| (name, status.exists))
+            .collect())
     }
 
     /// Attach metadata headers to a request.
diff --git a/python/tests/test_arrow_handoff.py 
b/python/tests/test_arrow_handoff.py
index 68af777..c85e0b7 100644
--- a/python/tests/test_arrow_handoff.py
+++ b/python/tests/test_arrow_handoff.py
@@ -457,9 +457,12 @@ def _arrow_input_service(
         "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(1024**3),
     }
     config.update(overrides)
+    # The tests may queue status codes in `status_failures` and 
`upload_failures`: each
+    # ArtifactStatus or AddArtifacts call fails with the next one, while any 
is left.
     state = {"cache": {}, "uploads": 0, "relations": [], "message_sizes": [],
              "config": config, "config_requests": [], "input_tables": [],
-             "upload_contexts": [], "status_contexts": []}
+             "upload_contexts": [], "status_contexts": [],
+             "status_failures": [], "upload_failures": [], "upload_attempts": 
0}
     server_id = "00000000-0000-0000-0000-000000000001"
 
     def attribution(request, context):
@@ -494,11 +497,16 @@ def _arrow_input_service(
     def statuses(raw, context):
         request = protocol("ArtifactStatusesRequest").FromString(raw)
         state["status_contexts"].append(attribution(request, context))
+        if state["status_failures"]:
+            context.abort(state["status_failures"].pop(0), "artifact status 
unavailable")
         return protocol("ArtifactStatusesResponse", 
session_id=request.session_id,
                         statuses={name: dict(exists=name in state["cache"])
                                   for name in 
request.names}).SerializeToString()
 
     def upload(requests, context):
+        state["upload_attempts"] += 1
+        if state["upload_failures"]:
+            context.abort(state["upload_failures"].pop(0), "artifact upload 
unavailable")
         summaries = []
         name, expected, buffer = None, 0, bytearray()
         session_id = None
@@ -708,7 +716,8 @@ def test_file_upload_attribution_on_every_frame(
         )
         assert state["cache"][artifact_names[0]] == small.read_bytes()
         assert state["cache"][artifact_names[1]] == large.read_bytes()
-        assert state["uploads"] == (1 if upload == "file" else 2)
+        # One upload per path, as PySpark's add_artifacts makes one.
+        assert state["uploads"] == 2
         assert not state["status_contexts"]
         _assert_artifact_contexts(
             state["upload_contexts"],
@@ -718,6 +727,51 @@ def test_file_upload_attribution_on_every_frame(
         )
 
 
+def 
test_cached_arrow_upload_retries_a_transient_failure_from_the_status_check(protocol):
+    # As SparkConnectClient.cache_artifacts, the status check and the upload 
are one attempt
+    # under the retry policy: a transient failure of either is retried from 
the status check.
+    table = pa.table({"id": range(10_000)})
+    with _arrow_input_service(protocol, **{
+        "spark.sql.session.localRelationCacheThreshold": "0",
+    }) as (spark, state):
+        state["status_failures"].append(grpc.StatusCode.UNAVAILABLE)
+        state["upload_failures"].append(grpc.StatusCode.UNAVAILABLE)
+        df = spark.createDataFrame(table)
+        assert not state["status_failures"] and not state["upload_failures"]
+        assert state["upload_attempts"] == state["uploads"] + 1
+        # The failed status check, the one before the failed upload, and the 
one before the
+        # upload; the server then reads the cached data: its rows, first and 
last values.
+        assert len(state["status_contexts"]) == 3
+        df.toArrow()
+        assert state["result"] == (10_000, 0, 9_999)
+
+
+def test_cached_arrow_upload_does_not_retry_another_failure(protocol):
+    from pyspark.errors.exceptions.connect import SparkConnectGrpcException
+
+    table = pa.table({"id": range(10_000)})
+    with _arrow_input_service(protocol, **{
+        "spark.sql.session.localRelationCacheThreshold": "0",
+    }) as (spark, state):
+        state["upload_failures"].append(grpc.StatusCode.PERMISSION_DENIED)
+        with pytest.raises(SparkConnectGrpcException, match="artifact upload 
unavailable"):
+            spark.createDataFrame(table)
+        assert state["upload_attempts"] == 1 and state["uploads"] == 0
+
+
+def test_artifact_upload_retries_a_transient_failure_of_each_path(protocol, 
tmp_path):
+    # As SparkConnectClient.add_artifacts, each path is uploaded in an attempt 
of its own.
+    first, second = tmp_path / "first.txt", tmp_path / "second.txt"
+    first.write_bytes(b"first artifact")
+    second.write_bytes(b"second artifact")
+    with _arrow_input_service(protocol, allow_file_artifacts=True) as (spark, 
state):
+        state["upload_failures"].append(grpc.StatusCode.UNAVAILABLE)
+        spark.addArtifacts(str(first), str(second), file=True)
+        assert state["cache"]["files/first.txt"] == b"first artifact"
+        assert state["cache"]["files/second.txt"] == b"second artifact"
+        assert (state["upload_attempts"], state["uploads"]) == (3, 2)
+
+
 @pytest.mark.parametrize("cached", [False, True], ids=["inline", "cached"])
 @pytest.mark.parametrize("explicit", [False, True], ids=["inferred", 
"explicit"])
 def test_arrow_input_schema_handoff(protocol, cached, explicit):


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to