This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 80ae749  [SPARK-61070][CONNECT][PYTHON] Stream the chunks of artifact 
uploads, and refuse class files in addArtifacts, as PySpark does
80ae749 is described below

commit 80ae7494c42a5043feb089d1eba03b68468b5606
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 12:29:08 2026 +0900

    [SPARK-61070][CONNECT][PYTHON] Stream the chunks of artifact uploads, and 
refuse class files in addArtifacts, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    - Artifact uploads stream their chunks: `ArtifactFrames` 
(`crates/spark-connect-core/src/artifact.rs`) reads one 32 KiB chunk per frame 
as the frames are sent, with the same framing as before (a batch for small 
artifacts, `BeginChunk` and `Chunk` frames otherwise), and each attempt of the 
retry policy reads the file again from the start (`upload_artifact` in 
`client.rs`), instead of building every frame up front and cloning them for 
each attempt.
    - `addArtifacts` of the Python client raises PySpark's `PySparkRuntimeError 
[UNSUPPORTED_OPERATION] ... file format is not supported.` for a class file, 
opens local paths as PySpark does (so a missing path or a directory raises 
`FileNotFoundError` or `IsADirectoryError` with its text), and puts a pyfile 
zip, egg or jar on `sys.path` and invalidates the import caches, as PySpark 
does (`crates/pyspark-rs/src/session.rs`). The Rust API still uploads class 
files, as its docs say.
    
    ### Why are the changes needed?
    
    [SPARK-61070](https://issues.apache.org/jira/browse/SPARK-61070): measured 
as the peak RSS of `addArtifacts` of a 300 MB jar to an in-process server: 496 
MB on the current master, 141 to 174 MB with this change, 173 MB with PySpark 
4.3.0. And `spark.addArtifacts("A.class")` uploaded the file where PySpark 
raises.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Uploads take less memory, and the Python client's `addArtifacts` 
raises PySpark's errors for class files and for missing paths.
    
    ### How was this patch tested?
    
    - `crates/spark-connect-core/tests/artifact_uploads.rs`, against a loopback 
AddArtifacts service: chunked and batched frames, class file names, a retried 
upload sent again from the start, CRC mismatches and permission errors not 
retried, and a missing file failing before any attempt; and unit tests of the 
frames (lazy reads, framing, read failures, the same frames on each attempt).
    - New offline tests in `python/tests/test_dropin_offline.py` for the errors 
of `addArtifacts` and the `sys.path` order; 
`test_add_artifacts_names_each_artifact_as_pyspark` uploads its class file with 
`file=True`, as PySpark requires.
    - 112 combinations of paths (jar, py, zip, egg, tar.gz, fragments, `%23`, 
`file:` URIs, a directory, missing paths, remote schemes, an empty path) and 
flags give the same outcome, exception and message as PySpark 4.3.0's 
`_parse_artifacts`.
    - The core tests and the build of every integration test of 
`apache-spark-connect`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #261 from HyukjinKwon/SPARK-61070-artifact-uploads.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/session.rs                   |  87 ++--
 crates/spark-connect-core/src/artifact.rs          | 386 +++++++++++++++--
 crates/spark-connect-core/src/client.rs            |  75 ++--
 .../spark-connect-core/tests/artifact_uploads.rs   | 462 +++++++++++++++++++++
 crates/spark-connect/src/session.rs                |   3 +-
 python/tests/test_arrow_handoff.py                 |  17 +-
 python/tests/test_dropin_offline.py                |  75 ++++
 7 files changed, 997 insertions(+), 108 deletions(-)

diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 06ea7cd..b8bcb16 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -6,6 +6,7 @@ use spark_connect::row::{Row, StructValue, Value};
 use spark_connect::session::SparkSession;
 use spark_connect::types::{DataType, StructField};
 use spark_connect::udf::{CommonInlineUserDefinedFunctionExpression, 
PythonUDFPayload};
+use spark_connect_core::artifact::{parse_artifact, UnsupportedArtifact};
 
 use crate::catalog::PyCatalog;
 use crate::dataframe::PyDataFrame;
@@ -1636,7 +1637,8 @@ impl PySparkSession {
     /// Add local files, or `file:` URIs, as artifacts of the session, as 
PySpark's
     /// `addArtifacts(*path, pyfile=False, archive=False, file=False)`: each 
path is named by
     /// the flags and its extension, and the errors for a path that is not 
supported are
-    /// PySpark's (see `spark_connect_core::artifact::parse_artifact`).
+    /// PySpark's (see `spark_connect_core::artifact::parse_artifact`). A 
class file is refused,
+    /// as PySpark's ArtifactManager does not upload one.
     #[pyo3(name = "addArtifacts", signature = (*path, pyfile=false, 
archive=false, file=false))]
     fn add_artifacts(
         &self,
@@ -1646,8 +1648,6 @@ impl PySparkSession {
         archive: bool,
         file: bool,
     ) -> PyResult<()> {
-        use spark_connect_core::artifact::{parse_artifact, 
UnsupportedArtifact};
-
         if [pyfile, archive, file].iter().filter(|flag| **flag).count() > 1 {
             return Err(crate::column::raise_pyspark(
                 py,
@@ -1659,29 +1659,18 @@ impl PySparkSession {
                 ],
             ));
         }
+        // As PySpark's _parse_artifacts, every path is parsed, and a local 
file opened, before
+        // any upload.
         for p in &path {
-            if let Err(unsupported) = parse_artifact(p, pyfile, archive, file) 
{
-                return Err(match unsupported {
-                    UnsupportedArtifact::Format(local) => 
crate::column::raise_pyspark(
-                        py,
-                        "PySparkRuntimeError",
-                        "UNSUPPORTED_OPERATION",
-                        &[("operation", &format!("{local} file format"))],
-                    ),
-                    UnsupportedArtifact::Scheme(scheme) => 
crate::column::raise_pyspark(
-                        py,
-                        "PySparkRuntimeError",
-                        "UNSUPPORTED_OPERATION",
-                        &[("operation", &format!("{scheme} scheme"))],
-                    ),
-                    UnsupportedArtifact::HashInArchivePath => 
crate::column::raise_pyspark(
-                        py,
-                        "PySparkValueError",
-                        "VALUE_ALLOWED",
-                        &[("arg_name", "artifact path"), ("disallowed_value", 
"#")],
-                    ),
-                });
+            let (name, local) = match parse_artifact(p, pyfile, archive, file) 
{
+                Ok(parsed) => parsed,
+                Err(unsupported) => return Err(unsupported_artifact_error(py, 
unsupported)),
+            };
+            if name.starts_with("classes/") {
+                let format = 
UnsupportedArtifact::Format(local.to_string_lossy().into_owned());
+                return Err(unsupported_artifact_error(py, format));
             }
+            check_local_artifact(py, &name, &local, pyfile)?;
         }
         let refs: Vec<&str> = path.iter().map(|s| s.as_str()).collect();
         py.detach(|| {
@@ -1731,6 +1720,56 @@ impl PySparkSession {
     }
 }
 
+/// PySpark's error for a path of `addArtifacts` that it does not upload.
+fn unsupported_artifact_error(py: Python<'_>, unsupported: 
UnsupportedArtifact) -> PyErr {
+    match unsupported {
+        UnsupportedArtifact::Format(local) => crate::column::raise_pyspark(
+            py,
+            "PySparkRuntimeError",
+            "UNSUPPORTED_OPERATION",
+            &[("operation", &format!("{local} file format"))],
+        ),
+        UnsupportedArtifact::Scheme(scheme) => crate::column::raise_pyspark(
+            py,
+            "PySparkRuntimeError",
+            "UNSUPPORTED_OPERATION",
+            &[("operation", &format!("{scheme} scheme"))],
+        ),
+        UnsupportedArtifact::HashInArchivePath => crate::column::raise_pyspark(
+            py,
+            "PySparkValueError",
+            "VALUE_ALLOWED",
+            &[("arg_name", "artifact path"), ("disallowed_value", "#")],
+        ),
+    }
+}
+
+/// The client side of parsing a local artifact, as PySpark's 
`_parse_artifacts` does it: a
+/// Python file, zip, egg or jar of `pyfile` is importable by the client, and 
the file is opened
+/// as PySpark's `LocalFile` opens it, so that a missing file or a directory 
raises the error of
+/// `open` before any upload.
+fn check_local_artifact(
+    py: Python<'_>,
+    name: &str,
+    local: &std::path::Path,
+    pyfile: bool,
+) -> PyResult<()> {
+    let local = local.to_string_lossy();
+    let importable = pyfile && name.starts_with("pyfiles/");
+    if importable && !name.ends_with(".py") {
+        py.import("sys")?
+            .getattr("path")?
+            .call_method1("insert", (1, local.as_ref()))?;
+    }
+    py.import("builtins")?
+        .call_method1("open", (local.as_ref(), "rb"))?
+        .call_method0("close")?;
+    if importable {
+        py.import("importlib")?.call_method0("invalidate_caches")?;
+    }
+    Ok(())
+}
+
 /// Minimal stand-in for the reference `SparkConnectClient`, returned by
 /// `SparkSession.client`. The real client is the Rust transport; this only 
surfaces the
 /// members that test/utility code (e.g. `ReusedConnectTestCase`) touches.
diff --git a/crates/spark-connect-core/src/artifact.rs 
b/crates/spark-connect-core/src/artifact.rs
index 692d5a5..385bea7 100644
--- a/crates/spark-connect-core/src/artifact.rs
+++ b/crates/spark-connect-core/src/artifact.rs
@@ -10,6 +10,7 @@
 use bytes::Bytes;
 use crc32fast::Hasher;
 use std::io::{Cursor, Read};
+use std::sync::{Arc, Mutex, PoisonError};
 
 use spark_connect_proto::{add_artifacts_request, AddArtifactsRequest};
 
@@ -275,61 +276,158 @@ pub fn build_artifact_request_stream(
     Ok(requests)
 }
 
-/// Build requests for a large chunked artifact.
-fn build_chunked_artifact_requests(
+/// The error that ended the frames of an upload early, if one did. The frames 
stop at a read
+/// that fails; the upload reports the error once its RPC has returned.
+pub(crate) type FrameFailure = Arc<Mutex<Option<SparkError>>>;
+
+/// Take the error that ended the frames of an upload early, if one did.
+pub(crate) fn take_frame_failure(failure: &FrameFailure) -> Result<()> {
+    match failure
+        .lock()
+        .unwrap_or_else(PoisonError::into_inner)
+        .take()
+    {
+        Some(error) => Err(error),
+        None => Ok(()),
+    }
+}
+
+/// The frames of an upload of one artifact, produced as the RPC consumes 
them, so that a large
+/// artifact is held one chunk at a time rather than as a whole. An attempt 
that is retried
+/// builds its frames again, from the start of the artifact.
+///
+/// The frames are those of [`build_artifact_request_stream`] for the 
artifact: one batch when it
+/// is at most [`CHUNK_SIZE`] bytes, otherwise a `BeginChunk` and then 
`Chunk`s, each a full
+/// chunk but the last.
+pub(crate) struct ArtifactFrames {
     session_id: String,
-    artifact: NamedArtifact,
-) -> Result<Vec<AddArtifactsRequest>> {
-    let mut requests = Vec::new();
-    let total_size = artifact.data.size()? as i64;
-    let num_chunks = (total_size as usize).div_ceil(CHUNK_SIZE);
+    name: String,
+    size: u64,
+    /// The data, until its last frame is produced or a read fails.
+    reader: Option<Box<dyn Read + Send>>,
+    /// Whether the first chunk of a chunked artifact has been produced.
+    begun: bool,
+    failure: FrameFailure,
+}
 
-    let mut stream = artifact.data.stream()?;
-    let mut first_chunk = true;
+impl ArtifactFrames {
+    /// Open the data of `artifact`, so that a data source that cannot be read 
is an error before
+    /// any frame is sent.
+    pub(crate) fn new(session_id: String, artifact: &NamedArtifact) -> 
Result<Self> {
+        let size = artifact.data.size()?;
+        let reader = artifact.data.stream()?;
+        Ok(Self {
+            session_id,
+            name: artifact.name.clone(),
+            size,
+            reader: Some(reader),
+            begun: false,
+            failure: Arc::default(),
+        })
+    }
+
+    /// The failure that ends these frames early, which the upload checks once 
its RPC returns.
+    pub(crate) fn failure(&self) -> FrameFailure {
+        self.failure.clone()
+    }
 
-    let mut buf = vec![0u8; CHUNK_SIZE];
-    loop {
-        let n = stream.read(&mut buf).map_err(|e| {
+    fn next_payload(&mut self) -> 
Result<Option<add_artifacts_request::Payload>> {
+        let Some(mut reader) = self.reader.take() else {
+            return Ok(None);
+        };
+        if self.size <= CHUNK_SIZE as u64 {
+            let mut data = Vec::new();
+            reader
+                .read_to_end(&mut data)
+                .map_err(|e| SparkError::connect_msg(format!("Failed to read 
artifact: {}", e)))?;
+            let crc = compute_crc32(&data) as i64;
+            let artifact = add_artifacts_request::SingleChunkArtifact {
+                name: self.name.clone(),
+                data: Some(add_artifacts_request::ArtifactChunk {
+                    data: Bytes::from(data),
+                    crc,
+                }),
+            };
+            return Ok(Some(add_artifacts_request::Payload::Batch(
+                add_artifacts_request::Batch {
+                    artifacts: vec![artifact],
+                },
+            )));
+        }
+        let mut buf = vec![0u8; CHUNK_SIZE];
+        let filled = read_full(reader.as_mut(), &mut buf).map_err(|e| {
             SparkError::connect_msg(format!("Failed to read artifact chunk: 
{}", e))
         })?;
-        if n == 0 {
-            break;
+        buf.truncate(filled);
+        if filled == 0 {
+            return Ok(None);
         }
-
-        let chunk_data = buf[..n].to_vec();
-        let crc = compute_crc32(&chunk_data) as i64;
-
-        if first_chunk {
-            // Begin chunked artifact with metadata and initial chunk
-            let mut request = AddArtifactsRequest::default();
-            request.session_id = session_id.clone();
-            request.payload = Some(add_artifacts_request::Payload::BeginChunk(
-                add_artifacts_request::BeginChunkedArtifact {
-                    name: artifact.name.clone(),
-                    total_bytes: total_size,
-                    num_chunks: num_chunks as i64,
-                    initial_chunk: Some(add_artifacts_request::ArtifactChunk {
-                        data: Bytes::from(chunk_data),
-                        crc,
-                    }),
-                },
-            ));
-            requests.push(request);
-            first_chunk = false;
+        if filled == CHUNK_SIZE {
+            self.reader = Some(reader);
+        }
+        let crc = compute_crc32(&buf) as i64;
+        let chunk = add_artifacts_request::ArtifactChunk {
+            data: Bytes::from(buf),
+            crc,
+        };
+        if self.begun {
+            Ok(Some(add_artifacts_request::Payload::Chunk(chunk)))
         } else {
-            // Subsequent chunk
-            let mut request = AddArtifactsRequest::default();
-            request.session_id = session_id.clone();
-            request.payload = Some(add_artifacts_request::Payload::Chunk(
-                add_artifacts_request::ArtifactChunk {
-                    data: Bytes::from(chunk_data),
-                    crc,
+            self.begun = true;
+            Ok(Some(add_artifacts_request::Payload::BeginChunk(
+                add_artifacts_request::BeginChunkedArtifact {
+                    name: self.name.clone(),
+                    total_bytes: self.size as i64,
+                    num_chunks: self.size.div_ceil(CHUNK_SIZE as u64) as i64,
+                    initial_chunk: Some(chunk),
                 },
-            ));
-            requests.push(request);
+            )))
+        }
+    }
+}
+
+impl Iterator for ArtifactFrames {
+    type Item = AddArtifactsRequest;
+
+    fn next(&mut self) -> Option<AddArtifactsRequest> {
+        match self.next_payload() {
+            Ok(Some(payload)) => Some(AddArtifactsRequest {
+                session_id: self.session_id.clone(),
+                payload: Some(payload),
+                ..Default::default()
+            }),
+            Ok(None) => None,
+            Err(error) => {
+                self.reader = None;
+                *self.failure.lock().unwrap_or_else(PoisonError::into_inner) = 
Some(error);
+                None
+            }
+        }
+    }
+}
+
+/// Fill `buf` from `reader` until it is full or the data ends, and return the 
count read.
+fn read_full(reader: &mut dyn Read, buf: &mut [u8]) -> std::io::Result<usize> {
+    let mut filled = 0;
+    while filled < buf.len() {
+        match reader.read(&mut buf[filled..]) {
+            Ok(0) => break,
+            Ok(n) => filled += n,
+            Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
+            Err(e) => return Err(e),
         }
     }
+    Ok(filled)
+}
 
+/// Build requests for a large chunked artifact.
+fn build_chunked_artifact_requests(
+    session_id: String,
+    artifact: NamedArtifact,
+) -> Result<Vec<AddArtifactsRequest>> {
+    let mut frames = ArtifactFrames::new(session_id, &artifact)?;
+    let requests: Vec<AddArtifactsRequest> = frames.by_ref().collect();
+    take_frame_failure(&frames.failure())?;
     Ok(requests)
 }
 
@@ -399,6 +497,12 @@ fn unquote(text: &str) -> String {
 /// - With `file`, any file is `files/<name>`.
 /// - Otherwise a `.class` is `classes/<name>`, a `.jar` is `jars/<name>`, and 
any other file
 ///   is not supported.
+///
+/// The `classes/` of a class file is an extension of this crate: PySpark's 
`ArtifactManager`
+/// does not upload class files, so the PySpark drop-in refuses them.
+///
+/// The local path is normalized as `pathlib` normalizes it (no trailing or 
repeated separators,
+/// and no `.` components), so that it reads the same in an error message.
 pub fn parse_artifact(
     path_or_uri: &str,
     pyfile: bool,
@@ -436,6 +540,7 @@ pub fn parse_artifact(
         "file" => std::path::PathBuf::from(unquote(path)),
         _ => return Err(UnsupportedArtifact::Scheme(scheme)),
     };
+    let local: std::path::PathBuf = local.components().collect();
     let local_path = local.to_string_lossy().into_owned();
     let name = local
         .file_name()
@@ -525,6 +630,12 @@ mod tests {
                 (false, true, false),
                 Err(HashInArchivePath),
             ),
+            // The local path is normalized as pathlib normalizes it, in the 
error too.
+            (
+                "/d//x/./n.txt/",
+                (false, false, false),
+                Err(Format("/d/x/n.txt".to_string())),
+            ),
         ];
         for (path, (pyfile, archive, file), expected) in cases {
             let parsed = parse_artifact(path, pyfile, archive, 
file).map(|(name, _)| name);
@@ -844,4 +955,191 @@ mod tests {
         assert_eq!(second_drops.load(Ordering::SeqCst), 1);
         assert_eq!(first_drops.load(Ordering::SeqCst), 1);
     }
+
+    /// Test data that is produced as it is read, and counts the bytes read of 
it.
+    struct Counted {
+        size: u64,
+        read: Arc<std::sync::atomic::AtomicUsize>,
+        /// The read that fails, when the data has one.
+        fails_at_read: Option<usize>,
+    }
+
+    impl ArtifactData for Counted {
+        fn size(&self) -> Result<u64> {
+            Ok(self.size)
+        }
+
+        fn stream(&self) -> Result<Box<dyn Read + Send>> {
+            Ok(Box::new(CountedReader {
+                remaining: self.size as usize,
+                read: self.read.clone(),
+                reads: 0,
+                fails_at_read: self.fails_at_read,
+            }))
+        }
+    }
+
+    struct CountedReader {
+        remaining: usize,
+        read: Arc<std::sync::atomic::AtomicUsize>,
+        reads: usize,
+        fails_at_read: Option<usize>,
+    }
+
+    impl Read for CountedReader {
+        fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+            self.reads += 1;
+            if self.fails_at_read == Some(self.reads) {
+                return Err(std::io::Error::other("disk gone"));
+            }
+            let n = buf.len().min(self.remaining);
+            buf[..n].fill(7);
+            self.remaining -= n;
+            self.read.fetch_add(n, std::sync::atomic::Ordering::SeqCst);
+            Ok(n)
+        }
+    }
+
+    fn counted(size: u64) -> (NamedArtifact, 
Arc<std::sync::atomic::AtomicUsize>) {
+        let read = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let data = Counted {
+            size,
+            read: read.clone(),
+            fails_at_read: None,
+        };
+        (
+            NamedArtifact::new("files/n.bin".to_string(), Box::new(data)),
+            read,
+        )
+    }
+
+    #[test]
+    fn artifact_frames_are_read_as_the_rpc_consumes_them() {
+        let size = (CHUNK_SIZE * 10 + 3) as u64;
+        let (artifact, read) = counted(size);
+        let mut frames = ArtifactFrames::new("s".to_string(), 
&artifact).unwrap();
+        // Opening the data reads none of it.
+        assert_eq!(read.load(std::sync::atomic::Ordering::SeqCst), 0);
+        frames.next().unwrap();
+        assert_eq!(read.load(std::sync::atomic::Ordering::SeqCst), CHUNK_SIZE);
+        frames.next().unwrap();
+        assert_eq!(
+            read.load(std::sync::atomic::Ordering::SeqCst),
+            2 * CHUNK_SIZE
+        );
+        assert_eq!(frames.by_ref().count(), 9);
+        assert_eq!(
+            read.load(std::sync::atomic::Ordering::SeqCst),
+            size as usize
+        );
+        assert!(take_frame_failure(&frames.failure()).is_ok());
+    }
+
+    #[test]
+    fn chunked_artifact_frames_begin_and_then_carry_each_chunk_with_its_crc() {
+        let size = CHUNK_SIZE * 2 + 5;
+        let data: Vec<u8> = (0..size).map(|i| (i % 251) as u8).collect();
+        let artifact = NamedArtifact::new(
+            "jars/c.jar".to_string(),
+            Box::new(InMemoryArtifact::new(data.clone())),
+        );
+        let frames: Vec<AddArtifactsRequest> = 
ArtifactFrames::new("s".to_string(), &artifact)
+            .unwrap()
+            .collect();
+        assert_eq!(frames.len(), 3);
+        let mut expected = data.chunks(CHUNK_SIZE);
+        for (index, frame) in frames.iter().enumerate() {
+            assert_eq!(frame.session_id, "s");
+            let chunk = expected.next().unwrap();
+            let (payload, crc) = match frame.payload.as_ref().unwrap() {
+                add_artifacts_request::Payload::BeginChunk(begin) => {
+                    assert_eq!(index, 0);
+                    assert_eq!(begin.name, "jars/c.jar");
+                    assert_eq!(begin.total_bytes, size as i64);
+                    assert_eq!(begin.num_chunks, 3);
+                    let initial = begin.initial_chunk.as_ref().unwrap();
+                    (initial.data.to_vec(), initial.crc)
+                }
+                add_artifacts_request::Payload::Chunk(chunk) => {
+                    assert_ne!(index, 0);
+                    (chunk.data.to_vec(), chunk.crc)
+                }
+                other => panic!("unexpected frame {other:?}"),
+            };
+            assert_eq!(payload, chunk);
+            assert_eq!(crc, compute_crc32(chunk) as i64);
+        }
+    }
+
+    #[test]
+    fn a_small_artifact_frame_is_one_batch() {
+        let artifact = NamedArtifact::new(
+            "classes/A.class".to_string(),
+            Box::new(InMemoryArtifact::new(b"cafebabe".to_vec())),
+        );
+        let frames: Vec<AddArtifactsRequest> = 
ArtifactFrames::new("s".to_string(), &artifact)
+            .unwrap()
+            .collect();
+        let [frame] = frames.as_slice() else {
+            panic!("expected one frame, got {}", frames.len());
+        };
+        let Some(add_artifacts_request::Payload::Batch(batch)) = 
&frame.payload else {
+            panic!("expected a batch");
+        };
+        let [only] = batch.artifacts.as_slice() else {
+            panic!("expected one artifact");
+        };
+        assert_eq!(only.name, "classes/A.class");
+        let chunk = only.data.as_ref().unwrap();
+        assert_eq!(chunk.data.as_ref(), b"cafebabe");
+        assert_eq!(chunk.crc, compute_crc32(b"cafebabe") as i64);
+    }
+
+    #[test]
+    fn a_failed_read_ends_the_frames_and_is_reported_once() {
+        let read = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let artifact = NamedArtifact::new(
+            "files/n.bin".to_string(),
+            Box::new(Counted {
+                size: (CHUNK_SIZE * 3) as u64,
+                read,
+                // The first chunk is read; the read of the second chunk fails.
+                fails_at_read: Some(2),
+            }),
+        );
+        let mut frames = ArtifactFrames::new("s".to_string(), 
&artifact).unwrap();
+        let failure = frames.failure();
+        assert!(frames.next().is_some());
+        assert!(frames.next().is_none());
+        assert!(frames.next().is_none());
+        let error = take_frame_failure(&failure).unwrap_err();
+        assert!(
+            error.to_string().contains("Failed to read artifact chunk"),
+            "{error}"
+        );
+        assert!(take_frame_failure(&failure).is_ok());
+    }
+
+    #[test]
+    fn artifact_frames_are_the_same_on_every_attempt() {
+        let size = CHUNK_SIZE * 2 + 1;
+        let data: Vec<u8> = (0..size).map(|i| (i % 13) as u8).collect();
+        let make = || {
+            NamedArtifact::new(
+                "jars/c.jar".to_string(),
+                Box::new(InMemoryArtifact::new(data.clone())),
+            )
+        };
+        let first = make();
+        let second = make();
+        let a: Vec<AddArtifactsRequest> = ArtifactFrames::new("s".to_string(), 
&first)
+            .unwrap()
+            .collect();
+        let b: Vec<AddArtifactsRequest> = ArtifactFrames::new("s".to_string(), 
&second)
+            .unwrap()
+            .collect();
+        assert_eq!(a, b);
+        let batch = build_artifact_request_stream("s".to_string(), 
vec![make()]).unwrap();
+        assert_eq!(a, batch);
+    }
 }
diff --git a/crates/spark-connect-core/src/client.rs 
b/crates/spark-connect-core/src/client.rs
index 9e26346..62c1652 100644
--- a/crates/spark-connect-core/src/client.rs
+++ b/crates/spark-connect-core/src/client.rs
@@ -20,8 +20,8 @@ use spark_connect_proto::{
 };
 
 use crate::artifact::{
-    build_artifact_request_stream, cache_artifact_requests, 
check_artifact_upload, FileArtifact,
-    NamedArtifact,
+    cache_artifact_requests, check_artifact_upload, take_frame_failure, 
ArtifactFrames,
+    FileArtifact, NamedArtifact,
 };
 use crate::channel::{ChannelBuilder, GRPC_MAX_MESSAGE_LENGTH_DEFAULT};
 use crate::error::{Result, SparkError};
@@ -1079,9 +1079,8 @@ impl SparkConnectClient {
         for path in paths {
             let (name, local) = crate::artifact::parse_artifact(path, pyfile, 
archive, file)
                 .map_err(|e| SparkError::connect_msg(e.to_string()))?;
-            let artifacts = vec![NamedArtifact::new(name, 
Box::new(FileArtifact::new(local)))];
-            let requests = 
build_artifact_request_stream(self.session_id.clone(), artifacts)?;
-            self.upload_artifact_requests(requests, None).await?;
+            let artifact = NamedArtifact::new(name, 
Box::new(FileArtifact::new(local)));
+            self.upload_artifact(&artifact).await?;
         }
         Ok(())
     }
@@ -1092,12 +1091,9 @@ impl SparkConnectClient {
     /// Used by `SparkSession::copy_from_local_to_fs`, which names the artifact
     /// `forward_to_fs/<dest>` so the server writes it to the target 
filesystem path.
     pub async fn add_named_artifact(&self, name: &str, local_path: &str) -> 
Result<()> {
-        let artifacts = vec![NamedArtifact::new(
-            name.to_string(),
-            Box::new(FileArtifact::new(local_path)),
-        )];
-        let requests = build_artifact_request_stream(self.session_id.clone(), 
artifacts)?;
-        self.upload_artifact_requests(requests, None).await
+        let artifact =
+            NamedArtifact::new(name.to_string(), 
Box::new(FileArtifact::new(local_path)));
+        self.upload_artifact(&artifact).await
     }
 
     /// Upload session-scoped cache blobs, returning SHA-256 hashes in input 
order.
@@ -1173,38 +1169,47 @@ impl SparkConnectClient {
         }
     }
 
-    /// Send artifact frames, retrying a transient failure of the whole upload 
under the retry
-    /// policy, and validate the response. An attempt sends all the frames 
again: a stream is
-    /// not resumed. Cache uploads also require an acknowledgement for every 
requested
-    /// artifact.
-    async fn upload_artifact_requests(
-        &self,
-        requests: Vec<spark_connect_proto::AddArtifactsRequest>,
-        expected_cache_names: Option<&std::collections::HashSet<String>>,
-    ) -> Result<()> {
-        let requests = &requests;
-        let response = self
-            .with_retry(|| self.add_artifacts_rpc(requests.clone()))
-            .await?;
-        check_artifact_upload(response, expected_cache_names)
+    /// Upload one artifact, retrying a transient failure of the whole upload 
under the retry
+    /// policy, and validate the response. An attempt sends all the frames 
again from the start:
+    /// a stream is not resumed. The frames are read as the RPC consumes them, 
so an attempt
+    /// holds one chunk of the artifact at a time.
+    async fn upload_artifact(&self, artifact: &NamedArtifact) -> Result<()> {
+        let mut state = RetryPolicyState::new(self.retry_policy.clone());
+        loop {
+            // A data source that cannot be opened is an error before any 
frame is sent.
+            let frames = ArtifactFrames::new(self.session_id.clone(), 
artifact)?;
+            let failure = frames.failure();
+            let result = self.add_artifacts_rpc(frames).await;
+            // A read that failed ended the frames early, whatever the RPC 
answered.
+            take_frame_failure(&failure)?;
+            match result {
+                Ok(response) => return check_artifact_upload(response, None),
+                Err(status) => {
+                    if self.retry_policy.can_retry(&status) {
+                        if let Some(wait_ms) = state.next_attempt(None) {
+                            
tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await;
+                            continue;
+                        }
+                    }
+                    return Err(self.error_from_status(status).await);
+                }
+            }
+        }
     }
 
     /// The AddArtifacts call of `requests`, once. Stamps user_context and 
client_type on
-    /// every frame; all upload paths must go through here.
+    /// every frame as it is sent; all upload paths must go through here.
     async fn add_artifacts_rpc(
         &self,
-        requests: impl IntoIterator<Item = 
spark_connect_proto::AddArtifactsRequest>,
+        requests: impl Iterator<Item = 
spark_connect_proto::AddArtifactsRequest> + Send + 'static,
     ) -> std::result::Result<spark_connect_proto::AddArtifactsResponse, 
tonic::Status> {
         let user_context = self.artifact_user_context();
         let client_type = self.user_agent.clone();
-        let requests: Vec<_> = requests
-            .into_iter()
-            .map(|mut request| {
-                request.user_context = Some(user_context.clone());
-                request.client_type = Some(client_type.clone());
-                request
-            })
-            .collect();
+        let requests = requests.map(move |mut request| {
+            request.user_context = Some(user_context.clone());
+            request.client_type = Some(client_type.clone());
+            request
+        });
         let deadline = self.rpc_deadlines.add_artifacts;
         let mut req = 
with_deadline(Request::new(futures::stream::iter(requests)), deadline);
         self._attach_metadata(&mut req);
diff --git a/crates/spark-connect-core/tests/artifact_uploads.rs 
b/crates/spark-connect-core/tests/artifact_uploads.rs
new file mode 100644
index 0000000..3f0cf9a
--- /dev/null
+++ b/crates/spark-connect-core/tests/artifact_uploads.rs
@@ -0,0 +1,462 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Artifact uploads on the wire: the frames of every AddArtifacts attempt, 
its CRCs and its
+//! retries, against a loopback gRPC service that receives and acknowledges 
them; no Spark
+//! server required.
+
+use std::collections::VecDeque;
+use std::convert::Infallible;
+use std::sync::{Arc, Mutex};
+use std::task::{Context, Poll};
+use std::time::Duration;
+
+use futures::{stream, StreamExt};
+use prost::Message;
+use spark_connect_core::bytes_codec::BytesCodec;
+use spark_connect_core::{
+    compute_crc32, ChannelBuilder, RetryPolicy, SparkConnectClient, CHUNK_SIZE,
+};
+use spark_connect_proto::add_artifacts_request::Payload;
+use spark_connect_proto::add_artifacts_response::ArtifactSummary;
+use spark_connect_proto::{AddArtifactsRequest, AddArtifactsResponse};
+use tokio::net::TcpListener;
+use tonic::body::Body;
+use tonic::codegen::{http, BoxFuture, Service};
+use tonic::server::{ClientStreamingService, NamedService, UnaryService};
+use tonic::transport::Server;
+use tonic::{Code, Request, Response, Status, Streaming};
+
+const WAIT: Duration = Duration::from_secs(5);
+const SERVICE: &str = "/spark.connect.SparkConnectService/";
+
+/// How the server handles one AddArtifacts attempt.
+#[derive(Clone, Copy)]
+enum Attempt {
+    /// Every frame is received, and every artifact is acknowledged with a 
good CRC.
+    Accept,
+    /// The first `n` frames are received, then the call fails with 
UNAVAILABLE.
+    FailAfter(usize),
+    /// Every frame is received, but every artifact is acknowledged with a bad 
CRC.
+    CorruptCrc,
+    /// The call fails with the code before any frame is received.
+    Refuse(Code),
+}
+
+struct State {
+    attempts: Mutex<VecDeque<Attempt>>,
+    /// The frames that each AddArtifacts attempt received, in order of the 
attempts.
+    received: Mutex<Vec<Vec<AddArtifactsRequest>>>,
+}
+
+struct Fixture {
+    state: Arc<State>,
+    server: tokio::task::JoinHandle<()>,
+    address: std::net::SocketAddr,
+}
+
+impl Fixture {
+    async fn start(attempts: impl IntoIterator<Item = Attempt>) -> Self {
+        let state = Arc::new(State {
+            attempts: Mutex::new(attempts.into_iter().collect()),
+            received: Mutex::new(Vec::new()),
+        });
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let incoming = stream::unfold(listener, |listener| async {
+            Some((listener.accept().await.map(|(socket, _)| socket), listener))
+        });
+        let service = TestService(state.clone());
+        let server = tokio::spawn(async move {
+            Server::builder()
+                .add_service(service)
+                .serve_with_incoming(incoming)
+                .await
+                .unwrap();
+        });
+        Self {
+            state,
+            server,
+            address,
+        }
+    }
+
+    async fn client(&self) -> SparkConnectClient {
+        let builder = ChannelBuilder::parse(&format!("sc://{}", 
self.address)).unwrap();
+        let mut client = tokio::time::timeout(WAIT, 
SparkConnectClient::connect(&builder))
+            .await
+            .expect("connecting timed out")
+            .unwrap();
+        client.set_retry_policy(RetryPolicy {
+            max_retries: Some(2),
+            initial_backoff_ms: 1,
+            max_backoff_ms: Some(2),
+            ..RetryPolicy::default()
+        });
+        client
+    }
+
+    /// The frames that each AddArtifacts attempt received, in order.
+    fn attempts(&self) -> Vec<Vec<AddArtifactsRequest>> {
+        self.state.received.lock().unwrap().clone()
+    }
+}
+
+impl Drop for Fixture {
+    fn drop(&mut self) {
+        self.server.abort();
+    }
+}
+
+/// Bytes that differ from one position to the next, so that a misplaced chunk 
shows.
+fn pattern(len: usize) -> Vec<u8> {
+    (0..len).map(|i| (i % 251) as u8).collect()
+}
+
+/// The data that the frames of one upload carry, which every chunk of a 
chunked artifact must
+/// carry in order.
+fn assembled(frames: &[AddArtifactsRequest]) -> Vec<u8> {
+    let mut data = Vec::new();
+    for frame in frames {
+        match frame.payload.as_ref().unwrap() {
+            Payload::Batch(batch) => {
+                for artifact in &batch.artifacts {
+                    
data.extend_from_slice(&artifact.data.as_ref().unwrap().data);
+                }
+            }
+            Payload::BeginChunk(begin) => {
+                
data.extend_from_slice(&begin.initial_chunk.as_ref().unwrap().data)
+            }
+            Payload::Chunk(chunk) => data.extend_from_slice(&chunk.data),
+        }
+    }
+    data
+}
+
+/// The data length of each frame's chunk, in order.
+fn chunk_lengths(frames: &[AddArtifactsRequest]) -> Vec<usize> {
+    frames
+        .iter()
+        .map(|frame| match frame.payload.as_ref().unwrap() {
+            Payload::Batch(batch) => 
batch.artifacts[0].data.as_ref().unwrap().data.len(),
+            Payload::BeginChunk(begin) => 
begin.initial_chunk.as_ref().unwrap().data.len(),
+            Payload::Chunk(chunk) => chunk.data.len(),
+        })
+        .collect()
+}
+
+/// A temporary file of `contents`, which is removed with the directory that 
holds it.
+fn file_with(dir: &tempfile::TempDir, name: &str, contents: &[u8]) -> String {
+    let path = dir.path().join(name);
+    std::fs::write(&path, contents).unwrap();
+    path.to_str().unwrap().to_string()
+}
+
+#[tokio::test]
+async fn a_large_file_is_uploaded_as_one_chunked_artifact() {
+    let fixture = Fixture::start([]).await;
+    let client = fixture.client().await;
+    let dir = tempfile::tempdir().unwrap();
+    let data = pattern(3 * CHUNK_SIZE + 5);
+    let path = file_with(&dir, "large.bin", &data);
+
+    client
+        .add_artifacts(&[path.as_str()], false, false, true)
+        .await
+        .unwrap();
+
+    let attempts = fixture.attempts();
+    assert_eq!(attempts.len(), 1);
+    let frames = &attempts[0];
+    // A BeginChunk with the first chunk, then a Chunk for each further chunk, 
the last short.
+    assert_eq!(
+        chunk_lengths(frames),
+        vec![CHUNK_SIZE, CHUNK_SIZE, CHUNK_SIZE, 5]
+    );
+    let Some(Payload::BeginChunk(begin)) = &frames[0].payload else {
+        panic!("the first frame must begin the chunked artifact");
+    };
+    assert_eq!(begin.name, "files/large.bin");
+    assert_eq!(begin.total_bytes, data.len() as i64);
+    assert_eq!(begin.num_chunks, 4);
+    for frame in frames.iter().skip(1) {
+        assert!(matches!(frame.payload, Some(Payload::Chunk(_))));
+    }
+    // Every frame carries the CRC of its chunk, and the session and user 
context of the client.
+    for frame in frames {
+        assert_eq!(frame.session_id, client.session_id());
+        assert!(frame.user_context.is_some());
+        assert!(frame.client_type.is_some());
+    }
+    let crcs: Vec<i64> = frames
+        .iter()
+        .map(|frame| match frame.payload.as_ref().unwrap() {
+            Payload::BeginChunk(begin) => 
begin.initial_chunk.as_ref().unwrap().crc,
+            Payload::Chunk(chunk) => chunk.crc,
+            Payload::Batch(_) => unreachable!(),
+        })
+        .collect();
+    let expected: Vec<i64> = data
+        .chunks(CHUNK_SIZE)
+        .map(|chunk| compute_crc32(chunk) as i64)
+        .collect();
+    assert_eq!(crcs, expected);
+    assert_eq!(assembled(frames), data);
+}
+
+#[tokio::test]
+async fn a_small_file_is_one_batch_named_by_its_extension() {
+    let fixture = Fixture::start([]).await;
+    let client = fixture.client().await;
+    let dir = tempfile::tempdir().unwrap();
+    let class = file_with(&dir, "A.class", b"cafebabe");
+
+    // A class file is uploaded by the Rust API, as classes/.
+    client
+        .add_artifacts(&[class.as_str()], false, false, false)
+        .await
+        .unwrap();
+
+    let attempts = fixture.attempts();
+    assert_eq!(attempts.len(), 1);
+    let [frame] = attempts[0].as_slice() else {
+        panic!("expected one frame");
+    };
+    let Some(Payload::Batch(batch)) = &frame.payload else {
+        panic!("expected a batch");
+    };
+    let [artifact] = batch.artifacts.as_slice() else {
+        panic!("expected one artifact");
+    };
+    assert_eq!(artifact.name, "classes/A.class");
+    let chunk = artifact.data.as_ref().unwrap();
+    assert_eq!(chunk.data.as_ref(), b"cafebabe");
+    assert_eq!(chunk.crc, compute_crc32(b"cafebabe") as i64);
+}
+
+#[tokio::test]
+async fn a_transient_failure_retries_the_whole_upload_from_its_start() {
+    // The first attempt receives two frames and then fails; the second 
receives the upload.
+    let fixture = Fixture::start([Attempt::FailAfter(2), 
Attempt::Accept]).await;
+    let client = fixture.client().await;
+    let dir = tempfile::tempdir().unwrap();
+    let data = pattern(3 * CHUNK_SIZE + 5);
+    let path = file_with(&dir, "large.bin", &data);
+
+    client
+        .add_artifacts(&[path.as_str()], false, false, true)
+        .await
+        .unwrap();
+
+    let attempts = fixture.attempts();
+    assert_eq!(attempts.len(), 2);
+    assert_eq!(attempts[0].len(), 2);
+    let whole = &attempts[1];
+    assert!(matches!(whole[0].payload, Some(Payload::BeginChunk(_))));
+    assert_eq!(
+        chunk_lengths(whole),
+        vec![CHUNK_SIZE, CHUNK_SIZE, CHUNK_SIZE, 5]
+    );
+    assert_eq!(assembled(whole), data);
+    // The frames of the retried attempt are those of the first, from the 
start.
+    assert_eq!(&whole[..2], attempts[0].as_slice());
+}
+
+#[tokio::test]
+async fn a_crc_mismatch_is_reported_and_not_retried() {
+    let fixture = Fixture::start([Attempt::CorruptCrc]).await;
+    let client = fixture.client().await;
+    let dir = tempfile::tempdir().unwrap();
+    let path = file_with(&dir, "small.txt", b"small");
+
+    let error = client
+        .add_artifacts(&[path.as_str()], false, false, true)
+        .await
+        .unwrap_err();
+
+    assert!(
+        error
+            .to_string()
+            .contains("CRC check failed for artifact: files/small.txt"),
+        "{error}"
+    );
+    assert_eq!(fixture.attempts().len(), 1);
+}
+
+#[tokio::test]
+async fn a_failure_that_is_not_transient_is_not_retried() {
+    let fixture = 
Fixture::start([Attempt::Refuse(Code::PermissionDenied)]).await;
+    let client = fixture.client().await;
+    let dir = tempfile::tempdir().unwrap();
+    let path = file_with(&dir, "small.txt", b"small");
+
+    assert!(client
+        .add_artifacts(&[path.as_str()], false, false, true)
+        .await
+        .is_err());
+    // The refused attempt received no frame, and there was no second attempt.
+    assert_eq!(fixture.attempts(), vec![Vec::<AddArtifactsRequest>::new()]);
+}
+
+#[tokio::test]
+async fn a_missing_file_fails_before_any_attempt() {
+    let fixture = Fixture::start([]).await;
+    let client = fixture.client().await;
+    let dir = tempfile::tempdir().unwrap();
+    let missing = dir.path().join("missing.jar");
+
+    assert!(client
+        .add_artifacts(&[missing.to_str().unwrap()], false, false, false)
+        .await
+        .is_err());
+    assert!(fixture.attempts().is_empty());
+}
+
+/// The AddArtifacts service, which receives each attempt's frames and 
acknowledges them.
+#[derive(Clone)]
+struct TestService(Arc<State>);
+
+impl NamedService for TestService {
+    const NAME: &'static str = "spark.connect.SparkConnectService";
+}
+
+impl Service<http::Request<Body>> for TestService {
+    type Response = http::Response<Body>;
+    type Error = Infallible;
+    type Future = BoxFuture<Self::Response, Self::Error>;
+
+    fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll<Result<(), 
Self::Error>> {
+        Poll::Ready(Ok(()))
+    }
+
+    fn call(&mut self, request: http::Request<Body>) -> Self::Future {
+        let method = 
request.uri().path().trim_start_matches(SERVICE).to_string();
+        let state = self.0.clone();
+        Box::pin(async move {
+            let mut grpc = tonic::server::Grpc::new(BytesCodec);
+            Ok(match method.as_str() {
+                "AddArtifacts" => grpc.client_streaming(AddArtifacts(state), 
request).await,
+                _ => grpc.unary(Unary, request).await,
+            })
+        })
+    }
+}
+
+struct AddArtifacts(Arc<State>);
+
+impl ClientStreamingService<Vec<u8>> for AddArtifacts {
+    type Response = Vec<u8>;
+    type Future = BoxFuture<Response<Vec<u8>>, Status>;
+
+    fn call(&mut self, request: Request<Streaming<Vec<u8>>>) -> Self::Future {
+        let state = self.0.clone();
+        Box::pin(async move {
+            let attempt = state
+                .attempts
+                .lock()
+                .unwrap()
+                .pop_front()
+                .unwrap_or(Attempt::Accept);
+            if let Attempt::Refuse(code) = attempt {
+                state.received.lock().unwrap().push(Vec::new());
+                return Err(Status::new(code, "refused"));
+            }
+            let limit = match attempt {
+                Attempt::FailAfter(n) => n,
+                _ => usize::MAX,
+            };
+            let mut frames = request.into_inner();
+            let mut received = Vec::new();
+            while received.len() < limit {
+                let Some(bytes) = frames.next().await.transpose()? else {
+                    break;
+                };
+                
received.push(AddArtifactsRequest::decode(bytes.as_slice()).unwrap());
+            }
+            let complete = received.len() < limit;
+            let response = acknowledge(&received, matches!(attempt, 
Attempt::CorruptCrc));
+            state.received.lock().unwrap().push(received);
+            if !complete {
+                return Err(Status::unavailable("connection reset"));
+            }
+            Ok(Response::new(response.encode_to_vec()))
+        })
+    }
+}
+
+/// The response for frames: each artifact in order, with whether every one of 
its CRCs is good
+/// (and not corrupted, when `corrupt`).
+fn acknowledge(frames: &[AddArtifactsRequest], corrupt: bool) -> 
AddArtifactsResponse {
+    /// Records one CRC of the artifact named `name`, and returns the 
artifact's index.
+    fn record(artifacts: &mut Vec<(String, bool)>, name: &str, good: bool) -> 
usize {
+        let index = match artifacts.iter().position(|(n, _)| n == name) {
+            Some(index) => index,
+            None => {
+                artifacts.push((name.to_string(), true));
+                artifacts.len() - 1
+            }
+        };
+        artifacts[index].1 &= good;
+        index
+    }
+    let mut artifacts: Vec<(String, bool)> = Vec::new();
+    let mut current: Option<usize> = None;
+    for frame in frames {
+        match frame.payload.as_ref().unwrap() {
+            Payload::Batch(batch) => {
+                for artifact in &batch.artifacts {
+                    let chunk = artifact.data.as_ref().unwrap();
+                    let good = compute_crc32(&chunk.data) as i64 == chunk.crc;
+                    record(&mut artifacts, &artifact.name, good);
+                }
+            }
+            Payload::BeginChunk(begin) => {
+                let chunk = begin.initial_chunk.as_ref().unwrap();
+                let good = compute_crc32(&chunk.data) as i64 == chunk.crc;
+                current = Some(record(&mut artifacts, &begin.name, good));
+            }
+            Payload::Chunk(chunk) => {
+                let good = compute_crc32(&chunk.data) as i64 == chunk.crc;
+                let index = current.expect("a chunk follows its BeginChunk");
+                artifacts[index].1 &= good;
+            }
+        }
+    }
+    AddArtifactsResponse {
+        artifacts: artifacts
+            .into_iter()
+            .map(|(name, good)| ArtifactSummary {
+                name,
+                is_crc_successful: good && !corrupt,
+            })
+            .collect(),
+        ..Default::default()
+    }
+}
+
+/// Every other method is answered with its empty response message.
+#[derive(Clone)]
+struct Unary;
+
+impl UnaryService<Vec<u8>> for Unary {
+    type Response = Vec<u8>;
+    type Future = BoxFuture<Response<Vec<u8>>, Status>;
+
+    fn call(&mut self, _: Request<Vec<u8>>) -> Self::Future {
+        Box::pin(async { Ok(Response::new(Vec::new())) })
+    }
+}
diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index d5adfc8..e552ed2 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -715,7 +715,8 @@ impl SparkSession {
 
     /// Add local files, or `file:` URIs, as artifacts of the session, 
mirroring
     /// `pyspark.sql.SparkSession.addArtifacts(*path)`: a `.jar` is added to 
the classpath and
-    /// a `.class` to the classes; any other file is not supported without a 
flag of
+    /// a `.class` to the classes (a Rust extension: PySpark does not upload 
class files); any
+    /// other file is not supported without a flag of
     /// [`add_artifacts_with`](Self::add_artifacts_with).
     pub fn add_artifacts(&self, paths: &[&str]) -> Result<()> {
         self.add_artifacts_with(paths, false, false, false)
diff --git a/python/tests/test_arrow_handoff.py 
b/python/tests/test_arrow_handoff.py
index 0fd5a5b..2f9f58b 100644
--- a/python/tests/test_arrow_handoff.py
+++ b/python/tests/test_arrow_handoff.py
@@ -1159,22 +1159,31 @@ def 
test_non_arrow_input_does_not_import_pyarrow(protocol, monkeypatch, data):
         assert state["config_requests"] == []
 
 
-def test_add_artifacts_names_each_artifact_as_pyspark(protocol, tmp_path):
+def test_add_artifacts_names_each_artifact_as_pyspark(protocol, tmp_path, 
monkeypatch):
+    import sys
+
     for name in ["a.jar", "A.class", "m.py", "b.zip", "n.txt"]:
         (tmp_path / name).write_bytes(name.encode())
+    monkeypatch.setattr(sys, "path", list(sys.path))
     with _arrow_input_service(protocol, allow_file_artifacts=True) as (spark, 
state):
-        # As PySpark's ArtifactManager names them, by the flags and the file 
extension.
-        spark.addArtifacts(str(tmp_path / "a.jar"), str(tmp_path / "A.class"))
+        # As PySpark's ArtifactManager names them, by the flags and the file 
extension. A class
+        # file is uploaded only with the file flag: PySpark refuses it 
otherwise.
+        spark.addArtifacts(str(tmp_path / "a.jar"))
         spark.addArtifacts(str(tmp_path / "m.py"), pyfile=True)
+        spark.addArtifacts(str(tmp_path / "b.zip"), pyfile=True)
         spark.addArtifacts(str(tmp_path / "b.zip") + "#alias", archive=True)
         spark.addArtifact(str(tmp_path / "n.txt"), file=True)
+        spark.addArtifacts(str(tmp_path / "A.class"), file=True)
         assert state["cache"] == {
             "jars/a.jar": b"a.jar",
-            "classes/A.class": b"A.class",
             "pyfiles/m.py": b"m.py",
+            "pyfiles/b.zip": b"b.zip",
             "archives/b.zip#alias": b"b.zip",
             "files/n.txt": b"n.txt",
+            "files/A.class": b"A.class",
         }
+        # As PySpark's _parse_artifacts, a zip of pyfile is importable by the 
client.
+        assert sys.path[1] == str(tmp_path / "b.zip")
 
 
 # Local relations of at least 32 MiB read the server's local-relation 
settings, and are uploaded
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 723c3ce..29b338c 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9847,6 +9847,81 @@ def 
test_merge_into_writer_mutates_and_returns_self(offline_spark):
     assert m.whenMatched().updateAll() is m
 
 
+# --------------------------------------------------------------------------- 
artifact uploads that stream their chunks, and class files
+
+def test_add_artifacts_refuses_the_paths_pyspark_refuses(tmp_path):
+    # Each error is PySpark's for the path and the flags, raised before any 
upload.
+    import errno
+
+    from pyspark.errors import PySparkRuntimeError, PySparkValueError
+
+    for name in ["A.class", "m.py", "n.txt", "b.tar.gz", "b.zip", "b#c.zip"]:
+        (tmp_path / name).write_bytes(b"x")
+    (tmp_path / "dirx.jar").mkdir()
+    (tmp_path / "sub").mkdir()
+
+    def at(name):
+        return str(tmp_path / name)
+
+    def unsupported(name):
+        return {"operation": f"{at(name)} file format"}
+
+    cases = [
+        # A class file is refused, unless it is a file.
+        (at("A.class"), {}, PySparkRuntimeError, "UNSUPPORTED_OPERATION", 
unsupported("A.class")),
+        (at("A.class"), {"pyfile": True}, PySparkRuntimeError, 
"UNSUPPORTED_OPERATION", unsupported("A.class")),
+        (at("A.class"), {"archive": True}, PySparkRuntimeError, 
"UNSUPPORTED_OPERATION", unsupported("A.class")),
+        # A flag takes only its formats, and a fragment names only an archive.
+        (at("m.py"), {"archive": True}, PySparkRuntimeError, 
"UNSUPPORTED_OPERATION", unsupported("m.py")),
+        (at("n.txt"), {"pyfile": True}, PySparkRuntimeError, 
"UNSUPPORTED_OPERATION", unsupported("n.txt")),
+        (at("b.tar.gz"), {"pyfile": True}, PySparkRuntimeError, 
"UNSUPPORTED_OPERATION", unsupported("b.tar.gz")),
+        (at("b.zip") + "#alias", {}, PySparkRuntimeError, 
"UNSUPPORTED_OPERATION", unsupported("b.zip")),
+        (at("b%23c.zip") + "#alias", {"archive": True}, PySparkValueError, 
"VALUE_ALLOWED",
+         {"arg_name": "artifact path", "disallowed_value": "#"}),
+        # Remote URIs are refused whatever the flags.
+        ("s3://bucket/a.jar", {}, PySparkRuntimeError, 
"UNSUPPORTED_OPERATION", {"operation": "s3 scheme"}),
+        ("hdfs://h/a.jar", {"file": True}, PySparkRuntimeError, 
"UNSUPPORTED_OPERATION",
+         {"operation": "hdfs scheme"}),
+        # A path without a suffix is a format, even when it is a directory.
+        (at("sub"), {}, PySparkRuntimeError, "UNSUPPORTED_OPERATION", 
unsupported("sub")),
+    ]
+    spark = _offline_session()
+    for path, flags, error_type, condition, parameters in cases:
+        with pytest.raises(error_type) as error:
+            spark.addArtifacts(path, **flags)
+        assert error.value.getCondition() == condition, path
+        assert error.value.getMessageParameters() == parameters, path
+    # A file that Python cannot open is the error of open, as PySpark's 
LocalFile raises it.
+    for path, flags, expected in [
+        (at("sub"), {"file": True}, f"[Errno 21] Is a directory: 
'{at('sub')}'"),
+        (at("dirx.jar"), {"archive": True}, f"[Errno 21] Is a directory: 
'{at('dirx.jar')}'"),
+        (at("missing.jar"), {}, f"[Errno 2] No such file or directory: 
'{at('missing.jar')}'"),
+        ("file://" + at("missing.jar"), {"pyfile": True},
+         f"[Errno 2] No such file or directory: '{at('missing.jar')}'"),
+    ]:
+        with pytest.raises(OSError) as error:
+            spark.addArtifacts(path, **flags)
+        assert str(error.value) == expected, path
+        assert error.value.errno in (errno.ENOENT, errno.EISDIR), path
+
+
+def 
test_add_artifacts_puts_a_pyfile_zip_on_the_client_path_before_opening_it(tmp_path,
 monkeypatch):
+    # As PySpark's _parse_artifacts: a zip, egg or jar of pyfile is put on 
sys.path before its
+    # file is opened, and a Python file is not.
+    import sys
+
+    monkeypatch.setattr(sys, "path", list(sys.path))
+    spark = _offline_session()
+    zipped = str(tmp_path / "missing.zip")
+    with pytest.raises(FileNotFoundError):
+        spark.addArtifacts(zipped, pyfile=True)
+    assert sys.path[1] == zipped
+    before = list(sys.path)
+    with pytest.raises(FileNotFoundError):
+        spark.addArtifacts(str(tmp_path / "missing.py"), pyfile=True)
+    assert sys.path == before
+
+
 # --------------------------------------------------------------------------- 
reader schema
 
 def _sent_read_schema(request):


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to