This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 80ae749 [SPARK-61070][CONNECT][PYTHON] Stream the chunks of artifact
uploads, and refuse class files in addArtifacts, as PySpark does
80ae749 is described below
commit 80ae7494c42a5043feb089d1eba03b68468b5606
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 12:29:08 2026 +0900
[SPARK-61070][CONNECT][PYTHON] Stream the chunks of artifact uploads, and
refuse class files in addArtifacts, as PySpark does
### What changes were proposed in this pull request?
- Artifact uploads stream their chunks: `ArtifactFrames`
(`crates/spark-connect-core/src/artifact.rs`) reads one 32 KiB chunk per frame
as the frames are sent, with the same framing as before (a batch for small
artifacts, `BeginChunk` and `Chunk` frames otherwise), and each attempt of the
retry policy reads the file again from the start (`upload_artifact` in
`client.rs`), instead of building every frame up front and cloning them for
each attempt.
- `addArtifacts` of the Python client raises PySpark's `PySparkRuntimeError
[UNSUPPORTED_OPERATION] ... file format is not supported.` for a class file,
opens local paths as PySpark does (so a missing path or a directory raises
`FileNotFoundError` or `IsADirectoryError` with its text), and puts a pyfile
zip, egg or jar on `sys.path` and invalidates the import caches, as PySpark
does (`crates/pyspark-rs/src/session.rs`). The Rust API still uploads class
files, as its docs say.
### Why are the changes needed?
[SPARK-61070](https://issues.apache.org/jira/browse/SPARK-61070): measured
as the peak RSS of `addArtifacts` of a 300 MB jar to an in-process server: 496
MB on the current master, 141 to 174 MB with this change, 173 MB with PySpark
4.3.0. And `spark.addArtifacts("A.class")` uploaded the file where PySpark
raises.
### Does this PR introduce _any_ user-facing change?
Yes. Uploads take less memory, and the Python client's `addArtifacts`
raises PySpark's errors for class files and for missing paths.
### How was this patch tested?
- `crates/spark-connect-core/tests/artifact_uploads.rs`, against a loopback
AddArtifacts service: chunked and batched frames, class file names, a retried
upload sent again from the start, CRC mismatches and permission errors not
retried, and a missing file failing before any attempt; and unit tests of the
frames (lazy reads, framing, read failures, the same frames on each attempt).
- New offline tests in `python/tests/test_dropin_offline.py` for the errors
of `addArtifacts` and the `sys.path` order;
`test_add_artifacts_names_each_artifact_as_pyspark` uploads its class file with
`file=True`, as PySpark requires.
- 112 combinations of paths (jar, py, zip, egg, tar.gz, fragments, `%23`,
`file:` URIs, a directory, missing paths, remote schemes, an empty path) and
flags give the same outcome, exception and message as PySpark 4.3.0's
`_parse_artifacts`.
- The core tests and the build of every integration test of
`apache-spark-connect`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #261 from HyukjinKwon/SPARK-61070-artifact-uploads.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/session.rs | 87 ++--
crates/spark-connect-core/src/artifact.rs | 386 +++++++++++++++--
crates/spark-connect-core/src/client.rs | 75 ++--
.../spark-connect-core/tests/artifact_uploads.rs | 462 +++++++++++++++++++++
crates/spark-connect/src/session.rs | 3 +-
python/tests/test_arrow_handoff.py | 17 +-
python/tests/test_dropin_offline.py | 75 ++++
7 files changed, 997 insertions(+), 108 deletions(-)
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 06ea7cd..b8bcb16 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -6,6 +6,7 @@ use spark_connect::row::{Row, StructValue, Value};
use spark_connect::session::SparkSession;
use spark_connect::types::{DataType, StructField};
use spark_connect::udf::{CommonInlineUserDefinedFunctionExpression,
PythonUDFPayload};
+use spark_connect_core::artifact::{parse_artifact, UnsupportedArtifact};
use crate::catalog::PyCatalog;
use crate::dataframe::PyDataFrame;
@@ -1636,7 +1637,8 @@ impl PySparkSession {
/// Add local files, or `file:` URIs, as artifacts of the session, as
PySpark's
/// `addArtifacts(*path, pyfile=False, archive=False, file=False)`: each
path is named by
/// the flags and its extension, and the errors for a path that is not
supported are
- /// PySpark's (see `spark_connect_core::artifact::parse_artifact`).
+ /// PySpark's (see `spark_connect_core::artifact::parse_artifact`). A
class file is refused,
+ /// as PySpark's ArtifactManager does not upload one.
#[pyo3(name = "addArtifacts", signature = (*path, pyfile=false,
archive=false, file=false))]
fn add_artifacts(
&self,
@@ -1646,8 +1648,6 @@ impl PySparkSession {
archive: bool,
file: bool,
) -> PyResult<()> {
- use spark_connect_core::artifact::{parse_artifact,
UnsupportedArtifact};
-
if [pyfile, archive, file].iter().filter(|flag| **flag).count() > 1 {
return Err(crate::column::raise_pyspark(
py,
@@ -1659,29 +1659,18 @@ impl PySparkSession {
],
));
}
+ // As PySpark's _parse_artifacts, every path is parsed, and a local
file opened, before
+ // any upload.
for p in &path {
- if let Err(unsupported) = parse_artifact(p, pyfile, archive, file)
{
- return Err(match unsupported {
- UnsupportedArtifact::Format(local) =>
crate::column::raise_pyspark(
- py,
- "PySparkRuntimeError",
- "UNSUPPORTED_OPERATION",
- &[("operation", &format!("{local} file format"))],
- ),
- UnsupportedArtifact::Scheme(scheme) =>
crate::column::raise_pyspark(
- py,
- "PySparkRuntimeError",
- "UNSUPPORTED_OPERATION",
- &[("operation", &format!("{scheme} scheme"))],
- ),
- UnsupportedArtifact::HashInArchivePath =>
crate::column::raise_pyspark(
- py,
- "PySparkValueError",
- "VALUE_ALLOWED",
- &[("arg_name", "artifact path"), ("disallowed_value",
"#")],
- ),
- });
+ let (name, local) = match parse_artifact(p, pyfile, archive, file)
{
+ Ok(parsed) => parsed,
+ Err(unsupported) => return Err(unsupported_artifact_error(py,
unsupported)),
+ };
+ if name.starts_with("classes/") {
+ let format =
UnsupportedArtifact::Format(local.to_string_lossy().into_owned());
+ return Err(unsupported_artifact_error(py, format));
}
+ check_local_artifact(py, &name, &local, pyfile)?;
}
let refs: Vec<&str> = path.iter().map(|s| s.as_str()).collect();
py.detach(|| {
@@ -1731,6 +1720,56 @@ impl PySparkSession {
}
}
+/// PySpark's error for a path of `addArtifacts` that it does not upload.
+fn unsupported_artifact_error(py: Python<'_>, unsupported:
UnsupportedArtifact) -> PyErr {
+ match unsupported {
+ UnsupportedArtifact::Format(local) => crate::column::raise_pyspark(
+ py,
+ "PySparkRuntimeError",
+ "UNSUPPORTED_OPERATION",
+ &[("operation", &format!("{local} file format"))],
+ ),
+ UnsupportedArtifact::Scheme(scheme) => crate::column::raise_pyspark(
+ py,
+ "PySparkRuntimeError",
+ "UNSUPPORTED_OPERATION",
+ &[("operation", &format!("{scheme} scheme"))],
+ ),
+ UnsupportedArtifact::HashInArchivePath => crate::column::raise_pyspark(
+ py,
+ "PySparkValueError",
+ "VALUE_ALLOWED",
+ &[("arg_name", "artifact path"), ("disallowed_value", "#")],
+ ),
+ }
+}
+
+/// The client side of parsing a local artifact, as PySpark's
`_parse_artifacts` does it: a
+/// Python file, zip, egg or jar of `pyfile` is importable by the client, and
the file is opened
+/// as PySpark's `LocalFile` opens it, so that a missing file or a directory
raises the error of
+/// `open` before any upload.
+fn check_local_artifact(
+ py: Python<'_>,
+ name: &str,
+ local: &std::path::Path,
+ pyfile: bool,
+) -> PyResult<()> {
+ let local = local.to_string_lossy();
+ let importable = pyfile && name.starts_with("pyfiles/");
+ if importable && !name.ends_with(".py") {
+ py.import("sys")?
+ .getattr("path")?
+ .call_method1("insert", (1, local.as_ref()))?;
+ }
+ py.import("builtins")?
+ .call_method1("open", (local.as_ref(), "rb"))?
+ .call_method0("close")?;
+ if importable {
+ py.import("importlib")?.call_method0("invalidate_caches")?;
+ }
+ Ok(())
+}
+
/// Minimal stand-in for the reference `SparkConnectClient`, returned by
/// `SparkSession.client`. The real client is the Rust transport; this only
surfaces the
/// members that test/utility code (e.g. `ReusedConnectTestCase`) touches.
diff --git a/crates/spark-connect-core/src/artifact.rs
b/crates/spark-connect-core/src/artifact.rs
index 692d5a5..385bea7 100644
--- a/crates/spark-connect-core/src/artifact.rs
+++ b/crates/spark-connect-core/src/artifact.rs
@@ -10,6 +10,7 @@
use bytes::Bytes;
use crc32fast::Hasher;
use std::io::{Cursor, Read};
+use std::sync::{Arc, Mutex, PoisonError};
use spark_connect_proto::{add_artifacts_request, AddArtifactsRequest};
@@ -275,61 +276,158 @@ pub fn build_artifact_request_stream(
Ok(requests)
}
-/// Build requests for a large chunked artifact.
-fn build_chunked_artifact_requests(
+/// The error that ended the frames of an upload early, if one did. The frames
stop at a read
+/// that fails; the upload reports the error once its RPC has returned.
+pub(crate) type FrameFailure = Arc<Mutex<Option<SparkError>>>;
+
+/// Take the error that ended the frames of an upload early, if one did.
+pub(crate) fn take_frame_failure(failure: &FrameFailure) -> Result<()> {
+ match failure
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .take()
+ {
+ Some(error) => Err(error),
+ None => Ok(()),
+ }
+}
+
+/// The frames of an upload of one artifact, produced as the RPC consumes
them, so that a large
+/// artifact is held one chunk at a time rather than as a whole. An attempt
that is retried
+/// builds its frames again, from the start of the artifact.
+///
+/// The frames are those of [`build_artifact_request_stream`] for the
artifact: one batch when it
+/// is at most [`CHUNK_SIZE`] bytes, otherwise a `BeginChunk` and then
`Chunk`s, each a full
+/// chunk but the last.
+pub(crate) struct ArtifactFrames {
session_id: String,
- artifact: NamedArtifact,
-) -> Result<Vec<AddArtifactsRequest>> {
- let mut requests = Vec::new();
- let total_size = artifact.data.size()? as i64;
- let num_chunks = (total_size as usize).div_ceil(CHUNK_SIZE);
+ name: String,
+ size: u64,
+ /// The data, until its last frame is produced or a read fails.
+ reader: Option<Box<dyn Read + Send>>,
+ /// Whether the first chunk of a chunked artifact has been produced.
+ begun: bool,
+ failure: FrameFailure,
+}
- let mut stream = artifact.data.stream()?;
- let mut first_chunk = true;
+impl ArtifactFrames {
+ /// Open the data of `artifact`, so that a data source that cannot be read
is an error before
+ /// any frame is sent.
+ pub(crate) fn new(session_id: String, artifact: &NamedArtifact) ->
Result<Self> {
+ let size = artifact.data.size()?;
+ let reader = artifact.data.stream()?;
+ Ok(Self {
+ session_id,
+ name: artifact.name.clone(),
+ size,
+ reader: Some(reader),
+ begun: false,
+ failure: Arc::default(),
+ })
+ }
+
+ /// The failure that ends these frames early, which the upload checks once
its RPC returns.
+ pub(crate) fn failure(&self) -> FrameFailure {
+ self.failure.clone()
+ }
- let mut buf = vec![0u8; CHUNK_SIZE];
- loop {
- let n = stream.read(&mut buf).map_err(|e| {
+ fn next_payload(&mut self) ->
Result<Option<add_artifacts_request::Payload>> {
+ let Some(mut reader) = self.reader.take() else {
+ return Ok(None);
+ };
+ if self.size <= CHUNK_SIZE as u64 {
+ let mut data = Vec::new();
+ reader
+ .read_to_end(&mut data)
+ .map_err(|e| SparkError::connect_msg(format!("Failed to read
artifact: {}", e)))?;
+ let crc = compute_crc32(&data) as i64;
+ let artifact = add_artifacts_request::SingleChunkArtifact {
+ name: self.name.clone(),
+ data: Some(add_artifacts_request::ArtifactChunk {
+ data: Bytes::from(data),
+ crc,
+ }),
+ };
+ return Ok(Some(add_artifacts_request::Payload::Batch(
+ add_artifacts_request::Batch {
+ artifacts: vec![artifact],
+ },
+ )));
+ }
+ let mut buf = vec![0u8; CHUNK_SIZE];
+ let filled = read_full(reader.as_mut(), &mut buf).map_err(|e| {
SparkError::connect_msg(format!("Failed to read artifact chunk:
{}", e))
})?;
- if n == 0 {
- break;
+ buf.truncate(filled);
+ if filled == 0 {
+ return Ok(None);
}
-
- let chunk_data = buf[..n].to_vec();
- let crc = compute_crc32(&chunk_data) as i64;
-
- if first_chunk {
- // Begin chunked artifact with metadata and initial chunk
- let mut request = AddArtifactsRequest::default();
- request.session_id = session_id.clone();
- request.payload = Some(add_artifacts_request::Payload::BeginChunk(
- add_artifacts_request::BeginChunkedArtifact {
- name: artifact.name.clone(),
- total_bytes: total_size,
- num_chunks: num_chunks as i64,
- initial_chunk: Some(add_artifacts_request::ArtifactChunk {
- data: Bytes::from(chunk_data),
- crc,
- }),
- },
- ));
- requests.push(request);
- first_chunk = false;
+ if filled == CHUNK_SIZE {
+ self.reader = Some(reader);
+ }
+ let crc = compute_crc32(&buf) as i64;
+ let chunk = add_artifacts_request::ArtifactChunk {
+ data: Bytes::from(buf),
+ crc,
+ };
+ if self.begun {
+ Ok(Some(add_artifacts_request::Payload::Chunk(chunk)))
} else {
- // Subsequent chunk
- let mut request = AddArtifactsRequest::default();
- request.session_id = session_id.clone();
- request.payload = Some(add_artifacts_request::Payload::Chunk(
- add_artifacts_request::ArtifactChunk {
- data: Bytes::from(chunk_data),
- crc,
+ self.begun = true;
+ Ok(Some(add_artifacts_request::Payload::BeginChunk(
+ add_artifacts_request::BeginChunkedArtifact {
+ name: self.name.clone(),
+ total_bytes: self.size as i64,
+ num_chunks: self.size.div_ceil(CHUNK_SIZE as u64) as i64,
+ initial_chunk: Some(chunk),
},
- ));
- requests.push(request);
+ )))
+ }
+ }
+}
+
+impl Iterator for ArtifactFrames {
+ type Item = AddArtifactsRequest;
+
+ fn next(&mut self) -> Option<AddArtifactsRequest> {
+ match self.next_payload() {
+ Ok(Some(payload)) => Some(AddArtifactsRequest {
+ session_id: self.session_id.clone(),
+ payload: Some(payload),
+ ..Default::default()
+ }),
+ Ok(None) => None,
+ Err(error) => {
+ self.reader = None;
+ *self.failure.lock().unwrap_or_else(PoisonError::into_inner) =
Some(error);
+ None
+ }
+ }
+ }
+}
+
+/// Fill `buf` from `reader` until it is full or the data ends, and return the
count read.
+fn read_full(reader: &mut dyn Read, buf: &mut [u8]) -> std::io::Result<usize> {
+ let mut filled = 0;
+ while filled < buf.len() {
+ match reader.read(&mut buf[filled..]) {
+ Ok(0) => break,
+ Ok(n) => filled += n,
+ Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
+ Err(e) => return Err(e),
}
}
+ Ok(filled)
+}
+/// Build requests for a large chunked artifact.
+fn build_chunked_artifact_requests(
+ session_id: String,
+ artifact: NamedArtifact,
+) -> Result<Vec<AddArtifactsRequest>> {
+ let mut frames = ArtifactFrames::new(session_id, &artifact)?;
+ let requests: Vec<AddArtifactsRequest> = frames.by_ref().collect();
+ take_frame_failure(&frames.failure())?;
Ok(requests)
}
@@ -399,6 +497,12 @@ fn unquote(text: &str) -> String {
/// - With `file`, any file is `files/<name>`.
/// - Otherwise a `.class` is `classes/<name>`, a `.jar` is `jars/<name>`, and
any other file
/// is not supported.
+///
+/// The `classes/` of a class file is an extension of this crate: PySpark's
`ArtifactManager`
+/// does not upload class files, so the PySpark drop-in refuses them.
+///
+/// The local path is normalized as `pathlib` normalizes it (no trailing or
repeated separators,
+/// and no `.` components), so that it reads the same in an error message.
pub fn parse_artifact(
path_or_uri: &str,
pyfile: bool,
@@ -436,6 +540,7 @@ pub fn parse_artifact(
"file" => std::path::PathBuf::from(unquote(path)),
_ => return Err(UnsupportedArtifact::Scheme(scheme)),
};
+ let local: std::path::PathBuf = local.components().collect();
let local_path = local.to_string_lossy().into_owned();
let name = local
.file_name()
@@ -525,6 +630,12 @@ mod tests {
(false, true, false),
Err(HashInArchivePath),
),
+ // The local path is normalized as pathlib normalizes it, in the
error too.
+ (
+ "/d//x/./n.txt/",
+ (false, false, false),
+ Err(Format("/d/x/n.txt".to_string())),
+ ),
];
for (path, (pyfile, archive, file), expected) in cases {
let parsed = parse_artifact(path, pyfile, archive,
file).map(|(name, _)| name);
@@ -844,4 +955,191 @@ mod tests {
assert_eq!(second_drops.load(Ordering::SeqCst), 1);
assert_eq!(first_drops.load(Ordering::SeqCst), 1);
}
+
+ /// Test data that is produced as it is read, and counts the bytes read of
it.
+ struct Counted {
+ size: u64,
+ read: Arc<std::sync::atomic::AtomicUsize>,
+ /// The read that fails, when the data has one.
+ fails_at_read: Option<usize>,
+ }
+
+ impl ArtifactData for Counted {
+ fn size(&self) -> Result<u64> {
+ Ok(self.size)
+ }
+
+ fn stream(&self) -> Result<Box<dyn Read + Send>> {
+ Ok(Box::new(CountedReader {
+ remaining: self.size as usize,
+ read: self.read.clone(),
+ reads: 0,
+ fails_at_read: self.fails_at_read,
+ }))
+ }
+ }
+
+ struct CountedReader {
+ remaining: usize,
+ read: Arc<std::sync::atomic::AtomicUsize>,
+ reads: usize,
+ fails_at_read: Option<usize>,
+ }
+
+ impl Read for CountedReader {
+ fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+ self.reads += 1;
+ if self.fails_at_read == Some(self.reads) {
+ return Err(std::io::Error::other("disk gone"));
+ }
+ let n = buf.len().min(self.remaining);
+ buf[..n].fill(7);
+ self.remaining -= n;
+ self.read.fetch_add(n, std::sync::atomic::Ordering::SeqCst);
+ Ok(n)
+ }
+ }
+
+ fn counted(size: u64) -> (NamedArtifact,
Arc<std::sync::atomic::AtomicUsize>) {
+ let read = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let data = Counted {
+ size,
+ read: read.clone(),
+ fails_at_read: None,
+ };
+ (
+ NamedArtifact::new("files/n.bin".to_string(), Box::new(data)),
+ read,
+ )
+ }
+
+ #[test]
+ fn artifact_frames_are_read_as_the_rpc_consumes_them() {
+ let size = (CHUNK_SIZE * 10 + 3) as u64;
+ let (artifact, read) = counted(size);
+ let mut frames = ArtifactFrames::new("s".to_string(),
&artifact).unwrap();
+ // Opening the data reads none of it.
+ assert_eq!(read.load(std::sync::atomic::Ordering::SeqCst), 0);
+ frames.next().unwrap();
+ assert_eq!(read.load(std::sync::atomic::Ordering::SeqCst), CHUNK_SIZE);
+ frames.next().unwrap();
+ assert_eq!(
+ read.load(std::sync::atomic::Ordering::SeqCst),
+ 2 * CHUNK_SIZE
+ );
+ assert_eq!(frames.by_ref().count(), 9);
+ assert_eq!(
+ read.load(std::sync::atomic::Ordering::SeqCst),
+ size as usize
+ );
+ assert!(take_frame_failure(&frames.failure()).is_ok());
+ }
+
+ #[test]
+ fn chunked_artifact_frames_begin_and_then_carry_each_chunk_with_its_crc() {
+ let size = CHUNK_SIZE * 2 + 5;
+ let data: Vec<u8> = (0..size).map(|i| (i % 251) as u8).collect();
+ let artifact = NamedArtifact::new(
+ "jars/c.jar".to_string(),
+ Box::new(InMemoryArtifact::new(data.clone())),
+ );
+ let frames: Vec<AddArtifactsRequest> =
ArtifactFrames::new("s".to_string(), &artifact)
+ .unwrap()
+ .collect();
+ assert_eq!(frames.len(), 3);
+ let mut expected = data.chunks(CHUNK_SIZE);
+ for (index, frame) in frames.iter().enumerate() {
+ assert_eq!(frame.session_id, "s");
+ let chunk = expected.next().unwrap();
+ let (payload, crc) = match frame.payload.as_ref().unwrap() {
+ add_artifacts_request::Payload::BeginChunk(begin) => {
+ assert_eq!(index, 0);
+ assert_eq!(begin.name, "jars/c.jar");
+ assert_eq!(begin.total_bytes, size as i64);
+ assert_eq!(begin.num_chunks, 3);
+ let initial = begin.initial_chunk.as_ref().unwrap();
+ (initial.data.to_vec(), initial.crc)
+ }
+ add_artifacts_request::Payload::Chunk(chunk) => {
+ assert_ne!(index, 0);
+ (chunk.data.to_vec(), chunk.crc)
+ }
+ other => panic!("unexpected frame {other:?}"),
+ };
+ assert_eq!(payload, chunk);
+ assert_eq!(crc, compute_crc32(chunk) as i64);
+ }
+ }
+
+ #[test]
+ fn a_small_artifact_frame_is_one_batch() {
+ let artifact = NamedArtifact::new(
+ "classes/A.class".to_string(),
+ Box::new(InMemoryArtifact::new(b"cafebabe".to_vec())),
+ );
+ let frames: Vec<AddArtifactsRequest> =
ArtifactFrames::new("s".to_string(), &artifact)
+ .unwrap()
+ .collect();
+ let [frame] = frames.as_slice() else {
+ panic!("expected one frame, got {}", frames.len());
+ };
+ let Some(add_artifacts_request::Payload::Batch(batch)) =
&frame.payload else {
+ panic!("expected a batch");
+ };
+ let [only] = batch.artifacts.as_slice() else {
+ panic!("expected one artifact");
+ };
+ assert_eq!(only.name, "classes/A.class");
+ let chunk = only.data.as_ref().unwrap();
+ assert_eq!(chunk.data.as_ref(), b"cafebabe");
+ assert_eq!(chunk.crc, compute_crc32(b"cafebabe") as i64);
+ }
+
+ #[test]
+ fn a_failed_read_ends_the_frames_and_is_reported_once() {
+ let read = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let artifact = NamedArtifact::new(
+ "files/n.bin".to_string(),
+ Box::new(Counted {
+ size: (CHUNK_SIZE * 3) as u64,
+ read,
+ // The first chunk is read; the read of the second chunk fails.
+ fails_at_read: Some(2),
+ }),
+ );
+ let mut frames = ArtifactFrames::new("s".to_string(),
&artifact).unwrap();
+ let failure = frames.failure();
+ assert!(frames.next().is_some());
+ assert!(frames.next().is_none());
+ assert!(frames.next().is_none());
+ let error = take_frame_failure(&failure).unwrap_err();
+ assert!(
+ error.to_string().contains("Failed to read artifact chunk"),
+ "{error}"
+ );
+ assert!(take_frame_failure(&failure).is_ok());
+ }
+
+ #[test]
+ fn artifact_frames_are_the_same_on_every_attempt() {
+ let size = CHUNK_SIZE * 2 + 1;
+ let data: Vec<u8> = (0..size).map(|i| (i % 13) as u8).collect();
+ let make = || {
+ NamedArtifact::new(
+ "jars/c.jar".to_string(),
+ Box::new(InMemoryArtifact::new(data.clone())),
+ )
+ };
+ let first = make();
+ let second = make();
+ let a: Vec<AddArtifactsRequest> = ArtifactFrames::new("s".to_string(),
&first)
+ .unwrap()
+ .collect();
+ let b: Vec<AddArtifactsRequest> = ArtifactFrames::new("s".to_string(),
&second)
+ .unwrap()
+ .collect();
+ assert_eq!(a, b);
+ let batch = build_artifact_request_stream("s".to_string(),
vec![make()]).unwrap();
+ assert_eq!(a, batch);
+ }
}
diff --git a/crates/spark-connect-core/src/client.rs
b/crates/spark-connect-core/src/client.rs
index 9e26346..62c1652 100644
--- a/crates/spark-connect-core/src/client.rs
+++ b/crates/spark-connect-core/src/client.rs
@@ -20,8 +20,8 @@ use spark_connect_proto::{
};
use crate::artifact::{
- build_artifact_request_stream, cache_artifact_requests,
check_artifact_upload, FileArtifact,
- NamedArtifact,
+ cache_artifact_requests, check_artifact_upload, take_frame_failure,
ArtifactFrames,
+ FileArtifact, NamedArtifact,
};
use crate::channel::{ChannelBuilder, GRPC_MAX_MESSAGE_LENGTH_DEFAULT};
use crate::error::{Result, SparkError};
@@ -1079,9 +1079,8 @@ impl SparkConnectClient {
for path in paths {
let (name, local) = crate::artifact::parse_artifact(path, pyfile,
archive, file)
.map_err(|e| SparkError::connect_msg(e.to_string()))?;
- let artifacts = vec![NamedArtifact::new(name,
Box::new(FileArtifact::new(local)))];
- let requests =
build_artifact_request_stream(self.session_id.clone(), artifacts)?;
- self.upload_artifact_requests(requests, None).await?;
+ let artifact = NamedArtifact::new(name,
Box::new(FileArtifact::new(local)));
+ self.upload_artifact(&artifact).await?;
}
Ok(())
}
@@ -1092,12 +1091,9 @@ impl SparkConnectClient {
/// Used by `SparkSession::copy_from_local_to_fs`, which names the artifact
/// `forward_to_fs/<dest>` so the server writes it to the target
filesystem path.
pub async fn add_named_artifact(&self, name: &str, local_path: &str) ->
Result<()> {
- let artifacts = vec![NamedArtifact::new(
- name.to_string(),
- Box::new(FileArtifact::new(local_path)),
- )];
- let requests = build_artifact_request_stream(self.session_id.clone(),
artifacts)?;
- self.upload_artifact_requests(requests, None).await
+ let artifact =
+ NamedArtifact::new(name.to_string(),
Box::new(FileArtifact::new(local_path)));
+ self.upload_artifact(&artifact).await
}
/// Upload session-scoped cache blobs, returning SHA-256 hashes in input
order.
@@ -1173,38 +1169,47 @@ impl SparkConnectClient {
}
}
- /// Send artifact frames, retrying a transient failure of the whole upload
under the retry
- /// policy, and validate the response. An attempt sends all the frames
again: a stream is
- /// not resumed. Cache uploads also require an acknowledgement for every
requested
- /// artifact.
- async fn upload_artifact_requests(
- &self,
- requests: Vec<spark_connect_proto::AddArtifactsRequest>,
- expected_cache_names: Option<&std::collections::HashSet<String>>,
- ) -> Result<()> {
- let requests = &requests;
- let response = self
- .with_retry(|| self.add_artifacts_rpc(requests.clone()))
- .await?;
- check_artifact_upload(response, expected_cache_names)
+ /// Upload one artifact, retrying a transient failure of the whole upload
under the retry
+ /// policy, and validate the response. An attempt sends all the frames
again from the start:
+ /// a stream is not resumed. The frames are read as the RPC consumes them,
so an attempt
+ /// holds one chunk of the artifact at a time.
+ async fn upload_artifact(&self, artifact: &NamedArtifact) -> Result<()> {
+ let mut state = RetryPolicyState::new(self.retry_policy.clone());
+ loop {
+ // A data source that cannot be opened is an error before any
frame is sent.
+ let frames = ArtifactFrames::new(self.session_id.clone(),
artifact)?;
+ let failure = frames.failure();
+ let result = self.add_artifacts_rpc(frames).await;
+ // A read that failed ended the frames early, whatever the RPC
answered.
+ take_frame_failure(&failure)?;
+ match result {
+ Ok(response) => return check_artifact_upload(response, None),
+ Err(status) => {
+ if self.retry_policy.can_retry(&status) {
+ if let Some(wait_ms) = state.next_attempt(None) {
+
tokio::time::sleep(std::time::Duration::from_millis(wait_ms)).await;
+ continue;
+ }
+ }
+ return Err(self.error_from_status(status).await);
+ }
+ }
+ }
}
/// The AddArtifacts call of `requests`, once. Stamps user_context and
client_type on
- /// every frame; all upload paths must go through here.
+ /// every frame as it is sent; all upload paths must go through here.
async fn add_artifacts_rpc(
&self,
- requests: impl IntoIterator<Item =
spark_connect_proto::AddArtifactsRequest>,
+ requests: impl Iterator<Item =
spark_connect_proto::AddArtifactsRequest> + Send + 'static,
) -> std::result::Result<spark_connect_proto::AddArtifactsResponse,
tonic::Status> {
let user_context = self.artifact_user_context();
let client_type = self.user_agent.clone();
- let requests: Vec<_> = requests
- .into_iter()
- .map(|mut request| {
- request.user_context = Some(user_context.clone());
- request.client_type = Some(client_type.clone());
- request
- })
- .collect();
+ let requests = requests.map(move |mut request| {
+ request.user_context = Some(user_context.clone());
+ request.client_type = Some(client_type.clone());
+ request
+ });
let deadline = self.rpc_deadlines.add_artifacts;
let mut req =
with_deadline(Request::new(futures::stream::iter(requests)), deadline);
self._attach_metadata(&mut req);
diff --git a/crates/spark-connect-core/tests/artifact_uploads.rs
b/crates/spark-connect-core/tests/artifact_uploads.rs
new file mode 100644
index 0000000..3f0cf9a
--- /dev/null
+++ b/crates/spark-connect-core/tests/artifact_uploads.rs
@@ -0,0 +1,462 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Artifact uploads on the wire: the frames of every AddArtifacts attempt,
its CRCs and its
+//! retries, against a loopback gRPC service that receives and acknowledges
them; no Spark
+//! server required.
+
+use std::collections::VecDeque;
+use std::convert::Infallible;
+use std::sync::{Arc, Mutex};
+use std::task::{Context, Poll};
+use std::time::Duration;
+
+use futures::{stream, StreamExt};
+use prost::Message;
+use spark_connect_core::bytes_codec::BytesCodec;
+use spark_connect_core::{
+ compute_crc32, ChannelBuilder, RetryPolicy, SparkConnectClient, CHUNK_SIZE,
+};
+use spark_connect_proto::add_artifacts_request::Payload;
+use spark_connect_proto::add_artifacts_response::ArtifactSummary;
+use spark_connect_proto::{AddArtifactsRequest, AddArtifactsResponse};
+use tokio::net::TcpListener;
+use tonic::body::Body;
+use tonic::codegen::{http, BoxFuture, Service};
+use tonic::server::{ClientStreamingService, NamedService, UnaryService};
+use tonic::transport::Server;
+use tonic::{Code, Request, Response, Status, Streaming};
+
+const WAIT: Duration = Duration::from_secs(5);
+const SERVICE: &str = "/spark.connect.SparkConnectService/";
+
+/// How the server handles one AddArtifacts attempt.
+#[derive(Clone, Copy)]
+enum Attempt {
+ /// Every frame is received, and every artifact is acknowledged with a
good CRC.
+ Accept,
+ /// The first `n` frames are received, then the call fails with
UNAVAILABLE.
+ FailAfter(usize),
+ /// Every frame is received, but every artifact is acknowledged with a bad
CRC.
+ CorruptCrc,
+ /// The call fails with the code before any frame is received.
+ Refuse(Code),
+}
+
+struct State {
+ attempts: Mutex<VecDeque<Attempt>>,
+ /// The frames that each AddArtifacts attempt received, in order of the
attempts.
+ received: Mutex<Vec<Vec<AddArtifactsRequest>>>,
+}
+
+struct Fixture {
+ state: Arc<State>,
+ server: tokio::task::JoinHandle<()>,
+ address: std::net::SocketAddr,
+}
+
+impl Fixture {
+ async fn start(attempts: impl IntoIterator<Item = Attempt>) -> Self {
+ let state = Arc::new(State {
+ attempts: Mutex::new(attempts.into_iter().collect()),
+ received: Mutex::new(Vec::new()),
+ });
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let address = listener.local_addr().unwrap();
+ let incoming = stream::unfold(listener, |listener| async {
+ Some((listener.accept().await.map(|(socket, _)| socket), listener))
+ });
+ let service = TestService(state.clone());
+ let server = tokio::spawn(async move {
+ Server::builder()
+ .add_service(service)
+ .serve_with_incoming(incoming)
+ .await
+ .unwrap();
+ });
+ Self {
+ state,
+ server,
+ address,
+ }
+ }
+
+ async fn client(&self) -> SparkConnectClient {
+ let builder = ChannelBuilder::parse(&format!("sc://{}",
self.address)).unwrap();
+ let mut client = tokio::time::timeout(WAIT,
SparkConnectClient::connect(&builder))
+ .await
+ .expect("connecting timed out")
+ .unwrap();
+ client.set_retry_policy(RetryPolicy {
+ max_retries: Some(2),
+ initial_backoff_ms: 1,
+ max_backoff_ms: Some(2),
+ ..RetryPolicy::default()
+ });
+ client
+ }
+
+ /// The frames that each AddArtifacts attempt received, in order.
+ fn attempts(&self) -> Vec<Vec<AddArtifactsRequest>> {
+ self.state.received.lock().unwrap().clone()
+ }
+}
+
+impl Drop for Fixture {
+ fn drop(&mut self) {
+ self.server.abort();
+ }
+}
+
+/// Bytes that differ from one position to the next, so that a misplaced chunk
shows.
+fn pattern(len: usize) -> Vec<u8> {
+ (0..len).map(|i| (i % 251) as u8).collect()
+}
+
+/// The data that the frames of one upload carry, which every chunk of a
chunked artifact must
+/// carry in order.
+fn assembled(frames: &[AddArtifactsRequest]) -> Vec<u8> {
+ let mut data = Vec::new();
+ for frame in frames {
+ match frame.payload.as_ref().unwrap() {
+ Payload::Batch(batch) => {
+ for artifact in &batch.artifacts {
+
data.extend_from_slice(&artifact.data.as_ref().unwrap().data);
+ }
+ }
+ Payload::BeginChunk(begin) => {
+
data.extend_from_slice(&begin.initial_chunk.as_ref().unwrap().data)
+ }
+ Payload::Chunk(chunk) => data.extend_from_slice(&chunk.data),
+ }
+ }
+ data
+}
+
+/// The data length of each frame's chunk, in order.
+fn chunk_lengths(frames: &[AddArtifactsRequest]) -> Vec<usize> {
+ frames
+ .iter()
+ .map(|frame| match frame.payload.as_ref().unwrap() {
+ Payload::Batch(batch) =>
batch.artifacts[0].data.as_ref().unwrap().data.len(),
+ Payload::BeginChunk(begin) =>
begin.initial_chunk.as_ref().unwrap().data.len(),
+ Payload::Chunk(chunk) => chunk.data.len(),
+ })
+ .collect()
+}
+
+/// A temporary file of `contents`, which is removed with the directory that
holds it.
+fn file_with(dir: &tempfile::TempDir, name: &str, contents: &[u8]) -> String {
+ let path = dir.path().join(name);
+ std::fs::write(&path, contents).unwrap();
+ path.to_str().unwrap().to_string()
+}
+
+#[tokio::test]
+async fn a_large_file_is_uploaded_as_one_chunked_artifact() {
+ let fixture = Fixture::start([]).await;
+ let client = fixture.client().await;
+ let dir = tempfile::tempdir().unwrap();
+ let data = pattern(3 * CHUNK_SIZE + 5);
+ let path = file_with(&dir, "large.bin", &data);
+
+ client
+ .add_artifacts(&[path.as_str()], false, false, true)
+ .await
+ .unwrap();
+
+ let attempts = fixture.attempts();
+ assert_eq!(attempts.len(), 1);
+ let frames = &attempts[0];
+ // A BeginChunk with the first chunk, then a Chunk for each further chunk,
the last short.
+ assert_eq!(
+ chunk_lengths(frames),
+ vec![CHUNK_SIZE, CHUNK_SIZE, CHUNK_SIZE, 5]
+ );
+ let Some(Payload::BeginChunk(begin)) = &frames[0].payload else {
+ panic!("the first frame must begin the chunked artifact");
+ };
+ assert_eq!(begin.name, "files/large.bin");
+ assert_eq!(begin.total_bytes, data.len() as i64);
+ assert_eq!(begin.num_chunks, 4);
+ for frame in frames.iter().skip(1) {
+ assert!(matches!(frame.payload, Some(Payload::Chunk(_))));
+ }
+ // Every frame carries the CRC of its chunk, and the session and user
context of the client.
+ for frame in frames {
+ assert_eq!(frame.session_id, client.session_id());
+ assert!(frame.user_context.is_some());
+ assert!(frame.client_type.is_some());
+ }
+ let crcs: Vec<i64> = frames
+ .iter()
+ .map(|frame| match frame.payload.as_ref().unwrap() {
+ Payload::BeginChunk(begin) =>
begin.initial_chunk.as_ref().unwrap().crc,
+ Payload::Chunk(chunk) => chunk.crc,
+ Payload::Batch(_) => unreachable!(),
+ })
+ .collect();
+ let expected: Vec<i64> = data
+ .chunks(CHUNK_SIZE)
+ .map(|chunk| compute_crc32(chunk) as i64)
+ .collect();
+ assert_eq!(crcs, expected);
+ assert_eq!(assembled(frames), data);
+}
+
+#[tokio::test]
+async fn a_small_file_is_one_batch_named_by_its_extension() {
+ let fixture = Fixture::start([]).await;
+ let client = fixture.client().await;
+ let dir = tempfile::tempdir().unwrap();
+ let class = file_with(&dir, "A.class", b"cafebabe");
+
+ // A class file is uploaded by the Rust API, as classes/.
+ client
+ .add_artifacts(&[class.as_str()], false, false, false)
+ .await
+ .unwrap();
+
+ let attempts = fixture.attempts();
+ assert_eq!(attempts.len(), 1);
+ let [frame] = attempts[0].as_slice() else {
+ panic!("expected one frame");
+ };
+ let Some(Payload::Batch(batch)) = &frame.payload else {
+ panic!("expected a batch");
+ };
+ let [artifact] = batch.artifacts.as_slice() else {
+ panic!("expected one artifact");
+ };
+ assert_eq!(artifact.name, "classes/A.class");
+ let chunk = artifact.data.as_ref().unwrap();
+ assert_eq!(chunk.data.as_ref(), b"cafebabe");
+ assert_eq!(chunk.crc, compute_crc32(b"cafebabe") as i64);
+}
+
+#[tokio::test]
+async fn a_transient_failure_retries_the_whole_upload_from_its_start() {
+ // The first attempt receives two frames and then fails; the second
receives the upload.
+ let fixture = Fixture::start([Attempt::FailAfter(2),
Attempt::Accept]).await;
+ let client = fixture.client().await;
+ let dir = tempfile::tempdir().unwrap();
+ let data = pattern(3 * CHUNK_SIZE + 5);
+ let path = file_with(&dir, "large.bin", &data);
+
+ client
+ .add_artifacts(&[path.as_str()], false, false, true)
+ .await
+ .unwrap();
+
+ let attempts = fixture.attempts();
+ assert_eq!(attempts.len(), 2);
+ assert_eq!(attempts[0].len(), 2);
+ let whole = &attempts[1];
+ assert!(matches!(whole[0].payload, Some(Payload::BeginChunk(_))));
+ assert_eq!(
+ chunk_lengths(whole),
+ vec![CHUNK_SIZE, CHUNK_SIZE, CHUNK_SIZE, 5]
+ );
+ assert_eq!(assembled(whole), data);
+ // The frames of the retried attempt are those of the first, from the
start.
+ assert_eq!(&whole[..2], attempts[0].as_slice());
+}
+
+#[tokio::test]
+async fn a_crc_mismatch_is_reported_and_not_retried() {
+ let fixture = Fixture::start([Attempt::CorruptCrc]).await;
+ let client = fixture.client().await;
+ let dir = tempfile::tempdir().unwrap();
+ let path = file_with(&dir, "small.txt", b"small");
+
+ let error = client
+ .add_artifacts(&[path.as_str()], false, false, true)
+ .await
+ .unwrap_err();
+
+ assert!(
+ error
+ .to_string()
+ .contains("CRC check failed for artifact: files/small.txt"),
+ "{error}"
+ );
+ assert_eq!(fixture.attempts().len(), 1);
+}
+
+#[tokio::test]
+async fn a_failure_that_is_not_transient_is_not_retried() {
+ let fixture =
Fixture::start([Attempt::Refuse(Code::PermissionDenied)]).await;
+ let client = fixture.client().await;
+ let dir = tempfile::tempdir().unwrap();
+ let path = file_with(&dir, "small.txt", b"small");
+
+ assert!(client
+ .add_artifacts(&[path.as_str()], false, false, true)
+ .await
+ .is_err());
+ // The refused attempt received no frame, and there was no second attempt.
+ assert_eq!(fixture.attempts(), vec![Vec::<AddArtifactsRequest>::new()]);
+}
+
+#[tokio::test]
+async fn a_missing_file_fails_before_any_attempt() {
+ let fixture = Fixture::start([]).await;
+ let client = fixture.client().await;
+ let dir = tempfile::tempdir().unwrap();
+ let missing = dir.path().join("missing.jar");
+
+ assert!(client
+ .add_artifacts(&[missing.to_str().unwrap()], false, false, false)
+ .await
+ .is_err());
+ assert!(fixture.attempts().is_empty());
+}
+
+/// The AddArtifacts service, which receives each attempt's frames and
acknowledges them.
+#[derive(Clone)]
+struct TestService(Arc<State>);
+
+impl NamedService for TestService {
+ const NAME: &'static str = "spark.connect.SparkConnectService";
+}
+
+impl Service<http::Request<Body>> for TestService {
+ type Response = http::Response<Body>;
+ type Error = Infallible;
+ type Future = BoxFuture<Self::Response, Self::Error>;
+
+ fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll<Result<(),
Self::Error>> {
+ Poll::Ready(Ok(()))
+ }
+
+ fn call(&mut self, request: http::Request<Body>) -> Self::Future {
+ let method =
request.uri().path().trim_start_matches(SERVICE).to_string();
+ let state = self.0.clone();
+ Box::pin(async move {
+ let mut grpc = tonic::server::Grpc::new(BytesCodec);
+ Ok(match method.as_str() {
+ "AddArtifacts" => grpc.client_streaming(AddArtifacts(state),
request).await,
+ _ => grpc.unary(Unary, request).await,
+ })
+ })
+ }
+}
+
+struct AddArtifacts(Arc<State>);
+
+impl ClientStreamingService<Vec<u8>> for AddArtifacts {
+ type Response = Vec<u8>;
+ type Future = BoxFuture<Response<Vec<u8>>, Status>;
+
+ fn call(&mut self, request: Request<Streaming<Vec<u8>>>) -> Self::Future {
+ let state = self.0.clone();
+ Box::pin(async move {
+ let attempt = state
+ .attempts
+ .lock()
+ .unwrap()
+ .pop_front()
+ .unwrap_or(Attempt::Accept);
+ if let Attempt::Refuse(code) = attempt {
+ state.received.lock().unwrap().push(Vec::new());
+ return Err(Status::new(code, "refused"));
+ }
+ let limit = match attempt {
+ Attempt::FailAfter(n) => n,
+ _ => usize::MAX,
+ };
+ let mut frames = request.into_inner();
+ let mut received = Vec::new();
+ while received.len() < limit {
+ let Some(bytes) = frames.next().await.transpose()? else {
+ break;
+ };
+
received.push(AddArtifactsRequest::decode(bytes.as_slice()).unwrap());
+ }
+ let complete = received.len() < limit;
+ let response = acknowledge(&received, matches!(attempt,
Attempt::CorruptCrc));
+ state.received.lock().unwrap().push(received);
+ if !complete {
+ return Err(Status::unavailable("connection reset"));
+ }
+ Ok(Response::new(response.encode_to_vec()))
+ })
+ }
+}
+
+/// The response for frames: each artifact in order, with whether every one of
its CRCs is good
+/// (and not corrupted, when `corrupt`).
+fn acknowledge(frames: &[AddArtifactsRequest], corrupt: bool) ->
AddArtifactsResponse {
+ /// Records one CRC of the artifact named `name`, and returns the
artifact's index.
+ fn record(artifacts: &mut Vec<(String, bool)>, name: &str, good: bool) ->
usize {
+ let index = match artifacts.iter().position(|(n, _)| n == name) {
+ Some(index) => index,
+ None => {
+ artifacts.push((name.to_string(), true));
+ artifacts.len() - 1
+ }
+ };
+ artifacts[index].1 &= good;
+ index
+ }
+ let mut artifacts: Vec<(String, bool)> = Vec::new();
+ let mut current: Option<usize> = None;
+ for frame in frames {
+ match frame.payload.as_ref().unwrap() {
+ Payload::Batch(batch) => {
+ for artifact in &batch.artifacts {
+ let chunk = artifact.data.as_ref().unwrap();
+ let good = compute_crc32(&chunk.data) as i64 == chunk.crc;
+ record(&mut artifacts, &artifact.name, good);
+ }
+ }
+ Payload::BeginChunk(begin) => {
+ let chunk = begin.initial_chunk.as_ref().unwrap();
+ let good = compute_crc32(&chunk.data) as i64 == chunk.crc;
+ current = Some(record(&mut artifacts, &begin.name, good));
+ }
+ Payload::Chunk(chunk) => {
+ let good = compute_crc32(&chunk.data) as i64 == chunk.crc;
+ let index = current.expect("a chunk follows its BeginChunk");
+ artifacts[index].1 &= good;
+ }
+ }
+ }
+ AddArtifactsResponse {
+ artifacts: artifacts
+ .into_iter()
+ .map(|(name, good)| ArtifactSummary {
+ name,
+ is_crc_successful: good && !corrupt,
+ })
+ .collect(),
+ ..Default::default()
+ }
+}
+
+/// Every other method is answered with its empty response message.
+#[derive(Clone)]
+struct Unary;
+
+impl UnaryService<Vec<u8>> for Unary {
+ type Response = Vec<u8>;
+ type Future = BoxFuture<Response<Vec<u8>>, Status>;
+
+ fn call(&mut self, _: Request<Vec<u8>>) -> Self::Future {
+ Box::pin(async { Ok(Response::new(Vec::new())) })
+ }
+}
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index d5adfc8..e552ed2 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -715,7 +715,8 @@ impl SparkSession {
/// Add local files, or `file:` URIs, as artifacts of the session,
mirroring
/// `pyspark.sql.SparkSession.addArtifacts(*path)`: a `.jar` is added to
the classpath and
- /// a `.class` to the classes; any other file is not supported without a
flag of
+ /// a `.class` to the classes (a Rust extension: PySpark does not upload
class files); any
+ /// other file is not supported without a flag of
/// [`add_artifacts_with`](Self::add_artifacts_with).
pub fn add_artifacts(&self, paths: &[&str]) -> Result<()> {
self.add_artifacts_with(paths, false, false, false)
diff --git a/python/tests/test_arrow_handoff.py
b/python/tests/test_arrow_handoff.py
index 0fd5a5b..2f9f58b 100644
--- a/python/tests/test_arrow_handoff.py
+++ b/python/tests/test_arrow_handoff.py
@@ -1159,22 +1159,31 @@ def
test_non_arrow_input_does_not_import_pyarrow(protocol, monkeypatch, data):
assert state["config_requests"] == []
-def test_add_artifacts_names_each_artifact_as_pyspark(protocol, tmp_path):
+def test_add_artifacts_names_each_artifact_as_pyspark(protocol, tmp_path,
monkeypatch):
+ import sys
+
for name in ["a.jar", "A.class", "m.py", "b.zip", "n.txt"]:
(tmp_path / name).write_bytes(name.encode())
+ monkeypatch.setattr(sys, "path", list(sys.path))
with _arrow_input_service(protocol, allow_file_artifacts=True) as (spark,
state):
- # As PySpark's ArtifactManager names them, by the flags and the file
extension.
- spark.addArtifacts(str(tmp_path / "a.jar"), str(tmp_path / "A.class"))
+ # As PySpark's ArtifactManager names them, by the flags and the file
extension. A class
+ # file is uploaded only with the file flag: PySpark refuses it
otherwise.
+ spark.addArtifacts(str(tmp_path / "a.jar"))
spark.addArtifacts(str(tmp_path / "m.py"), pyfile=True)
+ spark.addArtifacts(str(tmp_path / "b.zip"), pyfile=True)
spark.addArtifacts(str(tmp_path / "b.zip") + "#alias", archive=True)
spark.addArtifact(str(tmp_path / "n.txt"), file=True)
+ spark.addArtifacts(str(tmp_path / "A.class"), file=True)
assert state["cache"] == {
"jars/a.jar": b"a.jar",
- "classes/A.class": b"A.class",
"pyfiles/m.py": b"m.py",
+ "pyfiles/b.zip": b"b.zip",
"archives/b.zip#alias": b"b.zip",
"files/n.txt": b"n.txt",
+ "files/A.class": b"A.class",
}
+ # As PySpark's _parse_artifacts, a zip of pyfile is importable by the
client.
+ assert sys.path[1] == str(tmp_path / "b.zip")
# Local relations of at least 32 MiB read the server's local-relation
settings, and are uploaded
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 723c3ce..29b338c 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9847,6 +9847,81 @@ def
test_merge_into_writer_mutates_and_returns_self(offline_spark):
assert m.whenMatched().updateAll() is m
+# ---------------------------------------------------------------------------
artifact uploads that stream their chunks, and class files
+
+def test_add_artifacts_refuses_the_paths_pyspark_refuses(tmp_path):
+ # Each error is PySpark's for the path and the flags, raised before any
upload.
+ import errno
+
+ from pyspark.errors import PySparkRuntimeError, PySparkValueError
+
+ for name in ["A.class", "m.py", "n.txt", "b.tar.gz", "b.zip", "b#c.zip"]:
+ (tmp_path / name).write_bytes(b"x")
+ (tmp_path / "dirx.jar").mkdir()
+ (tmp_path / "sub").mkdir()
+
+ def at(name):
+ return str(tmp_path / name)
+
+ def unsupported(name):
+ return {"operation": f"{at(name)} file format"}
+
+ cases = [
+ # A class file is refused, unless it is a file.
+ (at("A.class"), {}, PySparkRuntimeError, "UNSUPPORTED_OPERATION",
unsupported("A.class")),
+ (at("A.class"), {"pyfile": True}, PySparkRuntimeError,
"UNSUPPORTED_OPERATION", unsupported("A.class")),
+ (at("A.class"), {"archive": True}, PySparkRuntimeError,
"UNSUPPORTED_OPERATION", unsupported("A.class")),
+ # A flag takes only its formats, and a fragment names only an archive.
+ (at("m.py"), {"archive": True}, PySparkRuntimeError,
"UNSUPPORTED_OPERATION", unsupported("m.py")),
+ (at("n.txt"), {"pyfile": True}, PySparkRuntimeError,
"UNSUPPORTED_OPERATION", unsupported("n.txt")),
+ (at("b.tar.gz"), {"pyfile": True}, PySparkRuntimeError,
"UNSUPPORTED_OPERATION", unsupported("b.tar.gz")),
+ (at("b.zip") + "#alias", {}, PySparkRuntimeError,
"UNSUPPORTED_OPERATION", unsupported("b.zip")),
+ (at("b%23c.zip") + "#alias", {"archive": True}, PySparkValueError,
"VALUE_ALLOWED",
+ {"arg_name": "artifact path", "disallowed_value": "#"}),
+ # Remote URIs are refused whatever the flags.
+ ("s3://bucket/a.jar", {}, PySparkRuntimeError,
"UNSUPPORTED_OPERATION", {"operation": "s3 scheme"}),
+ ("hdfs://h/a.jar", {"file": True}, PySparkRuntimeError,
"UNSUPPORTED_OPERATION",
+ {"operation": "hdfs scheme"}),
+ # A path without a suffix is a format, even when it is a directory.
+ (at("sub"), {}, PySparkRuntimeError, "UNSUPPORTED_OPERATION",
unsupported("sub")),
+ ]
+ spark = _offline_session()
+ for path, flags, error_type, condition, parameters in cases:
+ with pytest.raises(error_type) as error:
+ spark.addArtifacts(path, **flags)
+ assert error.value.getCondition() == condition, path
+ assert error.value.getMessageParameters() == parameters, path
+ # A file that Python cannot open is the error of open, as PySpark's
LocalFile raises it.
+ for path, flags, expected in [
+ (at("sub"), {"file": True}, f"[Errno 21] Is a directory:
'{at('sub')}'"),
+ (at("dirx.jar"), {"archive": True}, f"[Errno 21] Is a directory:
'{at('dirx.jar')}'"),
+ (at("missing.jar"), {}, f"[Errno 2] No such file or directory:
'{at('missing.jar')}'"),
+ ("file://" + at("missing.jar"), {"pyfile": True},
+ f"[Errno 2] No such file or directory: '{at('missing.jar')}'"),
+ ]:
+ with pytest.raises(OSError) as error:
+ spark.addArtifacts(path, **flags)
+ assert str(error.value) == expected, path
+ assert error.value.errno in (errno.ENOENT, errno.EISDIR), path
+
+
+def
test_add_artifacts_puts_a_pyfile_zip_on_the_client_path_before_opening_it(tmp_path,
monkeypatch):
+ # As PySpark's _parse_artifacts: a zip, egg or jar of pyfile is put on
sys.path before its
+ # file is opened, and a Python file is not.
+ import sys
+
+ monkeypatch.setattr(sys, "path", list(sys.path))
+ spark = _offline_session()
+ zipped = str(tmp_path / "missing.zip")
+ with pytest.raises(FileNotFoundError):
+ spark.addArtifacts(zipped, pyfile=True)
+ assert sys.path[1] == zipped
+ before = list(sys.path)
+ with pytest.raises(FileNotFoundError):
+ spark.addArtifacts(str(tmp_path / "missing.py"), pyfile=True)
+ assert sys.path == before
+
+
# ---------------------------------------------------------------------------
reader schema
def _sent_read_schema(request):
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]