This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs-object-store.git
The following commit(s) were added to refs/heads/main by this push:
new 135984f Add doc hints pointing large-object uploads at the multipart
API (#839)
135984f is described below
commit 135984fdf91f813c48c47b6288586d9b0e205164
Author: Andrew Lamb <[email protected]>
AuthorDate: Wed Sep 9 09:26:12 2026 -0400
Add doc hints pointing large-object uploads at the multipart API (#839)
---
src/lib.rs | 27 ++++++++++++++++++++++++++-
src/payload.rs | 8 ++++++++
2 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/src/lib.rs b/src/lib.rs
index 7138bdb..1dd749e 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -263,6 +263,9 @@
//!
//! Use the [`ObjectStoreExt::put`] method to atomically write data.
//!
+//! To upload large objects without buffering them entirely in [`PutPayload`],
+//! see [Multipart Upload](#multipart-upload) below.
+//!
//! ```ignore-wasm32
//! # use object_store::local::LocalFileSystem;
//! # use object_store::{ObjectStore, ObjectStoreExt, PutPayload};
@@ -283,7 +286,15 @@
//! # Multipart Upload
//!
//! Use the [`ObjectStoreExt::put_multipart`] /
[`ObjectStore::put_multipart_opts`] method to atomically write a large
-//! amount of data
+//! amount of data in multiple parts, without buffering the entire object in
memory.
+//! [`WriteMultipart`] uploads fixed size parts in parallel as data is written.
+//!
+//! [`BufWriter`](buffered::BufWriter) provides an [`AsyncWrite`]
+//! interface that automatically picks between a single and multipart upload
+//! based on the amount of data written.
+//!
+//! [`AsyncWrite`]: tokio::io::AsyncWrite
+//! [`BufWriter`]: buffered::BufWriter
//!
//! ```ignore-wasm32
//! # use object_store::local::LocalFileSystem;
@@ -911,6 +922,9 @@ pub trait ObjectStore: std::fmt::Display + Send + Sync +
Debug + 'static {
/// The operation is guaranteed to be atomic, it will either successfully
/// write the entirety of `payload` to `location`, or fail. No clients
/// should be able to observe a partially written object
+ ///
+ /// To upload large objects without buffering them entirely in memory, use
+ /// the multipart API: [`ObjectStore::put_multipart_opts`]
async fn put_opts(
&self,
location: &Path,
@@ -923,7 +937,13 @@ pub trait ObjectStore: std::fmt::Display + Send + Sync +
Debug + 'static {
/// Client should prefer [`ObjectStore::put_opts`] for small payloads, as
streaming uploads
/// typically require multiple separate requests. See [`MultipartUpload`]
for more information
///
+ /// See also [`BufWriter`](buffered::BufWriter) for an interface that
+ /// automatically picks between a single and multipart upload based on the
+ /// amount of data written.
+ ///
/// For more advanced multipart uploads see
[`MultipartStore`](multipart::MultipartStore)
+ ///
+ /// [`BufWriter`]: buffered::BufWriter
async fn put_multipart_opts(
&self,
location: &Path,
@@ -1385,6 +1405,11 @@ pub trait ObjectStoreExt: ObjectStore {
/// The operation is guaranteed to be atomic, it will either successfully
/// write the entirety of `payload` to `location`, or fail. No clients
/// should be able to observe a partially written object
+ ///
+ /// Note the entire `payload` is buffered in memory. To upload large
objects
+ /// without buffering them entirely in memory, use the multipart API:
+ /// [`ObjectStoreExt::put_multipart`], [`WriteMultipart`], or
+ /// [`BufWriter`](buffered::BufWriter)
fn put(&self, location: &Path, payload: PutPayload) -> impl Future<Output
= Result<PutResult>>;
/// Perform a multipart upload
diff --git a/src/payload.rs b/src/payload.rs
index 055336b..e7e7646 100644
--- a/src/payload.rs
+++ b/src/payload.rs
@@ -19,6 +19,14 @@ use bytes::Bytes;
use std::sync::Arc;
/// A cheaply cloneable, ordered collection of [`Bytes`]
+///
+/// A [`PutPayload`] is fully materialized in memory. To upload large objects
+/// without buffering them entirely in memory, use the multipart API instead:
+/// [`ObjectStore::put_multipart_opts`], [`WriteMultipart`], or [`BufWriter`]
+///
+/// [`ObjectStore::put_multipart_opts`]: crate::ObjectStore::put_multipart_opts
+/// [`WriteMultipart`]: crate::WriteMultipart
+/// [`BufWriter`]: crate::buffered::BufWriter
#[derive(Debug, Clone)]
pub struct PutPayload(Arc<[Bytes]>);
