This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch poll-write in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 19aff6cda8d93f952201fd8482e89d885e795a05 Author: Xuanwo <[email protected]> AuthorDate: Mon Sep 11 15:49:15 2023 +0800 refactor Signed-off-by: Xuanwo <[email protected]> --- core/src/raw/oio/write/multipart_upload_write.rs | 6 +++-- core/src/raw/oio/write/range_write.rs | 34 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs index 1b2148cb4..ec23b210c 100644 --- a/core/src/raw/oio/write/multipart_upload_write.rs +++ b/core/src/raw/oio/write/multipart_upload_write.rs @@ -25,9 +25,11 @@ use crate::*; /// MultipartUploadWrite is used to implement [`Write`] based on multipart /// uploads. By implementing MultipartUploadWrite, services don't need to -/// care about the details of buffering and uploading parts. +/// care about the details of uploading parts. /// -/// The layout after adopting [`MultipartUploadWrite`]: +/// # Architecture +/// +/// The architecture after adopting [`MultipartUploadWrite`]: /// /// - Services impl `MultipartUploadWrite` /// - `MultipartUploadWriter` impl `Write` diff --git a/core/src/raw/oio/write/range_write.rs b/core/src/raw/oio/write/range_write.rs index 56276327a..32f832630 100644 --- a/core/src/raw/oio/write/range_write.rs +++ b/core/src/raw/oio/write/range_write.rs @@ -24,10 +24,33 @@ use crate::raw::oio::WriteBuf; use crate::raw::*; use crate::*; +/// RangeWrite is used to implement [`Write`] based on range write. +/// +/// # Services +/// +/// Services like gcs support range write via [GCS Resumable Upload](https://cloud.google.com/storage/docs/resumable-uploads). +/// +/// GCS will support upload content by specifying the range of the file in `CONTENT-RANGE`. +/// +/// Most range based services will have the following limitations: +/// +/// - The size of chunk per upload must be aligned to a certain size. For example, GCS requires +/// to align with 256KiB. +/// - Some services requires to complete the write at the last chunk with the total size. +/// +/// # Architecture +/// +/// The architecture after adopting [`RangeWrite`]: +/// +/// - Services impl `RangeWrite` +/// - `RangeWriter` impl `Write` +/// - Expose `RangeWriter` as `Accessor::Writer` #[async_trait] pub trait RangeWrite: Send + Sync + Unpin + 'static { + /// Initiate range the range write, the returning value is the location. async fn initiate_range(&self) -> Result<String>; + /// write_range will write a range of data. async fn write_range( &self, location: &str, @@ -36,6 +59,7 @@ pub trait RangeWrite: Send + Sync + Unpin + 'static { body: AsyncBody, ) -> Result<()>; + /// complete_range will complete the range write by uploading the last chunk. async fn complete_range( &self, location: &str, @@ -44,9 +68,11 @@ pub trait RangeWrite: Send + Sync + Unpin + 'static { body: AsyncBody, ) -> Result<()>; + /// abort_range will abort the range write by abort all already uploaded data. async fn abort_range(&self, location: &str) -> Result<()>; } +/// RangeWriter will implements [`Write`] based on range write. pub struct RangeWriter<W: RangeWrite> { location: Option<String>, written: u64, @@ -83,6 +109,14 @@ impl<W: RangeWrite> RangeWriter<W> { } } + /// Set the align size. + /// + /// The size is default to 256 KiB. + /// + /// # Note + /// + /// Please don't mix this with the buffer size. Align size is usually the hard + /// limit for the service to accept the chunk. pub fn with_align_size(mut self, size: usize) -> Self { self.align_size = size; self
