This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 1aa43a9dc docs: Add comments for blocking layer (#3117)
1aa43a9dc is described below
commit 1aa43a9dc7063630ab71e9837f2417b48301e99f
Author: Xuanwo <[email protected]>
AuthorDate: Mon Sep 18 20:31:37 2023 +0800
docs: Add comments for blocking layer (#3117)
* Format code
Signed-off-by: Xuanwo <[email protected]>
* Update opendal.h
Signed-off-by: Xuanwo <[email protected]>
---------
Signed-off-by: Xuanwo <[email protected]>
---
bindings/c/include/opendal.h | 35 ++++++++-
bindings/java/src/lib.rs | 6 +-
bindings/java/src/operator.rs | 3 +-
core/src/layers/blocking.rs | 109 ++++++++++++++++++++++++++-
core/src/services/atomicserver/backend.rs | 9 +--
core/src/types/operator/blocking_operator.rs | 35 ++++++++-
core/tests/behavior/blocking_append.rs | 4 +-
7 files changed, 186 insertions(+), 15 deletions(-)
diff --git a/bindings/c/include/opendal.h b/bindings/c/include/opendal.h
index bc47bf5b5..95c0be52d 100644
--- a/bindings/c/include/opendal.h
+++ b/bindings/c/include/opendal.h
@@ -96,6 +96,8 @@ typedef struct BlockingLister BlockingLister;
*
* # Examples
*
+ * ## Init backends
+ *
* Read more backend init examples in [`services`]
*
* ```
@@ -103,8 +105,8 @@ typedef struct BlockingLister BlockingLister;
* use opendal::services::Fs;
* use opendal::BlockingOperator;
* use opendal::Operator;
- * #[tokio::main]
- * async fn main() -> Result<()> {
+ *
+ * fn main() -> Result<()> {
* // Create fs backend builder.
* let mut builder = Fs::default();
* // Set the root for fs, all operations will happen under this root.
@@ -118,6 +120,35 @@ typedef struct BlockingLister BlockingLister;
* Ok(())
* }
* ```
+ *
+ * ## Init backends with blocking layer
+ *
+ * Some services like s3, gcs doesn't have native blocking supports, we can
use [`layers::BlockingLayer`]
+ * to wrap the async operator to make it blocking.
+ *
+ * ```rust
+ * # use anyhow::Result;
+ * use opendal::layers::BlockingLayer;
+ * use opendal::services::S3;
+ * use opendal::BlockingOperator;
+ * use opendal::Operator;
+ *
+ * #[tokio::main]
+ * async fn main() -> Result<()> {
+ * // Create fs backend builder.
+ * let mut builder = S3::default();
+ * builder.bucket("test");
+ * builder.region("us-east-1");
+ *
+ * // Build an `BlockingOperator` with blocking layer to start operating
the storage.
+ * let _: BlockingOperator = Operator::new(builder)?
+ * .layer(BlockingLayer::create()?)
+ * .finish()
+ * .blocking();
+ *
+ * Ok(())
+ * }
+ * ```
*/
typedef struct BlockingOperator BlockingOperator;
diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs
index e1703b527..353855d4c 100644
--- a/bindings/java/src/lib.rs
+++ b/bindings/java/src/lib.rs
@@ -19,10 +19,10 @@ use std::cell::RefCell;
use std::collections::HashMap;
use std::ffi::c_void;
-use crate::error::Error;
+use jni::objects::JMap;
use jni::objects::JObject;
use jni::objects::JString;
-use jni::objects::{JMap, JValue};
+use jni::objects::JValue;
use jni::sys::jint;
use jni::sys::JNI_VERSION_1_8;
use jni::JNIEnv;
@@ -32,6 +32,8 @@ use opendal::raw::PresignedRequest;
use tokio::runtime::Builder;
use tokio::runtime::Runtime;
+use crate::error::Error;
+
mod blocking_operator;
mod error;
mod metadata;
diff --git a/bindings/java/src/operator.rs b/bindings/java/src/operator.rs
index f18a3cb0a..6787810f0 100644
--- a/bindings/java/src/operator.rs
+++ b/bindings/java/src/operator.rs
@@ -31,10 +31,11 @@ use opendal::raw::PresignedRequest;
use opendal::Operator;
use opendal::Scheme;
+use crate::get_current_env;
use crate::get_global_runtime;
use crate::jmap_to_hashmap;
+use crate::make_presigned_request;
use crate::Result;
-use crate::{get_current_env, make_presigned_request};
#[no_mangle]
pub extern "system" fn Java_org_apache_opendal_Operator_constructor(
diff --git a/core/src/layers/blocking.rs b/core/src/layers/blocking.rs
index 701191490..9fbf7c006 100644
--- a/core/src/layers/blocking.rs
+++ b/core/src/layers/blocking.rs
@@ -29,7 +29,114 @@ use crate::*;
///
/// # Notes
///
-/// Please only enable this layer when the underlying service does not support
blocking.
+/// - Please only enable this layer when the underlying service does not
support blocking.
+///
+/// # Examples
+///
+/// ## In async context
+///
+/// BlockingLayer will use current async context's runtime to handle the async
calls.
+///
+/// ```rust
+/// # use anyhow::Result;
+/// use opendal::layers::BlockingLayer;
+/// use opendal::services::S3;
+/// use opendal::BlockingOperator;
+/// use opendal::Operator;
+///
+/// #[tokio::main]
+/// async fn main() -> Result<()> {
+/// // Create fs backend builder.
+/// let mut builder = S3::default();
+/// builder.bucket("test");
+/// builder.region("us-east-1");
+///
+/// // Build an `BlockingOperator` with blocking layer to start operating
the storage.
+/// let _: BlockingOperator = Operator::new(builder)?
+/// .layer(BlockingLayer::create()?)
+/// .finish()
+/// .blocking();
+///
+/// Ok(())
+/// }
+/// ```
+///
+/// ## In async context with blocking functions
+///
+/// If `BlockingLayer` is called in blocking function, please fetch a
[`tokio::runtime::EnterGuard`]
+/// first. You can use [`Handle::try_current`] first to get the handle and
than call [`Handle::enter`].
+/// This often happens in the case that async function calls blocking function.
+///
+/// ```rust
+/// use opendal::layers::BlockingLayer;
+/// use opendal::services::S3;
+/// use opendal::BlockingOperator;
+/// use opendal::Operator;
+/// use opendal::Result;
+///
+/// #[tokio::main]
+/// async fn main() -> Result<()> {
+/// let _ = blocking_fn()?;
+/// Ok(())
+/// }
+///
+/// fn blocking_fn() -> Result<BlockingOperator> {
+/// // Create fs backend builder.
+/// let mut builder = S3::default();
+/// builder.bucket("test");
+/// builder.region("us-east-1");
+///
+/// let handle = tokio::runtime::Handle::try_current().unwrap();
+/// let _guard = handle.enter();
+/// // Build an `BlockingOperator` with blocking layer to start operating
the storage.
+/// let op: BlockingOperator = Operator::new(builder)?
+/// .layer(BlockingLayer::create()?)
+/// .finish()
+/// .blocking();
+/// Ok(op)
+/// }
+/// ```
+///
+/// ## In blocking context
+///
+/// In a pure blocking context, we can create a runtime and use it to create
the `BlockingLayer`.
+///
+/// > The following code uses a global statically created runtime as an
example, please manage the
+/// runtime on demand.
+///
+/// ```rust
+/// use once_cell::sync::Lazy;
+/// use opendal::layers::BlockingLayer;
+/// use opendal::services::S3;
+/// use opendal::BlockingOperator;
+/// use opendal::Operator;
+/// use opendal::Result;
+///
+/// static RUNTIME: Lazy<tokio::runtime::Runtime> = Lazy::new(|| {
+/// tokio::runtime::Builder::new_multi_thread()
+/// .enable_all()
+/// .build()
+/// .unwrap()
+/// });
+/// ///
+///
+/// fn main() -> Result<()> {
+/// // Create fs backend builder.
+/// let mut builder = S3::default();
+/// builder.bucket("test");
+/// builder.region("us-east-1");
+///
+/// // Fetch the `EnterGuard` from global runtime.
+/// let _guard = RUNTIME.enter();
+/// // Build an `BlockingOperator` with blocking layer to start operating
the storage.
+/// let _: BlockingOperator = Operator::new(builder)?
+/// .layer(BlockingLayer::create()?)
+/// .finish()
+/// .blocking();
+///
+/// Ok(())
+/// }
+/// ```
#[derive(Debug, Clone)]
pub struct BlockingLayer {
handle: Handle,
diff --git a/core/src/services/atomicserver/backend.rs
b/core/src/services/atomicserver/backend.rs
index 24893b270..80146af1d 100644
--- a/core/src/services/atomicserver/backend.rs
+++ b/core/src/services/atomicserver/backend.rs
@@ -15,22 +15,21 @@
// specific language governing permissions and limitations
// under the License.
-use bytes::Bytes;
use std::collections::HashMap;
use std::fmt::Debug;
use std::fmt::Formatter;
use async_trait::async_trait;
+use atomic_lib::agents::Agent;
+use atomic_lib::client::get_authentication_headers;
+use atomic_lib::commit::sign_message;
+use bytes::Bytes;
use http::header::CONTENT_DISPOSITION;
use http::header::CONTENT_TYPE;
use http::Request;
use serde::Deserialize;
use serde::Serialize;
-use atomic_lib::agents::Agent;
-use atomic_lib::client::get_authentication_headers;
-use atomic_lib::commit::sign_message;
-
use crate::raw::adapters::kv;
use crate::raw::new_json_deserialize_error;
use crate::raw::new_json_serialize_error;
diff --git a/core/src/types/operator/blocking_operator.rs
b/core/src/types/operator/blocking_operator.rs
index 1dea4b622..3dfa0d121 100644
--- a/core/src/types/operator/blocking_operator.rs
+++ b/core/src/types/operator/blocking_operator.rs
@@ -30,6 +30,8 @@ use crate::*;
///
/// # Examples
///
+/// ## Init backends
+///
/// Read more backend init examples in [`services`]
///
/// ```
@@ -37,8 +39,8 @@ use crate::*;
/// use opendal::services::Fs;
/// use opendal::BlockingOperator;
/// use opendal::Operator;
-/// #[tokio::main]
-/// async fn main() -> Result<()> {
+///
+/// fn main() -> Result<()> {
/// // Create fs backend builder.
/// let mut builder = Fs::default();
/// // Set the root for fs, all operations will happen under this root.
@@ -52,6 +54,35 @@ use crate::*;
/// Ok(())
/// }
/// ```
+///
+/// ## Init backends with blocking layer
+///
+/// Some services like s3, gcs doesn't have native blocking supports, we can
use [`layers::BlockingLayer`]
+/// to wrap the async operator to make it blocking.
+///
+/// ```rust
+/// # use anyhow::Result;
+/// use opendal::layers::BlockingLayer;
+/// use opendal::services::S3;
+/// use opendal::BlockingOperator;
+/// use opendal::Operator;
+///
+/// #[tokio::main]
+/// async fn main() -> Result<()> {
+/// // Create fs backend builder.
+/// let mut builder = S3::default();
+/// builder.bucket("test");
+/// builder.region("us-east-1");
+///
+/// // Build an `BlockingOperator` with blocking layer to start operating
the storage.
+/// let _: BlockingOperator = Operator::new(builder)?
+/// .layer(BlockingLayer::create()?)
+/// .finish()
+/// .blocking();
+///
+/// Ok(())
+/// }
+/// ```
#[derive(Clone, Debug)]
pub struct BlockingOperator {
accessor: FusedAccessor,
diff --git a/core/tests/behavior/blocking_append.rs
b/core/tests/behavior/blocking_append.rs
index ccb89f786..d402d97e5 100644
--- a/core/tests/behavior/blocking_append.rs
+++ b/core/tests/behavior/blocking_append.rs
@@ -15,13 +15,13 @@
// specific language governing permissions and limitations
// under the License.
+use std::io::BufReader;
+use std::io::Cursor;
use std::vec;
use anyhow::Result;
use sha2::Digest;
use sha2::Sha256;
-use std::io::BufReader;
-use std::io::Cursor;
use crate::*;
