This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 523269117 feat(integrations/object_store): add AmazonS3Builder (#5456)
523269117 is described below
commit 523269117cb8b9933617d2e8dcfb5c020ca74835
Author: meteorgan <[email protected]>
AuthorDate: Wed Apr 23 13:25:55 2025 +0800
feat(integrations/object_store): add AmazonS3Builder (#5456)
feat(integrations/object_store): add new_amazon_s3 for OpendalStore
---
core/src/types/scheme.rs | 2 +-
integrations/object_store/Cargo.toml | 3 +-
integrations/object_store/examples/basic.rs | 41 +++++++--------
integrations/object_store/src/amazon_s3.rs | 78 +++++++++++++++++++++++++++++
integrations/object_store/src/lib.rs | 3 ++
5 files changed, 100 insertions(+), 27 deletions(-)
diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs
index e120edb4a..20efe9b93 100644
--- a/core/src/types/scheme.rs
+++ b/core/src/types/scheme.rs
@@ -161,7 +161,7 @@ pub enum Scheme {
Lakefs,
/// [NebulaGraph](crate::services::NebulaGraph): NebulaGraph Services
NebulaGraph,
- /// Custom that allow users to implement services outside of OpenDAL.
+ /// Custom that allow users to implement services outside OpenDAL.
///
/// # NOTE
///
diff --git a/integrations/object_store/Cargo.toml
b/integrations/object_store/Cargo.toml
index 63b5dcaa8..2e2ea1cd5 100644
--- a/integrations/object_store/Cargo.toml
+++ b/integrations/object_store/Cargo.toml
@@ -29,6 +29,7 @@ version = "0.51.1"
[features]
send_wrapper = ["dep:send_wrapper"]
+services-s3 = ["opendal/services-s3", "object_store/aws"]
[[test]]
harness = false
@@ -38,9 +39,7 @@ path = "tests/behavior/main.rs"
[dependencies]
async-trait = "0.1"
bytes = "1"
-flagset = "0.4"
futures = "0.3"
-futures-util = "0.3"
object_store = "0.11"
opendal = { version = "0.53.0", path = "../../core", default-features = false }
pin-project = "1.1"
diff --git a/integrations/object_store/examples/basic.rs
b/integrations/object_store/examples/basic.rs
index 0157d5e4a..eaf4a18e5 100644
--- a/integrations/object_store/examples/basic.rs
+++ b/integrations/object_store/examples/basic.rs
@@ -1,39 +1,32 @@
-use std::sync::Arc;
-
use bytes::Bytes;
+#[cfg(feature = "services-s3")]
+use object_store::aws::AmazonS3Builder;
use object_store::path::Path;
use object_store::ObjectStore;
use object_store_opendal::OpendalStore;
-use opendal::services::S3Config;
-use opendal::Operator;
+#[cfg(feature = "services-s3")]
#[tokio::main]
async fn main() {
- let mut cfg = S3Config::default();
- cfg.access_key_id = Some("my_access_key".to_string());
- cfg.secret_access_key = Some("my_secret_key".to_string());
- cfg.endpoint = Some("my_endpoint".to_string());
- cfg.region = Some("my_region".to_string());
- cfg.bucket = "my_bucket".to_string();
-
- // Create a new operator
- let operator = Operator::from_config(cfg).unwrap().finish();
-
- // Create a new object store
- let object_store = Arc::new(OpendalStore::new(operator));
+ let builder = AmazonS3Builder::default()
+ .with_bucket_name("my_bucket")
+ .with_region("my_region")
+ .with_endpoint("my_endpoint")
+ .with_access_key_id("my_access_key")
+ .with_secret_access_key("my_secret_access_key");
+ let s3_store = OpendalStore::new_amazon_s3(builder).unwrap();
let path = Path::from("data/nested/test.txt");
let bytes = Bytes::from_static(b"hello, world! I am nested.");
- object_store.put(&path, bytes.clone().into()).await.unwrap();
+ s3_store.put(&path, bytes.clone().into()).await.unwrap();
- let content = object_store
- .get(&path)
- .await
- .unwrap()
- .bytes()
- .await
- .unwrap();
+ let content = s3_store.get(&path).await.unwrap().bytes().await.unwrap();
assert_eq!(content, bytes);
}
+
+#[cfg(not(feature = "services-s3"))]
+fn main() {
+ println!("The 'services-s3' feature is not enabled.");
+}
diff --git a/integrations/object_store/src/amazon_s3.rs
b/integrations/object_store/src/amazon_s3.rs
new file mode 100644
index 000000000..207c3c7e4
--- /dev/null
+++ b/integrations/object_store/src/amazon_s3.rs
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::utils::format_object_store_error;
+use crate::OpendalStore;
+use object_store::aws::{AmazonS3Builder, AmazonS3ConfigKey};
+use opendal::services::S3;
+use opendal::Operator;
+
+impl OpendalStore {
+ /// Create OpendalStore from object_store Amazon S3 builder.
+ pub fn new_amazon_s3(builder: AmazonS3Builder) ->
object_store::Result<OpendalStore> {
+ let mut s3 = S3::default();
+ if let Some(endpoint) =
builder.get_config_value(&AmazonS3ConfigKey::Endpoint) {
+ s3 = s3.endpoint(endpoint.as_str());
+ }
+ if let Some(region) =
builder.get_config_value(&AmazonS3ConfigKey::Region) {
+ s3 = s3.region(region.as_str());
+ }
+ if let Some(bucket_name) =
builder.get_config_value(&AmazonS3ConfigKey::Bucket) {
+ s3 = s3.bucket(bucket_name.as_str());
+ }
+ if let Some(access_key_id) =
builder.get_config_value(&AmazonS3ConfigKey::AccessKeyId) {
+ s3 = s3.access_key_id(access_key_id.as_str());
+ }
+ if let Some(secret_access_key) =
+ builder.get_config_value(&AmazonS3ConfigKey::SecretAccessKey)
+ {
+ s3 = s3.secret_access_key(secret_access_key.as_str());
+ }
+ if let Some(token) =
builder.get_config_value(&AmazonS3ConfigKey::Token) {
+ s3 = s3.session_token(token.as_str());
+ }
+ if let Some(virtual_hosted_style_request) =
+
builder.get_config_value(&AmazonS3ConfigKey::VirtualHostedStyleRequest)
+ {
+ let r = virtual_hosted_style_request
+ .parse::<bool>()
+ .map_err(|err| object_store::Error::Generic {
+ store: "s3",
+ source: Box::new(err),
+ })?;
+ if r {
+ s3 = s3.enable_virtual_host_style();
+ }
+ }
+ if let Some(skip_signature) =
builder.get_config_value(&AmazonS3ConfigKey::SkipSignature) {
+ let r = skip_signature
+ .parse::<bool>()
+ .map_err(|err| object_store::Error::Generic {
+ store: "s3",
+ source: Box::new(err),
+ })?;
+ if r {
+ s3 = s3.allow_anonymous();
+ }
+ }
+
+ let op = Operator::new(s3)
+ .map_err(|err| format_object_store_error(err, ""))?
+ .finish();
+ Ok(OpendalStore::new(op))
+ }
+}
diff --git a/integrations/object_store/src/lib.rs
b/integrations/object_store/src/lib.rs
index f9951b8ce..60a7126cd 100644
--- a/integrations/object_store/src/lib.rs
+++ b/integrations/object_store/src/lib.rs
@@ -65,6 +65,9 @@ pub use store::OpendalStore;
mod utils;
+#[cfg(feature = "services-s3")]
+mod amazon_s3;
+
// Make sure `send_wrapper` works as expected
#[cfg(all(feature = "send_wrapper", test))]
mod assert_send {
