This is an automated email from the ASF dual-hosted git repository. meteorgan pushed a commit to branch object_store_options in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 7c7ff2a44ef71e04ba6a5b93a7f55f9b256372ee Author: meteorgan <[email protected]> AuthorDate: Mon Apr 21 23:09:02 2025 +0800 feat(integrations/object_store): add new_amazon_s3 for OpendalStore --- core/src/types/scheme.rs | 2 +- integrations/object_store/Cargo.toml | 3 +- integrations/object_store/examples/basic.rs | 41 +++++++-------- integrations/object_store/src/amazon_s3.rs | 78 +++++++++++++++++++++++++++++ integrations/object_store/src/lib.rs | 3 ++ 5 files changed, 100 insertions(+), 27 deletions(-) diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs index e120edb4a..20efe9b93 100644 --- a/core/src/types/scheme.rs +++ b/core/src/types/scheme.rs @@ -161,7 +161,7 @@ pub enum Scheme { Lakefs, /// [NebulaGraph](crate::services::NebulaGraph): NebulaGraph Services NebulaGraph, - /// Custom that allow users to implement services outside of OpenDAL. + /// Custom that allow users to implement services outside OpenDAL. /// /// # NOTE /// diff --git a/integrations/object_store/Cargo.toml b/integrations/object_store/Cargo.toml index 63b5dcaa8..2e2ea1cd5 100644 --- a/integrations/object_store/Cargo.toml +++ b/integrations/object_store/Cargo.toml @@ -29,6 +29,7 @@ version = "0.51.1" [features] send_wrapper = ["dep:send_wrapper"] +services-s3 = ["opendal/services-s3", "object_store/aws"] [[test]] harness = false @@ -38,9 +39,7 @@ path = "tests/behavior/main.rs" [dependencies] async-trait = "0.1" bytes = "1" -flagset = "0.4" futures = "0.3" -futures-util = "0.3" object_store = "0.11" opendal = { version = "0.53.0", path = "../../core", default-features = false } pin-project = "1.1" diff --git a/integrations/object_store/examples/basic.rs b/integrations/object_store/examples/basic.rs index 0157d5e4a..eaf4a18e5 100644 --- a/integrations/object_store/examples/basic.rs +++ b/integrations/object_store/examples/basic.rs @@ -1,39 +1,32 @@ -use std::sync::Arc; - use bytes::Bytes; +#[cfg(feature = "services-s3")] +use object_store::aws::AmazonS3Builder; use object_store::path::Path; use object_store::ObjectStore; use object_store_opendal::OpendalStore; -use opendal::services::S3Config; -use opendal::Operator; +#[cfg(feature = "services-s3")] #[tokio::main] async fn main() { - let mut cfg = S3Config::default(); - cfg.access_key_id = Some("my_access_key".to_string()); - cfg.secret_access_key = Some("my_secret_key".to_string()); - cfg.endpoint = Some("my_endpoint".to_string()); - cfg.region = Some("my_region".to_string()); - cfg.bucket = "my_bucket".to_string(); - - // Create a new operator - let operator = Operator::from_config(cfg).unwrap().finish(); - - // Create a new object store - let object_store = Arc::new(OpendalStore::new(operator)); + let builder = AmazonS3Builder::default() + .with_bucket_name("my_bucket") + .with_region("my_region") + .with_endpoint("my_endpoint") + .with_access_key_id("my_access_key") + .with_secret_access_key("my_secret_access_key"); + let s3_store = OpendalStore::new_amazon_s3(builder).unwrap(); let path = Path::from("data/nested/test.txt"); let bytes = Bytes::from_static(b"hello, world! I am nested."); - object_store.put(&path, bytes.clone().into()).await.unwrap(); + s3_store.put(&path, bytes.clone().into()).await.unwrap(); - let content = object_store - .get(&path) - .await - .unwrap() - .bytes() - .await - .unwrap(); + let content = s3_store.get(&path).await.unwrap().bytes().await.unwrap(); assert_eq!(content, bytes); } + +#[cfg(not(feature = "services-s3"))] +fn main() { + println!("The 'services-s3' feature is not enabled."); +} diff --git a/integrations/object_store/src/amazon_s3.rs b/integrations/object_store/src/amazon_s3.rs new file mode 100644 index 000000000..207c3c7e4 --- /dev/null +++ b/integrations/object_store/src/amazon_s3.rs @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::utils::format_object_store_error; +use crate::OpendalStore; +use object_store::aws::{AmazonS3Builder, AmazonS3ConfigKey}; +use opendal::services::S3; +use opendal::Operator; + +impl OpendalStore { + /// Create OpendalStore from object_store Amazon S3 builder. + pub fn new_amazon_s3(builder: AmazonS3Builder) -> object_store::Result<OpendalStore> { + let mut s3 = S3::default(); + if let Some(endpoint) = builder.get_config_value(&AmazonS3ConfigKey::Endpoint) { + s3 = s3.endpoint(endpoint.as_str()); + } + if let Some(region) = builder.get_config_value(&AmazonS3ConfigKey::Region) { + s3 = s3.region(region.as_str()); + } + if let Some(bucket_name) = builder.get_config_value(&AmazonS3ConfigKey::Bucket) { + s3 = s3.bucket(bucket_name.as_str()); + } + if let Some(access_key_id) = builder.get_config_value(&AmazonS3ConfigKey::AccessKeyId) { + s3 = s3.access_key_id(access_key_id.as_str()); + } + if let Some(secret_access_key) = + builder.get_config_value(&AmazonS3ConfigKey::SecretAccessKey) + { + s3 = s3.secret_access_key(secret_access_key.as_str()); + } + if let Some(token) = builder.get_config_value(&AmazonS3ConfigKey::Token) { + s3 = s3.session_token(token.as_str()); + } + if let Some(virtual_hosted_style_request) = + builder.get_config_value(&AmazonS3ConfigKey::VirtualHostedStyleRequest) + { + let r = virtual_hosted_style_request + .parse::<bool>() + .map_err(|err| object_store::Error::Generic { + store: "s3", + source: Box::new(err), + })?; + if r { + s3 = s3.enable_virtual_host_style(); + } + } + if let Some(skip_signature) = builder.get_config_value(&AmazonS3ConfigKey::SkipSignature) { + let r = skip_signature + .parse::<bool>() + .map_err(|err| object_store::Error::Generic { + store: "s3", + source: Box::new(err), + })?; + if r { + s3 = s3.allow_anonymous(); + } + } + + let op = Operator::new(s3) + .map_err(|err| format_object_store_error(err, ""))? + .finish(); + Ok(OpendalStore::new(op)) + } +} diff --git a/integrations/object_store/src/lib.rs b/integrations/object_store/src/lib.rs index f9951b8ce..60a7126cd 100644 --- a/integrations/object_store/src/lib.rs +++ b/integrations/object_store/src/lib.rs @@ -65,6 +65,9 @@ pub use store::OpendalStore; mod utils; +#[cfg(feature = "services-s3")] +mod amazon_s3; + // Make sure `send_wrapper` works as expected #[cfg(all(feature = "send_wrapper", test))] mod assert_send {
