This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 3b6300642 feat(services/oss): Add allow anonymous support (#3321)
3b6300642 is described below
commit 3b630064251b421d3087396cdbfd46909ed64f53
Author: Xuanwo <[email protected]>
AuthorDate: Tue Oct 17 19:41:01 2023 +0800
feat(services/oss): Add allow anonymous support (#3321)
Signed-off-by: Xuanwo <[email protected]>
---
.github/workflows/service_test_s3.yml | 3 ---
core/src/docs/upgrade.md | 9 +++++++++
core/src/services/oss/backend.rs | 30 ++++++++++++++++--------------
core/src/services/oss/core.rs | 11 ++++++++++-
core/src/services/s3/backend.rs | 15 ---------------
core/src/services/s3/core.rs | 1 -
6 files changed, 35 insertions(+), 34 deletions(-)
diff --git a/.github/workflows/service_test_s3.yml
b/.github/workflows/service_test_s3.yml
index 423efab0a..a0a6f9433 100644
--- a/.github/workflows/service_test_s3.yml
+++ b/.github/workflows/service_test_s3.yml
@@ -218,9 +218,6 @@ jobs:
# This is the R2's limitation
# Refer to
https://opendal.apache.org/docs/services/s3#compatible-services for more
information
OPENDAL_S3_BATCH_MAX_OPERATIONS: 700
- # This is the R2's limitation
- # Refer to
https://opendal.apache.org/docs/services/s3#compatible-services for more
information
- OPENDAL_S3_ENABLE_EXACT_BUF_WRITE: true
java:
runs-on: ubuntu-latest
diff --git a/core/src/docs/upgrade.md b/core/src/docs/upgrade.md
index 14552a4e4..e061fdc25 100644
--- a/core/src/docs/upgrade.md
+++ b/core/src/docs/upgrade.md
@@ -6,6 +6,15 @@
OpenDAL bumps it's MSRV to 1.67.0.
+### S3 Service Configuration
+
+- The `enable_exact_buf_write` option has been deprecated and is superseded by
`BufferedWriter`, introduced in version 0.40.
+
+### Oss Service Configuration
+
+- The `write_min_size` option has been deprecated and replaced by
`BufferedWriter`, also introduced in version 0.40.
+- A new setting, `allow_anonymous`, has been added. Since v0.41, OSS will now
return an error if credential loading fails. Enabling `allow_anonymous` to
fallback to request without credentials.
+
# Upgrade to v0.41
There is no public API and raw API changes.
diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs
index 4f90ea957..8ce0d65d8 100644
--- a/core/src/services/oss/backend.rs
+++ b/core/src/services/oss/backend.rs
@@ -50,17 +50,16 @@ pub struct OssBuilder {
presign_endpoint: Option<String>,
bucket: String,
- // sse options
+ // OSS features
server_side_encryption: Option<String>,
server_side_encryption_key_id: Option<String>,
+ allow_anonymous: bool,
// authenticate options
access_key_id: Option<String>,
access_key_secret: Option<String>,
http_client: Option<HttpClient>,
- /// the size of each part, and the range is 5MB ~ 5 GB.
- write_min_size: Option<usize>,
/// batch_max_operations
batch_max_operations: Option<usize>,
}
@@ -70,7 +69,8 @@ impl Debug for OssBuilder {
let mut d = f.debug_struct("Builder");
d.field("root", &self.root)
.field("bucket", &self.bucket)
- .field("endpoint", &self.endpoint);
+ .field("endpoint", &self.endpoint)
+ .field("allow_anonymous", &self.allow_anonymous);
d.finish_non_exhaustive()
}
@@ -233,20 +233,19 @@ impl OssBuilder {
self
}
- /// set the minimum size of unsized write, it should be greater than 5 MB.
- /// Reference: [OSS Multipart
upload](https://www.alibabacloud.com/help/en/object-storage-service/latest/multipart-upload-6)
- pub fn write_min_size(&mut self, write_min_size: usize) -> &mut Self {
- self.write_min_size = Some(write_min_size);
-
- self
- }
-
/// Set maximum batch operations of this backend.
pub fn batch_max_operations(&mut self, batch_max_operations: usize) ->
&mut Self {
self.batch_max_operations = Some(batch_max_operations);
self
}
+
+ /// Allow anonymous will allow opendal to send request without signing
+ /// when credential is not loaded.
+ pub fn allow_anonymous(&mut self) -> &mut Self {
+ self.allow_anonymous = true;
+ self
+ }
}
impl Builder for OssBuilder {
@@ -268,10 +267,12 @@ impl Builder for OssBuilder {
.map(|v| builder.server_side_encryption(v));
map.get("server_side_encryption_key_id")
.map(|v| builder.server_side_encryption_key_id(v));
- map.get("write_min_size")
- .map(|v| builder.write_min_size(v.parse::<usize>().unwrap()));
map.get("batch_max_operations")
.map(|v|
builder.batch_max_operations(v.parse::<usize>().unwrap()));
+ map.get("allow_anonymous")
+ .filter(|v| *v == "on" || *v == "true")
+ .map(|_| builder.allow_anonymous());
+
builder
}
@@ -355,6 +356,7 @@ impl Builder for OssBuilder {
endpoint,
host,
presign_endpoint,
+ allow_anonymous: self.allow_anonymous,
signer,
loader,
client,
diff --git a/core/src/services/oss/core.rs b/core/src/services/oss/core.rs
index 25e870f99..28a4936a5 100644
--- a/core/src/services/oss/core.rs
+++ b/core/src/services/oss/core.rs
@@ -57,6 +57,7 @@ pub struct OssCore {
pub host: String,
pub endpoint: String,
pub presign_endpoint: String,
+ pub allow_anonymous: bool,
pub server_side_encryption: Option<HeaderValue>,
pub server_side_encryption_key_id: Option<HeaderValue>,
@@ -88,8 +89,16 @@ impl OssCore {
if let Some(cred) = cred {
Ok(Some(cred))
- } else {
+ } else if self.allow_anonymous {
+ // If allow_anonymous has been set, we will not sign the request.
Ok(None)
+ } else {
+ // Mark this error as temporary since it could be caused by Aliyun
STS.
+ Err(Error::new(
+ ErrorKind::PermissionDenied,
+ "no valid credential found, please check configuration or try
again",
+ )
+ .set_temporary())
}
}
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index 060106851..0216b1936 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -91,7 +91,6 @@ pub struct S3Builder {
default_storage_class: Option<String>,
enable_virtual_host_style: bool,
batch_max_operations: Option<usize>,
- enable_exact_buf_write: bool,
http_client: Option<HttpClient>,
}
@@ -517,16 +516,6 @@ impl S3Builder {
self
}
- /// Enable exact buf write so that opendal will write data with exact size.
- ///
- /// This option is used for services like R2 which requires all parts must
be the same size
- /// except the last part.
- pub fn enable_exact_buf_write(&mut self) -> &mut Self {
- self.enable_exact_buf_write = true;
-
- self
- }
-
/// Detect region of S3 bucket.
///
/// # Args
@@ -686,9 +675,6 @@ impl Builder for S3Builder {
.map(|v: &String| builder.default_storage_class(v));
map.get("batch_max_operations")
.map(|v| builder.batch_max_operations(v.parse().expect("input must
be a number")));
- map.get("enable_exact_buf_write")
- .filter(|v| *v == "on" || *v == "true")
- .map(|_| builder.enable_exact_buf_write());
builder
}
@@ -868,7 +854,6 @@ impl Builder for S3Builder {
server_side_encryption_customer_key_md5,
default_storage_class,
allow_anonymous: self.allow_anonymous,
- enable_exact_buf_write: self.enable_exact_buf_write,
signer,
loader,
client,
diff --git a/core/src/services/s3/core.rs b/core/src/services/s3/core.rs
index 3c809246c..1021399e4 100644
--- a/core/src/services/s3/core.rs
+++ b/core/src/services/s3/core.rs
@@ -79,7 +79,6 @@ pub struct S3Core {
pub server_side_encryption_customer_key_md5: Option<HeaderValue>,
pub default_storage_class: Option<HeaderValue>,
pub allow_anonymous: bool,
- pub enable_exact_buf_write: bool,
pub signer: AwsV4Signer,
pub loader: Box<dyn AwsCredentialLoad>,
