This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 3651322d7 feat(services/tos): add volcengine TOS support (#7233)
3651322d7 is described below
commit 3651322d7a7f6c62f0bff127705df6e8a00ccb94
Author: Xin Sun <[email protected]>
AuthorDate: Wed Mar 11 00:07:38 2026 +0800
feat(services/tos): add volcengine TOS support (#7233)
Change-Id: I12a490356a0996c4ce4f56614da62ca0bd8849ea
---
.env.example | 5 +
core/Cargo.lock | 10 ++
core/Cargo.toml | 2 +
core/services/tos/Cargo.toml | 36 +++++++
core/services/tos/src/backend.rs | 199 +++++++++++++++++++++++++++++++++++++
core/services/tos/src/config.rs | 210 +++++++++++++++++++++++++++++++++++++++
core/services/tos/src/core.rs | 39 ++++++++
core/services/tos/src/lib.rs | 35 +++++++
core/src/lib.rs | 3 +
9 files changed, 539 insertions(+)
diff --git a/.env.example b/.env.example
index bd6741dc6..df2b81e3c 100644
--- a/.env.example
+++ b/.env.example
@@ -199,3 +199,8 @@ OPENDAL_CLOUDFLARE_KV_ROOT=/path/to/dir
OPENDAL_CLOUDFLARE_KV_API_TOKEN=<api_token>
OPENDAL_CLOUDFLARE_KV_ACCOUNT_ID=<account_id>
OPENDAL_CLOUDFLARE_KV_NAMESPACE_ID=<namespace_id>
+# tos
+OPENDAL_TOS_BUCKET=<bucket>
+OPENDAL_TOS_ENDPOINT=<endpoint>
+OPENDAL_TOS_ACCESS_KEY_ID=<access_key_id>
+OPENDAL_TOS_ACCESS_KEY_SECRET=<access_key_secret>
diff --git a/core/Cargo.lock b/core/Cargo.lock
index 837d3f170..01cfa9ae8 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -5968,6 +5968,7 @@ dependencies = [
"opendal-service-surrealdb",
"opendal-service-swift",
"opendal-service-tikv",
+ "opendal-service-tos",
"opendal-service-upyun",
"opendal-service-vercel-artifacts",
"opendal-service-vercel-blob",
@@ -7088,6 +7089,15 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "opendal-service-tos"
+version = "0.55.0"
+dependencies = [
+ "opendal-core",
+ "serde",
+ "serde_json",
+]
+
[[package]]
name = "opendal-service-upyun"
version = "0.55.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index b64fdbe08..a8f05cb0f 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -182,6 +182,7 @@ services-sqlite = ["dep:opendal-service-sqlite"]
services-surrealdb = ["dep:opendal-service-surrealdb"]
services-swift = ["dep:opendal-service-swift"]
services-tikv = ["dep:opendal-service-tikv"]
+services-tos = ["dep:opendal-service-tos"]
services-upyun = ["dep:opendal-service-upyun"]
services-vercel-artifacts = ["dep:opendal-service-vercel-artifacts"]
services-vercel-blob = ["dep:opendal-service-vercel-blob"]
@@ -289,6 +290,7 @@ opendal-service-sqlite = { path = "services/sqlite",
version = "0.55.0", optiona
opendal-service-surrealdb = { path = "services/surrealdb", version = "0.55.0",
optional = true, default-features = false }
opendal-service-swift = { path = "services/swift", version = "0.55.0",
optional = true, default-features = false }
opendal-service-tikv = { path = "services/tikv", version = "0.55.0", optional
= true, default-features = false }
+opendal-service-tos = { path = "services/tos", version = "0.55.0", optional =
true, default-features = false }
opendal-service-upyun = { path = "services/upyun", version = "0.55.0",
optional = true, default-features = false }
opendal-service-vercel-artifacts = { path = "services/vercel-artifacts",
version = "0.55.0", optional = true, default-features = false }
opendal-service-vercel-blob = { path = "services/vercel-blob", version =
"0.55.0", optional = true, default-features = false }
diff --git a/core/services/tos/Cargo.toml b/core/services/tos/Cargo.toml
new file mode 100644
index 000000000..c11607f65
--- /dev/null
+++ b/core/services/tos/Cargo.toml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Volcengine TOS service implementation for Apache OpenDAL"
+name = "opendal-service-tos"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
diff --git a/core/services/tos/src/backend.rs b/core/services/tos/src/backend.rs
new file mode 100644
index 000000000..3e3963584
--- /dev/null
+++ b/core/services/tos/src/backend.rs
@@ -0,0 +1,199 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use crate::TosConfig;
+use crate::core::TosCore;
+use opendal_core::raw::*;
+use opendal_core::{Builder, Capability, Result};
+
+const TOS_SCHEME: &str = "tos";
+
+/// Builder for Volcengine TOS service.
+#[derive(Default)]
+pub struct TosBuilder {
+ pub(super) config: TosConfig,
+}
+
+impl Debug for TosBuilder {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("TosBuilder")
+ .field("config", &self.config)
+ .finish_non_exhaustive()
+ }
+}
+
+impl TosBuilder {
+ /// Set root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub fn root(mut self, root: &str) -> Self {
+ self.config.root = if root.is_empty() {
+ None
+ } else {
+ Some(root.to_string())
+ };
+ self
+ }
+
+ /// Set bucket name of this backend.
+ pub fn bucket(mut self, bucket: &str) -> Self {
+ self.config.bucket = bucket.to_string();
+ self
+ }
+
+ /// Set endpoint of this backend.
+ ///
+ /// Endpoint must be full uri, e.g.
+ /// - TOS: `https://tos-cn-beijing.volces.com`
+ /// - TOS with region: `https://tos-{region}.volces.com`
+ ///
+ /// If user inputs endpoint without scheme like
"tos-cn-beijing.volces.com", we
+ /// will prepend "https://"; before it.
+ pub fn endpoint(mut self, endpoint: &str) -> Self {
+ if !endpoint.is_empty() {
+ self.config.endpoint =
Some(endpoint.trim_end_matches('/').to_string());
+ }
+ self
+ }
+
+ /// Set region of this backend.
+ ///
+ /// Region represent the signing region of this endpoint.
+ ///
+ /// If region is not set, we will try to load it from environment.
+ /// If still not set, default to `cn-beijing`.
+ pub fn region(mut self, region: &str) -> Self {
+ if !region.is_empty() {
+ self.config.region = Some(region.to_string());
+ }
+ self
+ }
+
+ /// Set access_key_id of this backend.
+ ///
+ /// - If access_key_id is set, we will take user's input first.
+ /// - If not, we will try to load it from environment.
+ pub fn access_key_id(mut self, v: &str) -> Self {
+ self.config.access_key_id = Some(v.to_string());
+ self
+ }
+
+ /// Set secret_access_key of this backend.
+ ///
+ /// - If secret_access_key is set, we will take user's input first.
+ /// - If not, we will try to load it from environment.
+ pub fn secret_access_key(mut self, v: &str) -> Self {
+ self.config.secret_access_key = Some(v.to_string());
+ self
+ }
+
+ /// Set security_token of this backend.
+ pub fn security_token(mut self, v: &str) -> Self {
+ self.config.security_token = Some(v.to_string());
+ self
+ }
+
+ /// Allow anonymous will allow opendal to send request without signing
+ /// when credential is not loaded.
+ pub fn allow_anonymous(mut self, allow: bool) -> Self {
+ self.config.allow_anonymous = allow;
+ self
+ }
+
+ /// Set bucket versioning status for this backend.
+ ///
+ /// If set to true, OpenDAL will support versioned operations like list
with
+ /// versions, read with version, etc.
+ pub fn enable_versioning(mut self, enabled: bool) -> Self {
+ self.config.enable_versioning = enabled;
+ self
+ }
+}
+
+impl Builder for TosBuilder {
+ type Config = TosConfig;
+
+ fn build(self) -> Result<impl Access> {
+ let mut config = self.config;
+ let region = config
+ .region
+ .clone()
+ .unwrap_or_else(|| "cn-beijing".to_string());
+
+ if config.endpoint.is_none() {
+ config.endpoint = Some(format!("https://tos-{}.volces.com";,
region));
+ }
+
+ let endpoint = config.endpoint.clone().unwrap();
+ let bucket = config.bucket.clone();
+ let root = config.root.clone().unwrap_or_else(|| "/".to_string());
+
+ let info = {
+ let am = AccessorInfo::default();
+ am.set_scheme(TOS_SCHEME)
+ .set_root(&root)
+ .set_name(&bucket)
+ .set_native_capability(Capability {
+ read: false,
+
+ write: false,
+
+ delete: false,
+
+ list: false,
+
+ stat: false,
+
+ shared: false,
+
+ ..Default::default()
+ });
+
+ am.into()
+ };
+
+ let core = TosCore {
+ info,
+ bucket,
+ endpoint: endpoint.clone(),
+ root,
+ };
+
+ Ok(TosBackend {
+ core: Arc::new(core),
+ })
+ }
+}
+
+#[derive(Debug)]
+pub struct TosBackend {
+ core: Arc<TosCore>,
+}
+
+impl Access for TosBackend {
+ type Reader = ();
+ type Writer = ();
+ type Lister = ();
+ type Deleter = ();
+
+ fn info(&self) -> Arc<AccessorInfo> {
+ self.core.info.clone()
+ }
+}
diff --git a/core/services/tos/src/config.rs b/core/services/tos/src/config.rs
new file mode 100644
index 000000000..bde4dde03
--- /dev/null
+++ b/core/services/tos/src/config.rs
@@ -0,0 +1,210 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use opendal_core::Configurator;
+use opendal_core::OperatorUri;
+use opendal_core::Result;
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::backend::TosBuilder;
+
+/// Config for Volcengine TOS service.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct TosConfig {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ ///
+ /// default to `/` if not set.
+ pub root: Option<String>,
+ /// bucket name of this backend.
+ ///
+ /// required.
+ pub bucket: String,
+ /// endpoint of this backend.
+ ///
+ /// Endpoint must be full uri, e.g.
+ /// - TOS: `https://tos-cn-beijing.volces.com`
+ /// - TOS with region: `https://tos-{region}.volces.com`
+ ///
+ /// If user inputs endpoint without scheme like
"tos-cn-beijing.volces.com", we
+ /// will prepend "https://"; before it.
+ pub endpoint: Option<String>,
+ /// Region represent the signing region of this endpoint.
+ ///
+ /// Required if endpoint is not provided.
+ ///
+ /// - If region is set, we will take user's input first.
+ /// - If not, we will try to load it from environment.
+ /// - If still not set, default to `cn-beijing`.
+ pub region: Option<String>,
+ /// access_key_id of this backend.
+ ///
+ /// - If access_key_id is set, we will take user's input first.
+ /// - If not, we will try to load it from environment.
+ #[serde(alias = "tos_access_key_id", alias = "volcengine_access_key_id")]
+ pub access_key_id: Option<String>,
+ /// secret_access_key of this backend.
+ ///
+ /// - If secret_access_key is set, we will take user's input first.
+ /// - If not, we will try to load it from environment.
+ #[serde(
+ alias = "tos_secret_access_key",
+ alias = "volcengine_secret_access_key"
+ )]
+ pub secret_access_key: Option<String>,
+ /// security_token of this backend.
+ ///
+ /// This token will expire after sometime, it's recommended to set
security_token
+ /// by hand.
+ #[serde(alias = "tos_security_token", alias = "volcengine_session_token")]
+ pub security_token: Option<String>,
+ /// Disable config load so that opendal will not load config from
+ /// environment.
+ ///
+ /// For examples:
+ /// - envs like `TOS_ACCESS_KEY_ID`
+ pub disable_config_load: bool,
+ /// Allow anonymous will allow opendal to send request without signing
+ /// when credential is not loaded.
+ pub allow_anonymous: bool,
+ /// Enable bucket versioning for this backend.
+ ///
+ /// If set to true, OpenDAL will support versioned operations like list
with
+ /// versions, read with version, etc.
+ pub enable_versioning: bool,
+}
+
+impl Debug for TosConfig {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("TosConfig")
+ .field("root", &self.root)
+ .field("bucket", &self.bucket)
+ .field("endpoint", &self.endpoint)
+ .field("region", &self.region)
+ .finish_non_exhaustive()
+ }
+}
+
+impl Configurator for TosConfig {
+ type Builder = TosBuilder;
+
+ fn from_uri(uri: &OperatorUri) -> Result<Self> {
+ let mut map = uri.options().clone();
+
+ if let Some(name) = uri.name() {
+ map.insert("bucket".to_string(), name.to_string());
+ }
+
+ if let Some(root) = uri.root() {
+ map.insert("root".to_string(), root.to_string());
+ }
+
+ Self::from_iter(map)
+ }
+
+ fn into_builder(self) -> Self::Builder {
+ TosBuilder { config: self }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::iter;
+
+ use super::*;
+ use opendal_core::Configurator;
+ use opendal_core::OperatorUri;
+
+ #[test]
+ fn test_tos_config_original_field_names() {
+ let json = r#"{
+ "bucket": "test-bucket",
+ "access_key_id": "test-key",
+ "secret_access_key": "test-secret",
+ "region": "cn-beijing",
+ "endpoint": "https://tos-cn-beijing.volces.com";
+ }"#;
+
+ let config: TosConfig = serde_json::from_str(json).unwrap();
+ assert_eq!(config.bucket, "test-bucket");
+ assert_eq!(config.access_key_id, Some("test-key".to_string()));
+ assert_eq!(config.secret_access_key, Some("test-secret".to_string()));
+ assert_eq!(config.region, Some("cn-beijing".to_string()));
+ assert_eq!(
+ config.endpoint,
+ Some("https://tos-cn-beijing.volces.com".to_string())
+ );
+ }
+
+ #[test]
+ fn test_tos_config_tos_prefixed_aliases() {
+ let json = r#"{
+ "tos_access_key_id": "test-key",
+ "tos_secret_access_key": "test-secret",
+ "tos_security_token": "test-token"
+ }"#;
+
+ let config: TosConfig = serde_json::from_str(json).unwrap();
+ assert_eq!(config.access_key_id, Some("test-key".to_string()));
+ assert_eq!(config.secret_access_key, Some("test-secret".to_string()));
+ assert_eq!(config.security_token, Some("test-token".to_string()));
+ }
+
+ #[test]
+ fn test_tos_config_volcengine_prefixed_aliases() {
+ let json = r#"{
+ "volcengine_access_key_id": "test-key",
+ "volcengine_secret_access_key": "test-secret",
+ "volcengine_session_token": "test-token"
+ }"#;
+
+ let config: TosConfig = serde_json::from_str(json).unwrap();
+ assert_eq!(config.access_key_id, Some("test-key".to_string()));
+ assert_eq!(config.secret_access_key, Some("test-secret".to_string()));
+ assert_eq!(config.security_token, Some("test-token".to_string()));
+ }
+
+ #[test]
+ fn from_uri_extracts_bucket_and_root() {
+ let uri = OperatorUri::new("tos://example-bucket/path/to/root",
iter::empty()).unwrap();
+ let cfg = TosConfig::from_uri(&uri).unwrap();
+ assert_eq!(cfg.bucket, "example-bucket");
+ assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
+ }
+
+ #[test]
+ fn from_uri_extracts_endpoint() {
+ let uri = OperatorUri::new(
+
"tos://example-bucket/path/to/root?endpoint=https%3A%2F%2Fcustom-tos-endpoint.com",
+ iter::empty(),
+ )
+ .unwrap();
+ let cfg = TosConfig::from_uri(&uri).unwrap();
+ assert_eq!(cfg.bucket, "example-bucket");
+ assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
+ assert_eq!(
+ cfg.endpoint.as_deref(),
+ Some("https://custom-tos-endpoint.com";)
+ );
+ }
+}
diff --git a/core/services/tos/src/core.rs b/core/services/tos/src/core.rs
new file mode 100644
index 000000000..220957f92
--- /dev/null
+++ b/core/services/tos/src/core.rs
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use opendal_core::raw::*;
+
+pub struct TosCore {
+ pub info: Arc<AccessorInfo>,
+
+ pub bucket: String,
+ pub endpoint: String, // full endpoint with scheme, e.g.
https://tos-cn-beijing.volces.com
+ pub root: String,
+}
+
+impl Debug for TosCore {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("TosCore")
+ .field("bucket", &self.bucket)
+ .field("endpoint", &self.endpoint)
+ .field("root", &self.root)
+ .finish_non_exhaustive()
+ }
+}
diff --git a/core/services/tos/src/lib.rs b/core/services/tos/src/lib.rs
new file mode 100644
index 000000000..3216bc3e6
--- /dev/null
+++ b/core/services/tos/src/lib.rs
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#![cfg_attr(docsrs, feature(doc_cfg))]
+//! Volcengine TOS service implementation for Apache OpenDAL.
+#![deny(missing_docs)]
+
+mod backend;
+mod config;
+mod core;
+
+pub use backend::TosBuilder as Tos;
+pub use config::TosConfig;
+
+/// Default scheme for TOS service.
+pub const TOS_SCHEME: &str = "tos";
+
+/// Register this service into the given registry.
+pub fn register_tos_service(registry: &opendal_core::OperatorRegistry) {
+ registry.register::<Tos>(TOS_SCHEME);
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index f3e7a3bc2..302fa88dc 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -205,6 +205,9 @@ fn init_default_registry_inner(registry: &OperatorRegistry)
{
#[cfg(feature = "services-tikv")]
opendal_service_tikv::register_tikv_service(registry);
+ #[cfg(feature = "services-tos")]
+ opendal_service_tos::register_tos_service(registry);
+
#[cfg(feature = "services-upyun")]
opendal_service_upyun::register_upyun_service(registry);
