This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 69c8fab74 feat(services/swift): Added user metadata support for swift
service (#5601)
69c8fab74 is described below
commit 69c8fab746bfe9694189a627443e15f61eb02ab8
Author: zhaohaidao <[email protected]>
AuthorDate: Mon Feb 10 20:34:51 2025 +0800
feat(services/swift): Added user metadata support for swift service (#5601)
* feat(services/swift): Added user metadata support for swift service
* fix fmt issue
* fix clippy issue
* fix upper issue
* debug stat 404 issue
* debug stat 404 issue v1
* debug stat 404 issue v2
* debug stat 404 issue: add debug log
* debug stat 404 issue: add debug log
* debug stat 404 issue: try to fix
* debug stat 404 issue: try to fix
* address comments
* address comments
---------
Co-authored-by: zhaohaiyuan <[email protected]>
---
bin/ofs/Cargo.lock | 129 ++++++++++++++++++++++++++++++++++++-
bin/oli/Cargo.lock | 53 ++++++++++++++-
core/src/services/swift/backend.rs | 17 +++--
core/src/services/swift/core.rs | 8 +++
core/src/services/swift/writer.rs | 7 +-
5 files changed, 202 insertions(+), 12 deletions(-)
diff --git a/bin/ofs/Cargo.lock b/bin/ofs/Cargo.lock
index 0ef381431..2dcb5b922 100644
--- a/bin/ofs/Cargo.lock
+++ b/bin/ofs/Cargo.lock
@@ -198,6 +198,25 @@ version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
+[[package]]
+name = "cbindgen"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb"
+dependencies = [
+ "clap",
+ "heck 0.4.1",
+ "indexmap",
+ "log",
+ "proc-macro2",
+ "quote",
+ "serde",
+ "serde_json",
+ "syn",
+ "tempfile",
+ "toml",
+]
+
[[package]]
name = "cc"
version = "1.2.7"
@@ -261,7 +280,7 @@ version = "4.5.24"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c"
dependencies = [
- "heck",
+ "heck 0.5.0",
"proc-macro2",
"quote",
"syn",
@@ -355,6 +374,21 @@ dependencies = [
"libc",
]
+[[package]]
+name = "crc"
+version = "3.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636"
+dependencies = [
+ "crc-catalog",
+]
+
+[[package]]
+name = "crc-catalog"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
+
[[package]]
name = "crc32c"
version = "0.6.8"
@@ -364,6 +398,16 @@ dependencies = [
"rustc_version",
]
+[[package]]
+name = "crc64fast-nvme"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37"
+dependencies = [
+ "cbindgen",
+ "crc",
+]
+
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
@@ -449,6 +493,12 @@ dependencies = [
"regex",
]
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
[[package]]
name = "errno"
version = "0.3.10"
@@ -680,6 +730,18 @@ version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+[[package]]
+name = "hashbrown"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
+
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
[[package]]
name = "heck"
version = "0.5.0"
@@ -968,6 +1030,16 @@ dependencies = [
"icu_properties",
]
+[[package]]
+name = "indexmap"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.15.2",
+]
+
[[package]]
name = "ipnet"
version = "2.10.1"
@@ -1204,6 +1276,7 @@ dependencies = [
"bytes",
"chrono",
"crc32c",
+ "crc64fast-nvme",
"dotenvy",
"futures",
"getrandom 0.2.15",
@@ -1230,7 +1303,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79"
dependencies = [
"dlv-list",
- "hashbrown",
+ "hashbrown 0.14.5",
]
[[package]]
@@ -1665,6 +1738,15 @@ dependencies = [
"serde",
]
+[[package]]
+name = "serde_spanned"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@@ -1973,6 +2055,40 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "toml"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.22.23"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee"
+dependencies = [
+ "indexmap",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
+]
+
[[package]]
name = "tower"
version = "0.5.2"
@@ -2453,6 +2569,15 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+[[package]]
+name = "winnow"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "86e376c75f4f43f44db463cf729e0d3acbf954d13e22c51e26e4c264b4ab545f"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "winsafe"
version = "0.0.19"
diff --git a/bin/oli/Cargo.lock b/bin/oli/Cargo.lock
index 240310251..9b231348e 100644
--- a/bin/oli/Cargo.lock
+++ b/bin/oli/Cargo.lock
@@ -237,6 +237,25 @@ dependencies = [
"cipher",
]
+[[package]]
+name = "cbindgen"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb"
+dependencies = [
+ "clap",
+ "heck 0.4.1",
+ "indexmap",
+ "log",
+ "proc-macro2",
+ "quote",
+ "serde",
+ "serde_json",
+ "syn",
+ "tempfile",
+ "toml",
+]
+
[[package]]
name = "cc"
version = "1.2.7"
@@ -310,7 +329,7 @@ version = "4.5.24"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c"
dependencies = [
- "heck",
+ "heck 0.5.0",
"proc-macro2",
"quote",
"syn",
@@ -382,6 +401,21 @@ dependencies = [
"libc",
]
+[[package]]
+name = "crc"
+version = "3.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636"
+dependencies = [
+ "crc-catalog",
+]
+
+[[package]]
+name = "crc-catalog"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
+
[[package]]
name = "crc32c"
version = "0.6.8"
@@ -391,6 +425,16 @@ dependencies = [
"rustc_version",
]
+[[package]]
+name = "crc64fast-nvme"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37"
+dependencies = [
+ "cbindgen",
+ "crc",
+]
+
[[package]]
name = "crunchy"
version = "0.2.2"
@@ -677,6 +721,12 @@ version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
[[package]]
name = "heck"
version = "0.5.0"
@@ -1286,6 +1336,7 @@ dependencies = [
"bytes",
"chrono",
"crc32c",
+ "crc64fast-nvme",
"futures",
"getrandom",
"http",
diff --git a/core/src/services/swift/backend.rs
b/core/src/services/swift/backend.rs
index 0ae1923d1..bcc0b7bc4 100644
--- a/core/src/services/swift/backend.rs
+++ b/core/src/services/swift/backend.rs
@@ -149,7 +149,6 @@ impl Builder for SwiftBuilder {
));
}
};
- debug!("backend use container: {}", &container);
let token = self.config.token.unwrap_or_default();
@@ -198,11 +197,13 @@ impl Access for SwiftBackend {
stat_has_content_md5: true,
stat_has_last_modified: true,
stat_has_content_disposition: true,
-
+ stat_has_user_metadata: true,
read: true,
write: true,
write_can_empty: true,
+ write_with_user_metadata: true,
+
delete: true,
list: true,
@@ -222,11 +223,15 @@ impl Access for SwiftBackend {
async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> {
let resp = self.core.swift_get_metadata(path).await?;
- let status = resp.status();
-
- match status {
+ match resp.status() {
StatusCode::OK | StatusCode::NO_CONTENT => {
- let meta = parse_into_metadata(path, resp.headers())?;
+ let headers = resp.headers();
+ let mut meta = parse_into_metadata(path, headers)?;
+ let user_meta = parse_prefixed_headers(headers,
"x-object-meta-");
+ if !user_meta.is_empty() {
+ meta.with_user_metadata(user_meta);
+ }
+
Ok(RpStat::new(meta))
}
_ => Err(parse_error(resp)),
diff --git a/core/src/services/swift/core.rs b/core/src/services/swift/core.rs
index f4662d413..a48e49992 100644
--- a/core/src/services/swift/core.rs
+++ b/core/src/services/swift/core.rs
@@ -104,6 +104,7 @@ impl SwiftCore {
&self,
path: &str,
length: u64,
+ args: &OpWrite,
body: Buffer,
) -> Result<Response<Buffer>> {
let p = build_abs_path(&self.root, path);
@@ -116,6 +117,13 @@ impl SwiftCore {
let mut req = Request::put(&url);
+ // Set user metadata headers.
+ if let Some(user_metadata) = args.user_metadata() {
+ for (k, v) in user_metadata {
+ req = req.header(format!("X-Object-Meta-{}", k), v);
+ }
+ }
+
req = req.header("X-Auth-Token", &self.token);
req = req.header(header::CONTENT_LENGTH, length);
diff --git a/core/src/services/swift/writer.rs
b/core/src/services/swift/writer.rs
index a6590ee21..b6a0dbd9e 100644
--- a/core/src/services/swift/writer.rs
+++ b/core/src/services/swift/writer.rs
@@ -26,12 +26,13 @@ use crate::*;
pub struct SwiftWriter {
core: Arc<SwiftCore>,
+ op: OpWrite,
path: String,
}
impl SwiftWriter {
- pub fn new(core: Arc<SwiftCore>, _op: OpWrite, path: String) -> Self {
- SwiftWriter { core, path }
+ pub fn new(core: Arc<SwiftCore>, op: OpWrite, path: String) -> Self {
+ SwiftWriter { core, op, path }
}
}
@@ -39,7 +40,7 @@ impl oio::OneShotWrite for SwiftWriter {
async fn write_once(&self, bs: Buffer) -> Result<()> {
let resp = self
.core
- .swift_create_object(&self.path, bs.len() as u64, bs)
+ .swift_create_object(&self.path, bs.len() as u64, &self.op, bs)
.await?;
let status = resp.status();
