This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new e5ef650a4 feat(services/fs): Add user defined metadata support (#7061)
e5ef650a4 is described below
commit e5ef650a4aa20681461d28aa2142bc2bfd8bdd53
Author: zhan7236 <[email protected]>
AuthorDate: Fri Dec 19 14:17:05 2025 +0800
feat(services/fs): Add user defined metadata support (#7061)
- Add xattr crate dependency for Unix platforms
- Enable write_with_user_metadata capability in the fs backend
- Implement get_user_metadata() and set_user_metadata() methods using xattr
- Update FsWriter to store and write user metadata when closing files
- Update fs_stat() to read user metadata from xattr
User metadata is stored as extended attributes with the "user." prefix
on the filesystem, following the standard Linux xattr naming convention.
This allows interoperability with other applications that use xattr.
---
core/Cargo.lock | 11 +++++++
core/services/fs/Cargo.toml | 3 ++
core/services/fs/src/backend.rs | 2 ++
core/services/fs/src/core.rs | 73 ++++++++++++++++++++++++++++++++++++++++-
core/services/fs/src/writer.rs | 24 ++++++++++++++
5 files changed, 112 insertions(+), 1 deletion(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index a4817c0de..7ecfd8997 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -6173,6 +6173,7 @@ dependencies = [
"opendal-core",
"serde",
"tokio",
+ "xattr",
]
[[package]]
@@ -11617,6 +11618,16 @@ dependencies = [
"tap",
]
+[[package]]
+name = "xattr"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
+dependencies = [
+ "libc",
+ "rustix 1.1.2",
+]
+
[[package]]
name = "xml-rs"
version = "0.8.28"
diff --git a/core/services/fs/Cargo.toml b/core/services/fs/Cargo.toml
index 177a5ae2f..ce886b32d 100644
--- a/core/services/fs/Cargo.toml
+++ b/core/services/fs/Cargo.toml
@@ -39,3 +39,6 @@ opendal-core = { path = "../../core", version = "0.55.0",
default-features = fal
] }
serde = { workspace = true, features = ["derive"] }
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
+
+[target.'cfg(unix)'.dependencies]
+xattr = "1"
diff --git a/core/services/fs/src/backend.rs b/core/services/fs/src/backend.rs
index 4a0f38a80..7f2040ef7 100644
--- a/core/services/fs/src/backend.rs
+++ b/core/services/fs/src/backend.rs
@@ -150,6 +150,8 @@ impl Builder for FsBuilder {
write_can_append: true,
write_can_multi: true,
write_with_if_not_exists: true,
+ #[cfg(unix)]
+ write_with_user_metadata: true,
create_dir: true,
delete: true,
diff --git a/core/services/fs/src/core.rs b/core/services/fs/src/core.rs
index 0d512e1fd..b385c8485 100644
--- a/core/services/fs/src/core.rs
+++ b/core/services/fs/src/core.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use std::collections::HashMap;
use std::io::SeekFrom;
use std::path::Path;
use std::path::PathBuf;
@@ -79,11 +80,21 @@ impl FsCore {
} else {
EntryMode::Unknown
};
- let m = Metadata::new(mode)
+ let mut m = Metadata::new(mode)
.with_content_length(meta.len())
.with_last_modified(Timestamp::try_from(
meta.modified().map_err(new_std_io_error)?,
)?);
+
+ // Read user metadata from xattr on Unix systems
+ #[cfg(unix)]
+ {
+ let user_metadata = Self::get_user_metadata(&p)?;
+ if !user_metadata.is_empty() {
+ m = m.with_user_metadata(user_metadata);
+ }
+ }
+
Ok(m)
}
@@ -205,4 +216,64 @@ impl FsCore {
.map_err(new_std_io_error)?;
Ok(())
}
+
+ /// Get user metadata from extended attributes on Unix systems.
+ ///
+ /// Reads xattr in the "user." namespace and strips the prefix, allowing
+ /// interoperability with other applications that set user xattr.
+ #[cfg(unix)]
+ pub fn get_user_metadata(path: &Path) -> Result<HashMap<String, String>> {
+ let mut user_metadata = HashMap::new();
+
+ let attrs = match xattr::list(path) {
+ Ok(attrs) => attrs,
+ Err(e) => {
+ // xattr may not be supported on some filesystems, ignore the
error
+ if e.kind() == std::io::ErrorKind::Unsupported {
+ return Ok(user_metadata);
+ }
+ // Also ignore "operation not supported" errors
(ENOTSUP/EOPNOTSUPP)
+ // which may occur on filesystems that don't support xattr
+ if let Some(os_error) = e.raw_os_error() {
+ // ENOTSUP (95) and EOPNOTSUPP (95) on Linux
+ if os_error == 95 {
+ return Ok(user_metadata);
+ }
+ }
+ return Err(new_std_io_error(e));
+ }
+ };
+
+ for attr in attrs {
+ let attr_name = attr.to_string_lossy();
+ // Only read xattr in the "user." namespace and strip the prefix
+ if let Some(key) = attr_name.strip_prefix(XATTR_USER_PREFIX) {
+ if let Ok(Some(value)) = xattr::get(path, &attr) {
+ if let Ok(v) = String::from_utf8(value) {
+ user_metadata.insert(key.to_string(), v);
+ }
+ }
+ }
+ }
+
+ Ok(user_metadata)
+ }
+
+ /// Set user metadata as extended attributes on Unix systems.
+ ///
+ /// Keys are automatically prefixed with "user." to comply with Linux xattr
+ /// naming requirements. This allows interoperability with other
applications.
+ #[cfg(unix)]
+ pub fn set_user_metadata(path: &Path, user_metadata: &HashMap<String,
String>) -> Result<()> {
+ for (key, value) in user_metadata {
+ let attr_name = format!("{XATTR_USER_PREFIX}{key}");
+ xattr::set(path, &attr_name,
value.as_bytes()).map_err(new_std_io_error)?;
+ }
+ Ok(())
+ }
}
+
+/// Prefix for user xattr namespace on Linux.
+/// Using "user." as the standard namespace for user-defined attributes.
+#[cfg(unix)]
+const XATTR_USER_PREFIX: &str = "user.";
diff --git a/core/services/fs/src/writer.rs b/core/services/fs/src/writer.rs
index 4316ff329..d2d360cb1 100644
--- a/core/services/fs/src/writer.rs
+++ b/core/services/fs/src/writer.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use std::collections::HashMap;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
@@ -34,12 +35,19 @@ pub struct FsWriter {
/// The temp_path is used to specify whether we should move to target_path
after the file has been closed.
temp_path: Option<PathBuf>,
f: tokio::fs::File,
+ /// User metadata to be written to xattr on Unix systems.
+ #[cfg(unix)]
+ user_metadata: Option<HashMap<String, String>>,
}
impl FsWriter {
pub async fn create(core: Arc<FsCore>, path: &str, op: OpWrite) ->
Result<Self> {
let target_path = core.ensure_write_abs_path(&core.root, path).await?;
+ // Store user metadata for later use on Unix systems.
+ #[cfg(unix)]
+ let user_metadata = op.user_metadata().cloned();
+
// Quick path while atomic_write_dir is not set.
if core.atomic_write_dir.is_none() {
let target_file = core.fs_write(&target_path, &op).await?;
@@ -48,6 +56,8 @@ impl FsWriter {
target_path,
temp_path: None,
f: target_file,
+ #[cfg(unix)]
+ user_metadata,
});
}
@@ -75,6 +85,8 @@ impl FsWriter {
target_path,
temp_path,
f,
+ #[cfg(unix)]
+ user_metadata,
})
}
}
@@ -104,6 +116,12 @@ impl oio::Write for FsWriter {
.map_err(new_std_io_error)?;
}
+ // Write user metadata to xattr on Unix systems.
+ #[cfg(unix)]
+ if let Some(ref user_metadata) = self.user_metadata {
+ FsCore::set_user_metadata(&self.target_path, user_metadata)?;
+ }
+
let file_meta = self.f.metadata().await.map_err(new_std_io_error)?;
let meta = Metadata::new(EntryMode::FILE)
.with_content_length(file_meta.len())
@@ -173,6 +191,12 @@ impl oio::PositionWrite for FsWriter {
.map_err(new_std_io_error)?;
}
+ // Write user metadata to xattr on Unix systems.
+ #[cfg(unix)]
+ if let Some(ref user_metadata) = self.user_metadata {
+ FsCore::set_user_metadata(&self.target_path, user_metadata)?;
+ }
+
let file_meta = f.metadata().map_err(new_std_io_error)?;
let mode = if file_meta.is_file() {
EntryMode::FILE
