This is an automated email from the ASF dual-hosted git repository.
fanng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 3417fc33f9 [#6357] feat (gvfs-fuse): Passing file type argument in the
Filesystem::stat() to improve the performance of open_dal_filesystem (#6358)
3417fc33f9 is described below
commit 3417fc33f9515f1e02ef3d37559246d35515d031
Author: Yuhui <[email protected]>
AuthorDate: Fri Feb 7 16:24:10 2025 +0800
[#6357] feat (gvfs-fuse): Passing file type argument in the
Filesystem::stat() to improve the performance of open_dal_filesystem (#6358)
### What changes were proposed in this pull request?
Passing file type argument in the Filesystem::stat() to improve the
performance of open_dal_filesystem
### Why are the changes needed?
Fix: #6357
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
UI
---------
Co-authored-by: Qiming Teng <[email protected]>
---
.../filesystem-fuse/src/default_raw_filesystem.rs | 33 ++++++++++++++++------
clients/filesystem-fuse/src/filesystem.rs | 11 +++++---
.../src/gravitino_fileset_filesystem.rs | 13 +++++++--
clients/filesystem-fuse/src/memory_filesystem.rs | 8 ++++--
clients/filesystem-fuse/src/open_dal_filesystem.rs | 25 +++++++++++++---
clients/filesystem-fuse/src/s3_filesystem.rs | 9 ++++--
clients/filesystem-fuse/tests/bin/gvfs_fuse.sh | 2 --
.../filesystem-fuse/tests/bin/run_fuse_testers.sh | 2 ++
.../filesystem-fuse/tests/bin/run_s3fs_testers.sh | 1 -
9 files changed, 78 insertions(+), 26 deletions(-)
diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs
b/clients/filesystem-fuse/src/default_raw_filesystem.rs
index d1d8e7605d..08b480e43e 100644
--- a/clients/filesystem-fuse/src/default_raw_filesystem.rs
+++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs
@@ -26,6 +26,7 @@ use crate::opened_file::{FileHandle, OpenFileFlags,
OpenedFile};
use crate::opened_file_manager::OpenedFileManager;
use async_trait::async_trait;
use bytes::Bytes;
+use fuse3::FileType::{Directory, RegularFile};
use fuse3::{Errno, FileType};
use std::collections::HashMap;
use std::ffi::OsStr;
@@ -86,7 +87,12 @@ impl<T: PathFileSystem> DefaultRawFileSystem<T> {
None => {
// allocate new file id
file_stat.set_file_id(parent_file_id, self.next_file_id());
- file_manager.insert(file_stat.parent_file_id,
file_stat.file_id, &file_stat.path);
+ file_manager.insert(
+ file_stat.parent_file_id,
+ file_stat.file_id,
+ &file_stat.path,
+ file_stat.kind,
+ );
}
Some(file) => {
// use the exist file id
@@ -130,9 +136,15 @@ impl<T: PathFileSystem> DefaultRawFileSystem<T> {
file_manager.remove(path);
}
- async fn insert_file_entry_locked(&self, parent_file_id: u64, file_id:
u64, path: &Path) {
+ async fn insert_file_entry_locked(
+ &self,
+ parent_file_id: u64,
+ file_id: u64,
+ path: &Path,
+ kind: FileType,
+ ) {
let mut file_manager = self.file_entry_manager.write().await;
- file_manager.insert(parent_file_id, file_id, path);
+ file_manager.insert(parent_file_id, file_id, path, kind);
}
fn get_meta_file_stat(&self) -> FileStat {
@@ -159,6 +171,7 @@ impl<T: PathFileSystem> RawFileSystem for
DefaultRawFileSystem<T> {
ROOT_DIR_PARENT_FILE_ID,
ROOT_DIR_FILE_ID,
Path::new(ROOT_DIR_PATH),
+ Directory,
)
.await;
@@ -166,6 +179,7 @@ impl<T: PathFileSystem> RawFileSystem for
DefaultRawFileSystem<T> {
ROOT_DIR_FILE_ID,
FS_META_FILE_ID,
Path::new(FS_META_FILE_PATH),
+ RegularFile,
)
.await;
self.fs.init().await
@@ -197,7 +211,7 @@ impl<T: PathFileSystem> RawFileSystem for
DefaultRawFileSystem<T> {
}
let file_entry = self.get_file_entry(file_id).await?;
- let mut file_stat = self.fs.stat(&file_entry.path).await?;
+ let mut file_stat = self.fs.stat(&file_entry.path,
file_entry.kind).await?;
file_stat.set_file_id(file_entry.parent_file_id, file_entry.file_id);
Ok(file_stat)
}
@@ -209,7 +223,7 @@ impl<T: PathFileSystem> RawFileSystem for
DefaultRawFileSystem<T> {
let parent_file_entry = self.get_file_entry(parent_file_id).await?;
let path = parent_file_entry.path.join(name);
- let mut file_stat = self.fs.stat(&path).await?;
+ let mut file_stat = self.fs.lookup(&path).await?;
// fill the file id to file stat
self.resolve_file_id_to_filestat(&mut file_stat, parent_file_id)
.await;
@@ -270,6 +284,7 @@ impl<T: PathFileSystem> RawFileSystem for
DefaultRawFileSystem<T> {
parent_file_id,
file_without_id.file_stat.file_id,
&file_without_id.file_stat.path,
+ RegularFile,
)
.await;
@@ -287,7 +302,7 @@ impl<T: PathFileSystem> RawFileSystem for
DefaultRawFileSystem<T> {
filestat.set_file_id(parent_file_id, self.next_file_id());
// insert the new file to file entry manager
- self.insert_file_entry_locked(parent_file_id, filestat.file_id,
&filestat.path)
+ self.insert_file_entry_locked(parent_file_id, filestat.file_id,
&filestat.path, Directory)
.await;
Ok(filestat.file_id)
}
@@ -401,6 +416,7 @@ struct FileEntry {
file_id: u64,
parent_file_id: u64,
path: PathBuf,
+ kind: FileType,
}
/// FileEntryManager is manage all the file entries in memory. it is used
manger the file relationship and name mapping.
@@ -428,11 +444,12 @@ impl FileEntryManager {
self.file_path_map.get(path).cloned()
}
- fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &Path) {
+ fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &Path, kind:
FileType) {
let file_entry = FileEntry {
file_id,
parent_file_id,
path: path.into(),
+ kind: kind,
};
self.file_id_map.insert(file_id, file_entry.clone());
self.file_path_map.insert(path.into(), file_entry);
@@ -452,7 +469,7 @@ mod tests {
#[test]
fn test_file_entry_manager() {
let mut manager = FileEntryManager::new();
- manager.insert(1, 2, Path::new("a/b"));
+ manager.insert(1, 2, Path::new("a/b"), Directory);
let file = manager.get_file_entry_by_id(2).unwrap();
assert_eq!(file.file_id, 2);
assert_eq!(file.parent_file_id, 1);
diff --git a/clients/filesystem-fuse/src/filesystem.rs
b/clients/filesystem-fuse/src/filesystem.rs
index c0c27a5fbe..a260f2053a 100644
--- a/clients/filesystem-fuse/src/filesystem.rs
+++ b/clients/filesystem-fuse/src/filesystem.rs
@@ -115,7 +115,10 @@ pub(crate) trait PathFileSystem: Send + Sync {
async fn init(&self) -> Result<()>;
/// Get the file stat by file path, if the file exists, return the file
stat
- async fn stat(&self, path: &Path) -> Result<FileStat>;
+ async fn stat(&self, path: &Path, kind: FileType) -> Result<FileStat>;
+
+ /// Lookup the file stat by file path, if the file exists, return the file
stat
+ async fn lookup(&self, path: &Path) -> Result<FileStat>;
/// Read the directory by file path, if the directory exists, return the
file stat list
async fn read_dir(&self, path: &Path) -> Result<Vec<FileStat>>;
@@ -318,7 +321,7 @@ pub(crate) mod tests {
pub(crate) async fn test_path_file_system(&mut self) {
// test root dir
- let resutl = self.fs.stat(Path::new("/")).await;
+ let resutl = self.fs.stat(Path::new("/"), Directory).await;
assert!(resutl.is_ok());
let root_file_stat = resutl.unwrap();
self.assert_file_stat(&root_file_stat, Path::new("/"), Directory,
0);
@@ -347,7 +350,7 @@ pub(crate) mod tests {
}
async fn test_stat_file(&mut self, path: &Path, expect_kind: FileType,
expect_size: u64) {
- let file_stat = self.fs.stat(path).await;
+ let file_stat = self.fs.stat(path, expect_kind).await;
assert!(file_stat.is_ok());
let file_stat = file_stat.unwrap();
self.assert_file_stat(&file_stat, path, expect_kind, expect_size);
@@ -403,7 +406,7 @@ pub(crate) mod tests {
}
async fn test_file_not_found(&self, path: &Path) {
- let not_found_file = self.fs.stat(path).await;
+ let not_found_file = self.fs.stat(path, RegularFile).await;
assert!(not_found_file.is_err());
}
diff --git a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
index 04236dfe84..ba717673b9 100644
--- a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
+++ b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs
@@ -21,7 +21,7 @@ use crate::filesystem::{FileStat, FileSystemCapacity,
FileSystemContext, PathFil
use crate::gravitino_client::GravitinoClient;
use crate::opened_file::{OpenFileFlags, OpenedFile};
use async_trait::async_trait;
-use fuse3::Errno;
+use fuse3::{Errno, FileType};
use std::path::{Path, PathBuf};
/// GravitinoFileSystem is a filesystem that is associated with a fileset in
Gravitino.
@@ -74,9 +74,16 @@ impl PathFileSystem for GravitinoFilesetFileSystem {
self.physical_fs.init().await
}
- async fn stat(&self, path: &Path) -> Result<FileStat> {
+ async fn stat(&self, path: &Path, kind: FileType) -> Result<FileStat> {
let raw_path = self.gvfs_path_to_raw_path(path);
- let mut file_stat = self.physical_fs.stat(&raw_path).await?;
+ let mut file_stat = self.physical_fs.stat(&raw_path, kind).await?;
+ file_stat.path = self.raw_path_to_gvfs_path(&file_stat.path)?;
+ Ok(file_stat)
+ }
+
+ async fn lookup(&self, path: &Path) -> Result<FileStat> {
+ let raw_path = self.gvfs_path_to_raw_path(path);
+ let mut file_stat = self.physical_fs.lookup(&raw_path).await?;
file_stat.path = self.raw_path_to_gvfs_path(&file_stat.path)?;
Ok(file_stat)
}
diff --git a/clients/filesystem-fuse/src/memory_filesystem.rs
b/clients/filesystem-fuse/src/memory_filesystem.rs
index d038539072..204a91c171 100644
--- a/clients/filesystem-fuse/src/memory_filesystem.rs
+++ b/clients/filesystem-fuse/src/memory_filesystem.rs
@@ -70,7 +70,7 @@ impl PathFileSystem for MemoryFileSystem {
Ok(())
}
- async fn stat(&self, path: &Path) -> Result<FileStat> {
+ async fn stat(&self, path: &Path, _kind: FileType) -> Result<FileStat> {
self.file_map
.read()
.unwrap()
@@ -79,6 +79,10 @@ impl PathFileSystem for MemoryFileSystem {
.ok_or(Errno::from(libc::ENOENT))
}
+ async fn lookup(&self, path: &Path) -> Result<FileStat> {
+ self.stat(path, RegularFile).await
+ }
+
async fn read_dir(&self, path: &Path) -> Result<Vec<FileStat>> {
let file_map = self.file_map.read().unwrap();
@@ -92,7 +96,7 @@ impl PathFileSystem for MemoryFileSystem {
}
async fn open_file(&self, path: &Path, flags: OpenFileFlags) ->
Result<OpenedFile> {
- let file_stat = self.stat(path).await?;
+ let file_stat = self.stat(path, RegularFile).await?;
let mut opened_file = OpenedFile::new(file_stat);
match opened_file.file_stat.kind {
Directory => Ok(opened_file),
diff --git a/clients/filesystem-fuse/src/open_dal_filesystem.rs
b/clients/filesystem-fuse/src/open_dal_filesystem.rs
index 9e094873f5..01969d74e0 100644
--- a/clients/filesystem-fuse/src/open_dal_filesystem.rs
+++ b/clients/filesystem-fuse/src/open_dal_filesystem.rs
@@ -66,7 +66,24 @@ impl PathFileSystem for OpenDalFileSystem {
Ok(())
}
- async fn stat(&self, path: &Path) -> Result<FileStat> {
+ async fn stat(&self, path: &Path, kind: FileType) -> Result<FileStat> {
+ let file_name = match kind {
+ Directory => build_dir_path(path),
+ _ => path.to_string_lossy().to_string(),
+ };
+ let meta = self
+ .op
+ .stat(&file_name)
+ .await
+ .map_err(opendal_error_to_errno)?;
+
+ let mut file_stat = FileStat::new_file_filestat_with_path(path, 0);
+ self.opendal_meta_to_file_stat(&meta, &mut file_stat);
+
+ Ok(file_stat)
+ }
+
+ async fn lookup(&self, path: &Path) -> Result<FileStat> {
let file_name = path.to_string_lossy().to_string();
let meta_result = self.op.stat(&file_name).await;
@@ -114,7 +131,7 @@ impl PathFileSystem for OpenDalFileSystem {
}
async fn open_file(&self, path: &Path, flags: OpenFileFlags) ->
Result<OpenedFile> {
- let file_stat = self.stat(path).await?;
+ let file_stat = self.stat(path, RegularFile).await?;
debug_assert!(file_stat.kind == RegularFile);
let mut file = OpenedFile::new(file_stat);
@@ -155,7 +172,7 @@ impl PathFileSystem for OpenDalFileSystem {
}
async fn open_dir(&self, path: &Path, _flags: OpenFileFlags) ->
Result<OpenedFile> {
- let file_stat = self.stat(path).await?;
+ let file_stat = self.stat(path, Directory).await?;
debug_assert!(file_stat.kind == Directory);
let opened_file = OpenedFile::new(file_stat);
@@ -185,7 +202,7 @@ impl PathFileSystem for OpenDalFileSystem {
.create_dir(&dir_name)
.await
.map_err(opendal_error_to_errno)?;
- let file_stat = self.stat(path).await?;
+ let file_stat = self.stat(path, Directory).await?;
Ok(file_stat)
}
diff --git a/clients/filesystem-fuse/src/s3_filesystem.rs
b/clients/filesystem-fuse/src/s3_filesystem.rs
index e397d31ccf..f4e9fd440d 100644
--- a/clients/filesystem-fuse/src/s3_filesystem.rs
+++ b/clients/filesystem-fuse/src/s3_filesystem.rs
@@ -24,6 +24,7 @@ use crate::open_dal_filesystem::OpenDalFileSystem;
use crate::opened_file::{OpenFileFlags, OpenedFile};
use crate::utils::{parse_location, GvfsResult};
use async_trait::async_trait;
+use fuse3::FileType;
use log::error;
use opendal::layers::LoggingLayer;
use opendal::services::S3;
@@ -94,8 +95,12 @@ impl PathFileSystem for S3FileSystem {
Ok(())
}
- async fn stat(&self, path: &Path) -> Result<FileStat> {
- self.open_dal_fs.stat(path).await
+ async fn stat(&self, path: &Path, kind: FileType) -> Result<FileStat> {
+ self.open_dal_fs.stat(path, kind).await
+ }
+
+ async fn lookup(&self, path: &Path) -> Result<FileStat> {
+ self.open_dal_fs.lookup(path).await
}
async fn read_dir(&self, path: &Path) -> Result<Vec<FileStat>> {
diff --git a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
index 3089a9a1a1..4f500c7bf8 100644
--- a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
+++ b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh
@@ -37,8 +37,6 @@ check_gvfs_fuse_ready() {
}
start_gvfs_fuse() {
- MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs
-
umount $MOUNT_DIR > /dev/null 2>&1 || true
if [ ! -d "$MOUNT_DIR" ]; then
echo "Create the mount point"
diff --git a/clients/filesystem-fuse/tests/bin/run_fuse_testers.sh
b/clients/filesystem-fuse/tests/bin/run_fuse_testers.sh
index 7088a310b5..4d83ee0331 100755
--- a/clients/filesystem-fuse/tests/bin/run_fuse_testers.sh
+++ b/clients/filesystem-fuse/tests/bin/run_fuse_testers.sh
@@ -27,6 +27,8 @@ source ./gvfs_fuse.sh
source ./localstatck.sh
TEST_CONFIG_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml
+MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs
+
start_servers() {
start_localstack
diff --git a/clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh
b/clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh
old mode 100644
new mode 100755
index 8f25c0b395..d4451e0dbe
--- a/clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh
+++ b/clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh
@@ -70,4 +70,3 @@ else
exit 1
fi
-
