FANNG1 commented on code in PR #6013: URL: https://github.com/apache/gravitino/pull/6013#discussion_r1900407921
########## clients/filesystem-fuse/conf/gvfs_fuse.toml: ########## @@ -32,7 +32,7 @@ block_size = 8192 uri = "http://localhost:8090"; metalake = "your_metalake" -# extent settings +# extend settings [extend_config] -access_key = "your access_key" -secret_key = "your_secret_key" +access_key_id = "your access_key" Review Comment: add s3 prefix? ########## clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs: ########## @@ -30,32 +30,41 @@ use std::path::{Path, PathBuf}; pub(crate) struct GravitinoFilesetFileSystem { physical_fs: Box<dyn PathFileSystem>, client: GravitinoClient, - fileset_location: PathBuf, + // target_path is a absolute path in the physical filesystem that is associated with the fileset. Review Comment: seems not a absolute path? ########## clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs: ########## @@ -30,32 +30,41 @@ use std::path::{Path, PathBuf}; pub(crate) struct GravitinoFilesetFileSystem { physical_fs: Box<dyn PathFileSystem>, client: GravitinoClient, - fileset_location: PathBuf, + // target_path is a absolute path in the physical filesystem that is associated with the fileset. Review Comment: rename `fileset_location` to `target_path` seems confusing ########## clients/filesystem-fuse/src/filesystem.rs: ########## @@ -290,56 +293,47 @@ pub trait FileWriter: Sync + Send { pub(crate) mod tests { use super::*; use std::collections::HashMap; + use std::path::Component; pub(crate) struct TestPathFileSystem<F: PathFileSystem> { files: HashMap<PathBuf, FileStat>, fs: F, + cwd: PathBuf, } impl<F: PathFileSystem> TestPathFileSystem<F> { - pub(crate) fn new(fs: F) -> Self { + pub(crate) fn new(cwd: &Path, fs: F) -> Self { Self { files: HashMap::new(), fs, + cwd: cwd.into(), } } pub(crate) async fn test_path_file_system(&mut self) { - // Test root dir - self.test_root_dir().await; + // test root dir + self.test_stat_file(Path::new("/"), Directory, 0).await; - // Test stat file - self.test_stat_file(Path::new("/.gvfs_meta"), RegularFile, 0) - .await; + // test list root dir + self.test_list_dir(Path::new("/")).await; // Test create file - self.test_create_file(Path::new("/file1.txt")).await; + self.test_create_file(&self.cwd.join("file1.txt")).await; // Test create dir - self.test_create_dir(Path::new("/dir1")).await; + self.test_create_dir(&self.cwd.join("dir1")).await; // Test list dir - self.test_list_dir(Path::new("/")).await; + self.test_list_dir(&self.cwd).await; // Test remove file - self.test_remove_file(Path::new("/file1.txt")).await; + self.test_remove_file(&self.cwd.join("file1.txt")).await; // Test remove dir - self.test_remove_dir(Path::new("/dir1")).await; + self.test_remove_dir(&self.cwd.join("dir1")).await; // Test file not found - self.test_file_not_found(Path::new("unknown")).await; Review Comment: why removing? ########## clients/filesystem-fuse/src/default_raw_filesystem.rs: ########## @@ -389,17 +441,4 @@ mod tests { assert!(manager.get_file_entry_by_id(2).is_none()); assert!(manager.get_file_entry_by_path(Path::new("a/b")).is_none()); } - - #[tokio::test] - async fn test_default_raw_file_system() { Review Comment: why removing the test? ########## clients/filesystem-fuse/src/open_dal_filesystem.rs: ########## @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use crate::config::AppConfig; +use crate::filesystem::{ + FileReader, FileStat, FileSystemCapacity, FileSystemContext, FileWriter, PathFileSystem, Result, +}; +use crate::opened_file::{OpenFileFlags, OpenedFile}; +use async_trait::async_trait; +use bytes::Bytes; +use fuse3::FileType::{Directory, RegularFile}; +use fuse3::{Errno, FileType, Timestamp}; +use log::{debug, error}; +use opendal::{EntryMode, ErrorKind, Metadata, Operator}; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +pub(crate) struct OpenDalFileSystem { + op: Operator, +} + +impl OpenDalFileSystem {} + +impl OpenDalFileSystem { + pub(crate) fn new(op: Operator, _config: &AppConfig, _fs_context: &FileSystemContext) -> Self { + Self { op: op } + } + + fn opendal_meta_to_file_stat(&self, meta: &Metadata, file_stat: &mut FileStat) { + let now = SystemTime::now(); + let mtime = meta.last_modified().map(|x| x.into()).unwrap_or(now); + + file_stat.size = meta.content_length(); + file_stat.kind = opendal_filemode_to_filetype(meta.mode()); + file_stat.ctime = Timestamp::from(mtime); + file_stat.atime = Timestamp::from(now); + file_stat.mtime = Timestamp::from(mtime); + } +} + +#[async_trait] +impl PathFileSystem for OpenDalFileSystem { + async fn init(&self) -> Result<()> { + Ok(()) + } + + async fn stat(&self, path: &Path) -> Result<FileStat> { + let file_name = path.to_string_lossy().to_string(); + let meta_result = self.op.stat(&file_name).await; + + // path may be a directory, so try to stat it as a directory + let meta = match meta_result { + Ok(meta) => meta, + Err(err) => { + if err.kind() == ErrorKind::NotFound { + let dir_name = format!("{}/", file_name); + self.op + .stat(&dir_name) + .await + .map_err(opendal_error_to_errno)? + } else { + return Err(opendal_error_to_errno(err)); + } + } + }; + + let mut file_stat = FileStat::new_file_filestat_with_path(path, 0); + self.opendal_meta_to_file_stat(&meta, &mut file_stat); + + Ok(file_stat) + } + + async fn read_dir(&self, path: &Path) -> Result<Vec<FileStat>> { + let dir_name = path.to_string_lossy().to_string() + "/"; Review Comment: please add comment why adding `"/"` to origin path, is it possible path endswith `/`? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
