FANNG1 commented on code in PR #5835: URL: https://github.com/apache/gravitino/pull/5835#discussion_r1880116443
########## clients/filesystem-fuse/src/filesystem.rs: ########## @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use async_trait::async_trait; +use bytes::Bytes; +use fuse3::{Errno, FileType, Timestamp}; + +pub(crate) type Result<T> = std::result::Result<T, Errno>; + +/// RawFileSystem interface for the file system implementation. it use by FuseApiHandle Review Comment: please polish the comment to make it understand easily. ########## clients/filesystem-fuse/src/filesystem.rs: ########## @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use async_trait::async_trait; +use bytes::Bytes; +use fuse3::{Errno, FileType, Timestamp}; + +pub(crate) type Result<T> = std::result::Result<T, Errno>; + +/// RawFileSystem interface for the file system implementation. it use by FuseApiHandle +/// the `file_id` and `parent_file_id` it is the unique identifier for the file system, it is used to identify the file or directory +/// the `fh` it is the file handle, it is used to identify the opened file, it is used to read or write the file content +#[async_trait] +pub(crate) trait RawFileSystem: Send + Sync { Review Comment: The name of `RawFileSystem` is confusing, could you provide better name? ########## clients/filesystem-fuse/src/fuse_api_handle.rs: ########## @@ -0,0 +1,445 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use crate::filesystem::{FileStat, FileSystemContext, RawFileSystem}; +use fuse3::path::prelude::{ReplyData, ReplyOpen, ReplyStatFs, ReplyWrite}; +use fuse3::path::Request; +use fuse3::raw::prelude::{ + FileAttr, ReplyAttr, ReplyCreated, ReplyDirectory, ReplyDirectoryPlus, ReplyEntry, ReplyInit, +}; +use fuse3::raw::reply::{DirectoryEntry, DirectoryEntryPlus}; +use fuse3::raw::Filesystem; +use fuse3::FileType::{Directory, RegularFile}; +use fuse3::{Errno, FileType, Inode, SetAttr, Timestamp}; +use futures_util::stream; +use futures_util::stream::BoxStream; +use futures_util::StreamExt; +use std::ffi::{OsStr, OsString}; +use std::num::NonZeroU32; +use std::time::{Duration, SystemTime}; + +pub(crate) struct FuseApiHandle<T: RawFileSystem> { + local_fs: T, + default_ttl: Duration, + fs_context: FileSystemContext, +} + +impl<T: RawFileSystem> FuseApiHandle<T> { + const DEFAULT_TTL: Duration = Duration::from_secs(1); + const DEFAULT_MAX_WRITE: u32 = 16 * 1024; + + pub fn new(fs: T, context: FileSystemContext) -> Self { + Self { + local_fs: fs, + default_ttl: Self::DEFAULT_TTL, + fs_context: context, + } + } + + pub async fn get_file_path(&self, inode: u64) -> String { + self.local_fs.get_file_path(inode).await + } + + async fn get_modified_file_stat( + &self, + inode: u64, + size: Option<u64>, + atime: Option<Timestamp>, + mtime: Option<Timestamp>, + ) -> Result<FileStat, Errno> { + let file_stat = self.local_fs.stat(inode).await?; Review Comment: How about clone directly? ```rust let mut file_stat = self.local_fs.stat(inode).await?.clone(); ``` ########## clients/filesystem-fuse/src/fuse_api_handle.rs: ########## @@ -0,0 +1,445 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use crate::filesystem::{FileStat, FileSystemContext, RawFileSystem}; +use fuse3::path::prelude::{ReplyData, ReplyOpen, ReplyStatFs, ReplyWrite}; +use fuse3::path::Request; +use fuse3::raw::prelude::{ + FileAttr, ReplyAttr, ReplyCreated, ReplyDirectory, ReplyDirectoryPlus, ReplyEntry, ReplyInit, +}; +use fuse3::raw::reply::{DirectoryEntry, DirectoryEntryPlus}; +use fuse3::raw::Filesystem; +use fuse3::FileType::{Directory, RegularFile}; +use fuse3::{Errno, FileType, Inode, SetAttr, Timestamp}; +use futures_util::stream; +use futures_util::stream::BoxStream; +use futures_util::StreamExt; +use std::ffi::{OsStr, OsString}; +use std::num::NonZeroU32; +use std::time::{Duration, SystemTime}; + +pub(crate) struct FuseApiHandle<T: RawFileSystem> { + local_fs: T, + default_ttl: Duration, + fs_context: FileSystemContext, +} + +impl<T: RawFileSystem> FuseApiHandle<T> { + const DEFAULT_TTL: Duration = Duration::from_secs(1); + const DEFAULT_MAX_WRITE: u32 = 16 * 1024; + + pub fn new(fs: T, context: FileSystemContext) -> Self { + Self { + local_fs: fs, + default_ttl: Self::DEFAULT_TTL, + fs_context: context, + } + } + + pub async fn get_file_path(&self, inode: u64) -> String { + self.local_fs.get_file_path(inode).await + } + + async fn get_modified_file_stat( + &self, + inode: u64, + size: Option<u64>, + atime: Option<Timestamp>, + mtime: Option<Timestamp>, + ) -> Result<FileStat, Errno> { + let file_stat = self.local_fs.stat(inode).await?; + let mut nf = FileStat::clone(&file_stat); + + if let Some(size) = size { + nf.size = size; + }; + + if let Some(atime) = atime { + nf.atime = atime; + }; + + if let Some(mtime) = mtime { + nf.mtime = mtime; + }; + + Ok(nf) + } +} + +impl<T: RawFileSystem> Filesystem for FuseApiHandle<T> { + async fn init(&self, _req: Request) -> fuse3::Result<ReplyInit> { + self.local_fs.init().await; + Ok(ReplyInit { + max_write: NonZeroU32::new(Self::DEFAULT_MAX_WRITE).unwrap(), + }) + } + + async fn destroy(&self, _req: Request) {} + + async fn lookup( + &self, + _req: Request, + parent: Inode, + name: &OsStr, + ) -> fuse3::Result<ReplyEntry> { + let name = name.to_string_lossy(); + let file_stat = self.local_fs.lookup(parent, &name).await?; + Ok(ReplyEntry { + ttl: self.default_ttl, + attr: fstat_to_file_attr(&file_stat, &self.fs_context), + generation: 0, + }) + } + + async fn getattr( + &self, + _req: Request, + inode: Inode, + fh: Option<u64>, + _flags: u32, + ) -> fuse3::Result<ReplyAttr> { + // check the opened file inode is the same as the inode + if let Some(fh) = fh { + self.local_fs.valid_file_id(inode, fh).await?; + } + + let file_stat = self.local_fs.stat(inode).await?; + Ok(ReplyAttr { + ttl: self.default_ttl, + attr: fstat_to_file_attr(&file_stat, &self.fs_context), + }) + } + + async fn setattr( + &self, + _req: Request, + inode: Inode, + _fh: Option<u64>, + set_attr: SetAttr, + ) -> fuse3::Result<ReplyAttr> { + let new_file_stat = self + .get_modified_file_stat(inode, set_attr.size, set_attr.atime, set_attr.mtime) + .await?; + let attr = fstat_to_file_attr(&new_file_stat, &self.fs_context); + self.local_fs.set_attr(inode, &new_file_stat).await?; + Ok(ReplyAttr { + ttl: self.default_ttl, + attr: attr, + }) + } + + async fn mkdir( + &self, + _req: Request, + parent: Inode, + name: &OsStr, + _mode: u32, + _umask: u32, + ) -> fuse3::Result<ReplyEntry> { + let name = name.to_string_lossy(); + let handle_id = self.local_fs.create_dir(parent, &name).await?; + Ok(ReplyEntry { + ttl: self.default_ttl, + attr: dummy_file_attr( + handle_id.file_id, + Directory, + Timestamp::from(SystemTime::now()), + &self.fs_context, + ), + generation: 0, + }) + } + + async fn unlink(&self, _req: Request, parent: Inode, name: &OsStr) -> fuse3::Result<()> { Review Comment: is it possible `unlink` a directory? ########## clients/filesystem-fuse/src/filesystem.rs: ########## @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use async_trait::async_trait; +use bytes::Bytes; +use fuse3::{Errno, FileType, Timestamp}; + +pub(crate) type Result<T> = std::result::Result<T, Errno>; + +/// RawFileSystem interface for the file system implementation. it use by FuseApiHandle +/// the `file_id` and `parent_file_id` it is the unique identifier for the file system, it is used to identify the file or directory +/// the `fh` it is the file handle, it is used to identify the opened file, it is used to read or write the file content +#[async_trait] +pub(crate) trait RawFileSystem: Send + Sync { + async fn init(&self); Review Comment: Could you add a clear description to all interfaces? ########## clients/filesystem-fuse/src/filesystem.rs: ########## @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use async_trait::async_trait; +use bytes::Bytes; +use fuse3::{Errno, FileType, Timestamp}; + +pub(crate) type Result<T> = std::result::Result<T, Errno>; + +/// RawFileSystem interface for the file system implementation. it use by FuseApiHandle +/// the `file_id` and `parent_file_id` it is the unique identifier for the file system, it is used to identify the file or directory +/// the `fh` it is the file handle, it is used to identify the opened file, it is used to read or write the file content +#[async_trait] +pub(crate) trait RawFileSystem: Send + Sync { + async fn init(&self); + + async fn get_file_path(&self, file_id: u64) -> String; Review Comment: is `file_id` the same as `inode id`? why not use inode id directly? ########## clients/filesystem-fuse/src/filesystem.rs: ########## @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use async_trait::async_trait; +use bytes::Bytes; +use fuse3::{Errno, FileType, Timestamp}; + +pub(crate) type Result<T> = std::result::Result<T, Errno>; + +/// RawFileSystem interface for the file system implementation. it use by FuseApiHandle +/// the `file_id` and `parent_file_id` it is the unique identifier for the file system, it is used to identify the file or directory +/// the `fh` it is the file handle, it is used to identify the opened file, it is used to read or write the file content +#[async_trait] +pub(crate) trait RawFileSystem: Send + Sync { + async fn init(&self); + + async fn get_file_path(&self, file_id: u64) -> String; + + async fn valid_file_id(&self, file_id: u64, fh: u64) -> Result<()>; + + async fn stat(&self, file_id: u64) -> Result<FileStat>; + + async fn lookup(&self, parent_file_id: u64, name: &str) -> Result<FileStat>; + + async fn read_dir(&self, dir_file_id: u64) -> Result<Vec<FileStat>>; + + async fn open_file(&self, file_id: u64, flags: u32) -> Result<FileHandle>; + + async fn open_dir(&self, file_id: u64, flags: u32) -> Result<FileHandle>; + + async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result<FileHandle>; + + async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result<FileHandle>; + + async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> Result<()>; + + async fn remove_file(&self, parent_file_id: u64, name: &str) -> Result<()>; + + async fn remove_dir(&self, parent_file_id: u64, name: &str) -> Result<()>; + + async fn close_file(&self, file_id: u64, fh: u64) -> Result<()>; + + async fn read(&self, file_id: u64, fh: u64, offset: u64, size: u32) -> Result<Bytes>; + + async fn write(&self, file_id: u64, fh: u64, offset: u64, data: &[u8]) -> Result<u32>; +} + +/// PathFileSystem is the interface for the file system implementation, it use to interact with other file system +/// it is used file name or path to operate the file system +#[async_trait] +pub(crate) trait PathFileSystem: Send + Sync { Review Comment: I didn't see the usage of `PathFileSystem`, could you provide an example? ########## clients/filesystem-fuse/src/fuse_api_handle.rs: ########## @@ -0,0 +1,445 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use crate::filesystem::{FileStat, FileSystemContext, RawFileSystem}; +use fuse3::path::prelude::{ReplyData, ReplyOpen, ReplyStatFs, ReplyWrite}; +use fuse3::path::Request; +use fuse3::raw::prelude::{ + FileAttr, ReplyAttr, ReplyCreated, ReplyDirectory, ReplyDirectoryPlus, ReplyEntry, ReplyInit, +}; +use fuse3::raw::reply::{DirectoryEntry, DirectoryEntryPlus}; +use fuse3::raw::Filesystem; +use fuse3::FileType::{Directory, RegularFile}; +use fuse3::{Errno, FileType, Inode, SetAttr, Timestamp}; +use futures_util::stream; +use futures_util::stream::BoxStream; +use futures_util::StreamExt; +use std::ffi::{OsStr, OsString}; +use std::num::NonZeroU32; +use std::time::{Duration, SystemTime}; + +pub(crate) struct FuseApiHandle<T: RawFileSystem> { + local_fs: T, + default_ttl: Duration, + fs_context: FileSystemContext, +} + +impl<T: RawFileSystem> FuseApiHandle<T> { + const DEFAULT_TTL: Duration = Duration::from_secs(1); + const DEFAULT_MAX_WRITE: u32 = 16 * 1024; + + pub fn new(fs: T, context: FileSystemContext) -> Self { + Self { + local_fs: fs, + default_ttl: Self::DEFAULT_TTL, + fs_context: context, + } + } + + pub async fn get_file_path(&self, inode: u64) -> String { + self.local_fs.get_file_path(inode).await + } + + async fn get_modified_file_stat( + &self, + inode: u64, + size: Option<u64>, + atime: Option<Timestamp>, + mtime: Option<Timestamp>, + ) -> Result<FileStat, Errno> { + let file_stat = self.local_fs.stat(inode).await?; + let mut nf = FileStat::clone(&file_stat); + + if let Some(size) = size { + nf.size = size; + }; + + if let Some(atime) = atime { + nf.atime = atime; + }; + + if let Some(mtime) = mtime { + nf.mtime = mtime; + }; + + Ok(nf) + } +} + +impl<T: RawFileSystem> Filesystem for FuseApiHandle<T> { + async fn init(&self, _req: Request) -> fuse3::Result<ReplyInit> { + self.local_fs.init().await; + Ok(ReplyInit { + max_write: NonZeroU32::new(Self::DEFAULT_MAX_WRITE).unwrap(), + }) + } + + async fn destroy(&self, _req: Request) {} Review Comment: why not release the resource in local_fs here? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
