This is an automated email from the ASF dual-hosted git repository.
koushiro pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 1ce6e0614 refactor: split sftp service into a separate crate (#7060)
1ce6e0614 is described below
commit 1ce6e0614258f55dc51114df62a69e23ac5ad5ee
Author: Sahil Shadwal <[email protected]>
AuthorDate: Sat Dec 20 13:10:42 2025 +0530
refactor: split sftp service into a separate crate (#7060)
* splitting sftp
* rust format and licence fix
* behavioral fix
* trigger ci
* behavioral fix
* merge conflict
* f
* Merge branch 'main' into sftp
* f
* merge conflict
* f
* Update core/services/sftp/src/core.rs
Co-authored-by: Qinxuan Chen <[email protected]>
* fix: update sftp service documentation import and add development
dependencies
* style: format sftp Cargo.toml with taplo
---------
Co-authored-by: Qinxuan Chen <[email protected]>
---
core/Cargo.lock | 20 ++++++-
core/Cargo.toml | 3 +-
core/core/Cargo.toml | 7 ---
core/core/src/services/mod.rs | 5 --
core/core/src/services/sftp/utils.rs | 54 -------------------
core/services/sftp/Cargo.toml | 48 +++++++++++++++++
.../services/sftp => services/sftp/src}/backend.rs | 7 +--
.../services/sftp => services/sftp/src}/config.rs | 8 +--
.../services/sftp => services/sftp/src}/core.rs | 4 +-
.../services/sftp => services/sftp/src}/deleter.rs | 4 +-
.../services/sftp => services/sftp/src}/docs.md | 2 +-
.../services/sftp => services/sftp/src}/error.rs | 4 +-
.../sftp/mod.rs => services/sftp/src/lib.rs} | 11 ++--
.../services/sftp => services/sftp/src}/lister.rs | 11 ++--
.../services/sftp => services/sftp/src}/reader.rs | 4 +-
.../sftp/deleter.rs => services/sftp/src/utils.rs} | 62 ++++++++++------------
.../services/sftp => services/sftp/src}/writer.rs | 4 +-
core/src/lib.rs | 2 +
18 files changed, 131 insertions(+), 129 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index d0bf60233..0c1dcd2db 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -5599,6 +5599,7 @@ dependencies = [
"opendal-service-redb",
"opendal-service-s3",
"opendal-service-seafile",
+ "opendal-service-sftp",
"opendal-service-sled",
"opendal-service-sqlite",
"opendal-service-surrealdb",
@@ -5671,8 +5672,6 @@ dependencies = [
"moka",
"mongodb",
"mongodb-internal-macros",
- "openssh",
- "openssh-sftp-client",
"percent-encoding",
"pretty_assertions",
"probe",
@@ -6584,6 +6583,23 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "opendal-service-sftp"
+version = "0.55.0"
+dependencies = [
+ "anyhow",
+ "bytes",
+ "ctor",
+ "fastpool",
+ "futures",
+ "log",
+ "opendal-core",
+ "openssh",
+ "openssh-sftp-client",
+ "serde",
+ "tokio",
+]
+
[[package]]
name = "opendal-service-sled"
version = "0.55.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index a5c86bbfe..337ba6b05 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -162,7 +162,7 @@ services-redis-native-tls =
["opendal-core/services-redis-native-tls"]
services-rocksdb = ["opendal-core/services-rocksdb"]
services-s3 = ["dep:opendal-service-s3"]
services-seafile = ["dep:opendal-service-seafile"]
-services-sftp = ["opendal-core/services-sftp"]
+services-sftp = ["dep:opendal-service-sftp"]
services-sled = ["dep:opendal-service-sled"]
services-sqlite = ["dep:opendal-service-sqlite"]
services-surrealdb = ["dep:opendal-service-surrealdb"]
@@ -260,6 +260,7 @@ opendal-service-postgresql = { path =
"services/postgresql", version = "0.55.0",
opendal-service-redb = { path = "services/redb", version = "0.55.0", optional
= true, default-features = false }
opendal-service-s3 = { path = "services/s3", version = "0.55.0", optional =
true, default-features = false }
opendal-service-seafile = { path = "services/seafile", version = "0.55.0",
optional = true, default-features = false }
+opendal-service-sftp = { path = "services/sftp", version = "0.55.0", optional
= true, default-features = false }
opendal-service-sled = { path = "services/sled", version = "0.55.0", optional
= true, default-features = false }
opendal-service-sqlite = { path = "services/sqlite", version = "0.55.0",
optional = true, default-features = false }
opendal-service-surrealdb = { path = "services/surrealdb", version = "0.55.0",
optional = true, default-features = false }
diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml
index 3311d59d0..8de41c8fc 100644
--- a/core/core/Cargo.toml
+++ b/core/core/Cargo.toml
@@ -65,7 +65,6 @@ services-mongodb = ["dep:mongodb",
"dep:mongodb-internal-macros"]
services-redis = ["dep:redis", "dep:fastpool", "redis?/tokio-rustls-comp"]
services-redis-native-tls = ["services-redis", "redis?/tokio-native-tls-comp"]
services-rocksdb = ["dep:rocksdb", "internal-tokio-rt"]
-services-sftp = ["dep:openssh", "dep:openssh-sftp-client", "dep:fastpool"]
services-vercel-artifacts = []
services-webhdfs = []
@@ -112,12 +111,6 @@ moka = { version = "0.12", optional = true, features =
["future", "sync"] }
# for services-mongodb
mongodb = { version = "3.3.0", optional = true }
mongodb-internal-macros = { version = "3.2.4", optional = true }
-# for services-sftp
-openssh = { version = "0.11.0", optional = true }
-openssh-sftp-client = { version = "0.15.3", optional = true, features = [
- "openssh",
- "tracing",
-] }
# for services-redis
redis = { version = "1.0", features = [
"cluster-async",
diff --git a/core/core/src/services/mod.rs b/core/core/src/services/mod.rs
index 7a4adc065..62ee7588e 100644
--- a/core/core/src/services/mod.rs
+++ b/core/core/src/services/mod.rs
@@ -43,8 +43,3 @@ pub use self::redis::*;
mod rocksdb;
#[cfg(feature = "services-rocksdb")]
pub use self::rocksdb::*;
-
-#[cfg(feature = "services-sftp")]
-mod sftp;
-#[cfg(feature = "services-sftp")]
-pub use sftp::*;
diff --git a/core/core/src/services/sftp/utils.rs
b/core/core/src/services/sftp/utils.rs
deleted file mode 100644
index 98e15bfbf..000000000
--- a/core/core/src/services/sftp/utils.rs
+++ /dev/null
@@ -1,54 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use openssh_sftp_client::metadata::MetaData as SftpMeta;
-
-use crate::EntryMode;
-use crate::Metadata;
-use crate::raw::Timestamp;
-
-/// REMOVE ME: we should not implement `From<SftpMeta> for Metadata`.
-impl From<SftpMeta> for Metadata {
- fn from(meta: SftpMeta) -> Self {
- let mode = meta
- .file_type()
- .map(|filetype| {
- if filetype.is_file() {
- EntryMode::FILE
- } else if filetype.is_dir() {
- EntryMode::DIR
- } else {
- EntryMode::Unknown
- }
- })
- .unwrap_or(EntryMode::Unknown);
-
- let mut metadata = Metadata::new(mode);
-
- if let Some(size) = meta.len() {
- metadata.set_content_length(size);
- }
-
- if let Some(modified) = meta.modified() {
- if let Ok(m) = Timestamp::try_from(modified.as_system_time()) {
- metadata.set_last_modified(m);
- }
- }
-
- metadata
- }
-}
diff --git a/core/services/sftp/Cargo.toml b/core/services/sftp/Cargo.toml
new file mode 100644
index 000000000..5186394ef
--- /dev/null
+++ b/core/services/sftp/Cargo.toml
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL SFTP service implementation"
+name = "opendal-service-sftp"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
+
+bytes = { workspace = true }
+ctor = { workspace = true }
+fastpool = "1.0.2"
+futures = { workspace = true }
+log = { workspace = true }
+openssh = "0.11.0"
+openssh-sftp-client = { version = "0.15.3", features = ["openssh", "tracing"] }
+serde = { workspace = true, features = ["derive"] }
+tokio = { workspace = true }
+
+[dev-dependencies]
+anyhow = "1.0"
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/core/core/src/services/sftp/backend.rs
b/core/services/sftp/src/backend.rs
similarity index 98%
rename from core/core/src/services/sftp/backend.rs
rename to core/services/sftp/src/backend.rs
index 384c0b3e4..35e9e1eb5 100644
--- a/core/core/src/services/sftp/backend.rs
+++ b/core/services/sftp/src/backend.rs
@@ -33,9 +33,10 @@ use super::error::is_sftp_protocol_error;
use super::error::parse_sftp_error;
use super::lister::SftpLister;
use super::reader::SftpReader;
+use super::utils::to_metadata;
use super::writer::SftpWriter;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
/// SFTP services support. (only works on unix)
///
@@ -244,7 +245,7 @@ impl Access for SftpBackend {
let mut fs = client.fs();
fs.set_cwd(&self.core.root);
- let meta: Metadata =
fs.metadata(path).await.map_err(parse_sftp_error)?.into();
+ let meta: Metadata =
to_metadata(fs.metadata(path).await.map_err(parse_sftp_error)?);
Ok(RpStat::new(meta))
}
diff --git a/core/core/src/services/sftp/config.rs
b/core/services/sftp/src/config.rs
similarity index 94%
rename from core/core/src/services/sftp/config.rs
rename to core/services/sftp/src/config.rs
index 033268e7d..48445770a 100644
--- a/core/core/src/services/sftp/config.rs
+++ b/core/services/sftp/src/config.rs
@@ -50,10 +50,10 @@ impl Debug for SftpConfig {
}
}
-impl crate::Configurator for SftpConfig {
+impl opendal_core::Configurator for SftpConfig {
type Builder = SftpBuilder;
- fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> {
+ fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result<Self>
{
let mut map = uri.options().clone();
if let Some(authority) = uri.authority() {
map.insert("endpoint".to_string(), authority.to_string());
@@ -74,8 +74,8 @@ impl crate::Configurator for SftpConfig {
#[cfg(test)]
mod tests {
use super::*;
- use crate::Configurator;
- use crate::types::OperatorUri;
+ use opendal_core::Configurator;
+ use opendal_core::OperatorUri;
#[test]
fn from_uri_sets_endpoint_and_root() {
diff --git a/core/core/src/services/sftp/core.rs
b/core/services/sftp/src/core.rs
similarity index 98%
rename from core/core/src/services/sftp/core.rs
rename to core/services/sftp/src/core.rs
index 83ff99258..bb8e24bb3 100644
--- a/core/core/src/services/sftp/core.rs
+++ b/core/services/sftp/src/core.rs
@@ -18,10 +18,10 @@
use super::error::is_sftp_protocol_error;
use super::error::parse_sftp_error;
use super::error::parse_ssh_error;
-use crate::raw::*;
-use crate::*;
use fastpool::{ManageObject, ObjectStatus, bounded};
use log::debug;
+use opendal_core::raw::*;
+use opendal_core::*;
use openssh::KnownHosts;
use openssh::SessionBuilder;
use openssh_sftp_client::Sftp;
diff --git a/core/core/src/services/sftp/deleter.rs
b/core/services/sftp/src/deleter.rs
similarity index 97%
copy from core/core/src/services/sftp/deleter.rs
copy to core/services/sftp/src/deleter.rs
index 9376b1f4b..43cb11faa 100644
--- a/core/core/src/services/sftp/deleter.rs
+++ b/core/services/sftp/src/deleter.rs
@@ -20,8 +20,8 @@ use std::sync::Arc;
use super::core::SftpCore;
use super::error::is_not_found;
use super::error::parse_sftp_error;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct SftpDeleter {
core: Arc<SftpCore>,
diff --git a/core/core/src/services/sftp/docs.md
b/core/services/sftp/src/docs.md
similarity index 97%
rename from core/core/src/services/sftp/docs.md
rename to core/services/sftp/src/docs.md
index 88e33953a..123a167f7 100644
--- a/core/core/src/services/sftp/docs.md
+++ b/core/services/sftp/src/docs.md
@@ -31,7 +31,7 @@ You can refer to [`SftpBuilder`]'s docs for more information
```rust,no_run
use anyhow::Result;
-use opendal_core::services::Sftp;
+use opendal_service_sftp::Sftp;
use opendal_core::Operator;
#[tokio::main]
diff --git a/core/core/src/services/sftp/error.rs
b/core/services/sftp/src/error.rs
similarity index 97%
rename from core/core/src/services/sftp/error.rs
rename to core/services/sftp/src/error.rs
index 5cc071d5c..3fdb749f3 100644
--- a/core/core/src/services/sftp/error.rs
+++ b/core/services/sftp/src/error.rs
@@ -19,8 +19,8 @@ use openssh::Error as SshError;
use openssh_sftp_client::Error as SftpClientError;
use openssh_sftp_client::error::SftpErrorKind;
-use crate::Error;
-use crate::ErrorKind;
+use opendal_core::Error;
+use opendal_core::ErrorKind;
pub fn parse_sftp_error(e: SftpClientError) -> Error {
let kind = match &e {
diff --git a/core/core/src/services/sftp/mod.rs b/core/services/sftp/src/lib.rs
similarity index 85%
rename from core/core/src/services/sftp/mod.rs
rename to core/services/sftp/src/lib.rs
index 5712c4bdc..878be1956 100644
--- a/core/core/src/services/sftp/mod.rs
+++ b/core/services/sftp/src/lib.rs
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-/// Default scheme for sftp service.
-pub const SFTP_SCHEME: &str = "sftp";
+//! SFTP service implementation for Apache OpenDAL.
-use crate::types::DEFAULT_OPERATOR_REGISTRY;
+#![cfg_attr(docsrs, feature(doc_cfg))]
+#![deny(missing_docs)]
mod backend;
mod config;
@@ -33,7 +33,10 @@ mod writer;
pub use backend::SftpBuilder as Sftp;
pub use config::SftpConfig;
+/// Default scheme for sftp service.
+pub const SFTP_SCHEME: &str = "sftp";
+
#[ctor::ctor]
fn register_sftp_service() {
- DEFAULT_OPERATOR_REGISTRY.register::<Sftp>(SFTP_SCHEME);
+ opendal_core::DEFAULT_OPERATOR_REGISTRY.register::<Sftp>(SFTP_SCHEME);
}
diff --git a/core/core/src/services/sftp/lister.rs
b/core/services/sftp/src/lister.rs
similarity index 90%
rename from core/core/src/services/sftp/lister.rs
rename to core/services/sftp/src/lister.rs
index c36fb2f42..6c8dc759d 100644
--- a/core/core/src/services/sftp/lister.rs
+++ b/core/services/sftp/src/lister.rs
@@ -22,9 +22,10 @@ use openssh_sftp_client::fs::DirEntry;
use openssh_sftp_client::fs::ReadDir;
use super::error::parse_sftp_error;
-use crate::Result;
-use crate::raw::oio;
-use crate::raw::oio::Entry;
+use super::utils::to_metadata;
+use opendal_core::Result;
+use opendal_core::raw::oio;
+use opendal_core::raw::oio::Entry;
pub struct SftpLister {
dir: Pin<Box<ReadDir>>,
@@ -61,7 +62,7 @@ impl oio::List for SftpLister {
if self.prefix.is_empty() {
path = "/";
}
- return Ok(Some(Entry::new(path, e.metadata().into())));
+ return Ok(Some(Entry::new(path,
to_metadata(e.metadata()))));
} else {
return Ok(Some(map_entry(self.prefix.as_str(), e)));
}
@@ -84,5 +85,5 @@ fn map_entry(prefix: &str, value: DirEntry) -> Entry {
}
);
- Entry::new(path.as_str(), value.metadata().into())
+ Entry::new(path.as_str(), to_metadata(value.metadata()))
}
diff --git a/core/core/src/services/sftp/reader.rs
b/core/services/sftp/src/reader.rs
similarity index 97%
rename from core/core/src/services/sftp/reader.rs
rename to core/services/sftp/src/reader.rs
index 676389123..f345d2208 100644
--- a/core/core/src/services/sftp/reader.rs
+++ b/core/services/sftp/src/reader.rs
@@ -21,8 +21,8 @@ use openssh_sftp_client::file::File;
use super::core::Manager;
use super::error::parse_sftp_error;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct SftpReader {
/// Keep the connection alive while data stream is alive.
diff --git a/core/core/src/services/sftp/deleter.rs
b/core/services/sftp/src/utils.rs
similarity index 50%
rename from core/core/src/services/sftp/deleter.rs
rename to core/services/sftp/src/utils.rs
index 9376b1f4b..070fef47c 100644
--- a/core/core/src/services/sftp/deleter.rs
+++ b/core/services/sftp/src/utils.rs
@@ -15,41 +15,37 @@
// specific language governing permissions and limitations
// under the License.
-use std::sync::Arc;
-
-use super::core::SftpCore;
-use super::error::is_not_found;
-use super::error::parse_sftp_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct SftpDeleter {
- core: Arc<SftpCore>,
-}
-
-impl SftpDeleter {
- pub fn new(core: Arc<SftpCore>) -> Self {
- Self { core }
+use openssh_sftp_client::metadata::MetaData as SftpMeta;
+
+use opendal_core::EntryMode;
+use opendal_core::Metadata;
+use opendal_core::raw::Timestamp;
+
+pub fn to_metadata(meta: SftpMeta) -> Metadata {
+ let mode = meta
+ .file_type()
+ .map(|filetype| {
+ if filetype.is_file() {
+ EntryMode::FILE
+ } else if filetype.is_dir() {
+ EntryMode::DIR
+ } else {
+ EntryMode::Unknown
+ }
+ })
+ .unwrap_or(EntryMode::Unknown);
+
+ let mut metadata = Metadata::new(mode);
+
+ if let Some(size) = meta.len() {
+ metadata.set_content_length(size);
}
-}
-impl oio::OneShotDelete for SftpDeleter {
- async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
- let client = self.core.connect().await?;
-
- let mut fs = client.fs();
- fs.set_cwd(&self.core.root);
-
- let res = if path.ends_with('/') {
- fs.remove_dir(path).await
- } else {
- fs.remove_file(path).await
- };
-
- match res {
- Ok(()) => Ok(()),
- Err(e) if is_not_found(&e) => Ok(()),
- Err(e) => Err(parse_sftp_error(e)),
+ if let Some(modified) = meta.modified() {
+ if let Ok(m) = Timestamp::try_from(modified.as_system_time()) {
+ metadata.set_last_modified(m);
}
}
+
+ metadata
}
diff --git a/core/core/src/services/sftp/writer.rs
b/core/services/sftp/src/writer.rs
similarity index 97%
rename from core/core/src/services/sftp/writer.rs
rename to core/services/sftp/src/writer.rs
index 7667069c9..542db18a9 100644
--- a/core/core/src/services/sftp/writer.rs
+++ b/core/services/sftp/src/writer.rs
@@ -22,8 +22,8 @@ use openssh_sftp_client::file::File;
use openssh_sftp_client::file::TokioCompatFile;
use tokio::io::AsyncWriteExt;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct SftpWriter {
/// TODO: maybe we can use `File` directly?
diff --git a/core/src/lib.rs b/core/src/lib.rs
index c6b596cbd..956d5b33b 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -120,6 +120,8 @@ pub mod services {
pub use opendal_service_s3::*;
#[cfg(feature = "services-seafile")]
pub use opendal_service_seafile::*;
+ #[cfg(feature = "services-sftp")]
+ pub use opendal_service_sftp::*;
#[cfg(feature = "services-sled")]
pub use opendal_service_sled::*;
#[cfg(feature = "services-sqlite")]
