This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 1cc0410f5 refactor: Split compfs service to new crate (#7033)
1cc0410f5 is described below
commit 1cc0410f536264eead6fce6467f94c4d63e2b197
Author: WaterWhisperer <[email protected]>
AuthorDate: Wed Dec 17 01:51:56 2025 +0800
refactor: Split compfs service to new crate (#7033)
---
core/Cargo.lock | 12 +++++-
core/Cargo.toml | 3 +-
core/core/Cargo.toml | 8 ----
core/core/src/services/mod.rs | 5 ---
core/services/compfs/Cargo.toml | 45 ++++++++++++++++++++++
.../compfs => services/compfs/src}/backend.rs | 4 +-
.../compfs => services/compfs/src}/config.rs | 8 ++--
.../compfs => services/compfs/src}/core.rs | 42 +++++++++++++-------
.../compfs => services/compfs/src}/deleter.rs | 4 +-
.../compfs/mod.rs => services/compfs/src/lib.rs} | 12 +++---
.../compfs => services/compfs/src}/lister.rs | 4 +-
.../compfs => services/compfs/src}/reader.rs | 8 ++--
.../compfs => services/compfs/src}/writer.rs | 9 +++--
core/src/lib.rs | 2 +
14 files changed, 115 insertions(+), 51 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index efdc312cd..6173f05e6 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -5558,6 +5558,7 @@ dependencies = [
"opendal-service-azfile",
"opendal-service-b2",
"opendal-service-cloudflare-kv",
+ "opendal-service-compfs",
"opendal-service-cos",
"opendal-service-d1",
"opendal-service-dashmap",
@@ -5632,7 +5633,6 @@ dependencies = [
"base64 0.22.1",
"bytes",
"cacache",
- "compio",
"criterion",
"ctor",
"divan",
@@ -5987,6 +5987,16 @@ dependencies = [
"serde_json",
]
+[[package]]
+name = "opendal-service-compfs"
+version = "0.55.0"
+dependencies = [
+ "compio",
+ "ctor",
+ "opendal-core",
+ "serde",
+]
+
[[package]]
name = "opendal-service-cos"
version = "0.55.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 998807645..b17ae5ea3 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -109,7 +109,7 @@ services-azfile = ["dep:opendal-service-azfile"]
services-b2 = ["dep:opendal-service-b2"]
services-cacache = ["opendal-core/services-cacache"]
services-cloudflare-kv = ["dep:opendal-service-cloudflare-kv"]
-services-compfs = ["opendal-core/services-compfs"]
+services-compfs = ["dep:opendal-service-compfs"]
services-cos = ["dep:opendal-service-cos"]
services-d1 = ["dep:opendal-service-d1"]
services-dashmap = ["dep:opendal-service-dashmap"]
@@ -210,6 +210,7 @@ opendal-service-azdls = { path = "services/azdls", version
= "0.55.0", optional
opendal-service-azfile = { path = "services/azfile", version = "0.55.0",
optional = true, default-features = false }
opendal-service-b2 = { path = "services/b2", version = "0.55.0", optional =
true, default-features = false }
opendal-service-cloudflare-kv = { path = "services/cloudflare-kv", version =
"0.55.0", optional = true, default-features = false }
+opendal-service-compfs = { path = "services/compfs", version = "0.55.0",
optional = true, default-features = false }
opendal-service-cos = { path = "services/cos", version = "0.55.0", optional =
true, default-features = false }
opendal-service-d1 = { path = "services/d1", version = "0.55.0", optional =
true, default-features = false }
opendal-service-dashmap = { path = "services/dashmap", version = "0.55.0",
optional = true, default-features = false }
diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml
index b053bb60b..298db1678 100644
--- a/core/core/Cargo.toml
+++ b/core/core/Cargo.toml
@@ -58,7 +58,6 @@ executors-tokio = ["tokio/rt"]
layers-dtrace = ["dep:probe"]
services-cacache = ["dep:cacache"]
-services-compfs = ["dep:compio"]
services-dbfs = []
services-dropbox = []
services-etcd = ["dep:etcd-client", "dep:fastpool"]
@@ -180,13 +179,6 @@ rocksdb = { version = "0.21.0", default-features = false,
optional = true }
hdfs-native = { version = "0.13", optional = true }
# for services-surrealdb
surrealdb = { version = "2", optional = true, features = ["protocol-http"] }
-# for services-compfs
-compio = { version = "0.16.0", optional = true, features = [
- "runtime",
- "bytes",
- "polling",
- "dispatcher",
-] }
# for services-monoiofs
flume = { version = "0.11", optional = true }
monoio = { version = "0.2.4", optional = true, features = [
diff --git a/core/core/src/services/mod.rs b/core/core/src/services/mod.rs
index 86c211bf2..e89d1a6d9 100644
--- a/core/core/src/services/mod.rs
+++ b/core/core/src/services/mod.rs
@@ -24,11 +24,6 @@ mod cacache;
#[cfg(feature = "services-cacache")]
pub use self::cacache::*;
-#[cfg(feature = "services-compfs")]
-mod compfs;
-#[cfg(feature = "services-compfs")]
-pub use compfs::*;
-
#[cfg(feature = "services-dbfs")]
mod dbfs;
#[cfg(feature = "services-dbfs")]
diff --git a/core/services/compfs/Cargo.toml b/core/services/compfs/Cargo.toml
new file mode 100644
index 000000000..0de5a3c23
--- /dev/null
+++ b/core/services/compfs/Cargo.toml
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL Compfs service implementation"
+name = "opendal-service-compfs"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false, features = [
+ "internal-tokio-rt",
+] }
+
+compio = { version = "0.16.0", features = [
+ "runtime",
+ "bytes",
+ "polling",
+ "dispatcher",
+] }
+ctor = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
diff --git a/core/core/src/services/compfs/backend.rs
b/core/services/compfs/src/backend.rs
similarity index 99%
rename from core/core/src/services/compfs/backend.rs
rename to core/services/compfs/src/backend.rs
index 7d9d7c62e..d4c4611b5 100644
--- a/core/core/src/services/compfs/backend.rs
+++ b/core/services/compfs/src/backend.rs
@@ -28,8 +28,8 @@ use super::deleter::CompfsDeleter;
use super::lister::CompfsLister;
use super::reader::CompfsReader;
use super::writer::CompfsWriter;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
/// [`compio`]-based file system support.
#[derive(Debug, Default)]
diff --git a/core/core/src/services/compfs/config.rs
b/core/services/compfs/src/config.rs
similarity index 90%
rename from core/core/src/services/compfs/config.rs
rename to core/services/compfs/src/config.rs
index 88991c024..52085187a 100644
--- a/core/core/src/services/compfs/config.rs
+++ b/core/services/compfs/src/config.rs
@@ -29,10 +29,10 @@ pub struct CompfsConfig {
pub root: Option<String>,
}
-impl crate::Configurator for CompfsConfig {
+impl opendal_core::Configurator for CompfsConfig {
type Builder = CompfsBuilder;
- fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> {
+ fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result<Self>
{
let mut map = uri.options().clone();
if let Some(root) = uri.root() {
@@ -52,8 +52,8 @@ impl crate::Configurator for CompfsConfig {
#[cfg(test)]
mod tests {
use super::*;
- use crate::Configurator;
- use crate::types::OperatorUri;
+ use opendal_core::Configurator;
+ use opendal_core::OperatorUri;
#[test]
fn from_uri_sets_root() {
diff --git a/core/core/src/services/compfs/core.rs
b/core/services/compfs/src/core.rs
similarity index 82%
rename from core/core/src/services/compfs/core.rs
rename to core/services/compfs/src/core.rs
index 6151ac8e7..4ec94fc68 100644
--- a/core/core/src/services/compfs/core.rs
+++ b/core/services/compfs/src/core.rs
@@ -22,21 +22,43 @@ use std::sync::Arc;
use compio::buf::{IoBuf, IoBuffer, IoVectoredBuf};
use compio::dispatcher::Dispatcher;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
-unsafe impl IoBuf for Buffer {
+// Wrapper type to avoid orphan rules
+#[derive(Debug, Clone)]
+pub struct CompfsBuffer(pub opendal_core::Buffer);
+
+unsafe impl IoBuf for CompfsBuffer {
fn as_buf_ptr(&self) -> *const u8 {
- self.current().as_ptr()
+ self.0.current().as_ptr()
}
fn buf_len(&self) -> usize {
- self.current().len()
+ self.0.current().len()
}
fn buf_capacity(&self) -> usize {
// `Bytes` doesn't expose uninitialized capacity, so treat it as the
same as `len`
- self.current().len()
+ self.0.current().len()
+ }
+}
+
+impl From<CompfsBuffer> for opendal_core::Buffer {
+ fn from(buf: CompfsBuffer) -> Self {
+ buf.0
+ }
+}
+
+impl From<opendal_core::Buffer> for CompfsBuffer {
+ fn from(buf: opendal_core::Buffer) -> Self {
+ Self(buf)
+ }
+}
+
+impl IoVectoredBuf for CompfsBuffer {
+ unsafe fn iter_io_buffer(&self) -> impl Iterator<Item = IoBuffer> {
+ self.0.clone().map(|b| unsafe { b.as_io_buffer() })
}
}
@@ -54,7 +76,7 @@ impl CompfsCore {
self.root.join(path.trim_end_matches('/'))
}
- pub async fn exec<Fn, Fut, R>(&self, f: Fn) -> crate::Result<R>
+ pub async fn exec<Fn, Fut, R>(&self, f: Fn) -> opendal_core::Result<R>
where
Fn: FnOnce() -> Fut + Send + 'static,
Fut: Future<Output = std::io::Result<R>> + 'static,
@@ -81,12 +103,6 @@ impl CompfsCore {
}
}
-impl IoVectoredBuf for Buffer {
- unsafe fn iter_io_buffer(&self) -> impl Iterator<Item = IoBuffer> {
- self.clone().map(|b| unsafe { b.as_io_buffer() })
- }
-}
-
#[cfg(test)]
mod tests {
use bytes::Buf;
diff --git a/core/core/src/services/compfs/deleter.rs
b/core/services/compfs/src/deleter.rs
similarity index 97%
rename from core/core/src/services/compfs/deleter.rs
rename to core/services/compfs/src/deleter.rs
index bef19a529..7be9b396c 100644
--- a/core/core/src/services/compfs/deleter.rs
+++ b/core/services/compfs/src/deleter.rs
@@ -18,8 +18,8 @@
use std::sync::Arc;
use super::core::*;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct CompfsDeleter {
core: Arc<CompfsCore>,
diff --git a/core/core/src/services/compfs/mod.rs
b/core/services/compfs/src/lib.rs
similarity index 84%
rename from core/core/src/services/compfs/mod.rs
rename to core/services/compfs/src/lib.rs
index 5f51aa2ed..afa7a90b7 100644
--- a/core/core/src/services/compfs/mod.rs
+++ b/core/services/compfs/src/lib.rs
@@ -15,10 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-/// Default scheme for compfs service.
-pub const COMPFS_SCHEME: &str = "compfs";
-
-use crate::types::DEFAULT_OPERATOR_REGISTRY;
+#![cfg_attr(docsrs, feature(doc_cfg))]
+//! Compfs service implementation for Apache OpenDAL.
+#![deny(missing_docs)]
mod backend;
mod config;
@@ -31,7 +30,10 @@ mod writer;
pub use backend::CompfsBuilder as Compfs;
pub use config::CompfsConfig;
+/// Default scheme for compfs service.
+pub const COMPFS_SCHEME: &str = "compfs";
+
#[ctor::ctor]
fn register_compfs_service() {
- DEFAULT_OPERATOR_REGISTRY.register::<Compfs>(COMPFS_SCHEME);
+ opendal_core::DEFAULT_OPERATOR_REGISTRY.register::<Compfs>(COMPFS_SCHEME);
}
diff --git a/core/core/src/services/compfs/lister.rs
b/core/services/compfs/src/lister.rs
similarity index 98%
rename from core/core/src/services/compfs/lister.rs
rename to core/services/compfs/src/lister.rs
index 2f2eb5339..78a27fd3a 100644
--- a/core/core/src/services/compfs/lister.rs
+++ b/core/services/compfs/src/lister.rs
@@ -20,8 +20,8 @@ use std::path::Path;
use std::sync::Arc;
use super::core::CompfsCore;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
#[derive(Debug)]
pub struct CompfsLister {
diff --git a/core/core/src/services/compfs/reader.rs
b/core/services/compfs/src/reader.rs
similarity index 94%
rename from core/core/src/services/compfs/reader.rs
rename to core/services/compfs/src/reader.rs
index ba56be7cf..105397abd 100644
--- a/core/core/src/services/compfs/reader.rs
+++ b/core/services/compfs/src/reader.rs
@@ -22,9 +22,9 @@ use compio::buf::IoBuf;
use compio::buf::buf_try;
use compio::io::AsyncReadAt;
-use super::core::CompfsCore;
-use crate::raw::*;
-use crate::*;
+use super::core::{CompfsBuffer, CompfsCore};
+use opendal_core::raw::*;
+use opendal_core::*;
#[derive(Debug)]
pub struct CompfsReader {
@@ -74,6 +74,6 @@ impl oio::Read for CompfsReader {
let frozen = bs.split_to(n).freeze();
self.offset += frozen.len() as u64;
self.core.buf_pool.put(bs);
- Ok(Buffer::from(frozen))
+ Ok(CompfsBuffer::from(Buffer::from(frozen)).into())
}
}
diff --git a/core/core/src/services/compfs/writer.rs
b/core/services/compfs/src/writer.rs
similarity index 91%
rename from core/core/src/services/compfs/writer.rs
rename to core/services/compfs/src/writer.rs
index 50269efd2..87e9f2d78 100644
--- a/core/core/src/services/compfs/writer.rs
+++ b/core/services/compfs/src/writer.rs
@@ -22,9 +22,9 @@ use compio::buf::buf_try;
use compio::fs::File;
use compio::io::AsyncWriteExt;
-use super::core::CompfsCore;
-use crate::raw::*;
-use crate::*;
+use super::core::{CompfsBuffer, CompfsCore};
+use opendal_core::raw::*;
+use opendal_core::*;
#[derive(Debug)]
pub struct CompfsWriter {
@@ -47,10 +47,11 @@ impl oio::Write for CompfsWriter {
return Err(Error::new(ErrorKind::Unexpected, "file has closed"));
};
+ let compfs_buf = CompfsBuffer::from(bs);
let pos = self
.core
.exec(move || async move {
- buf_try!(@try file.write_vectored_all(bs).await);
+ buf_try!(@try file.write_vectored_all(compfs_buf).await);
Ok(file.position())
})
.await?;
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 659b58c28..34edce71d 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -44,6 +44,8 @@ pub mod services {
pub use opendal_service_b2::*;
#[cfg(feature = "services-cloudflare-kv")]
pub use opendal_service_cloudflare_kv::*;
+ #[cfg(feature = "services-compfs")]
+ pub use opendal_service_compfs::*;
#[cfg(feature = "services-cos")]
pub use opendal_service_cos::*;
#[cfg(feature = "services-d1")]
