This is an automated email from the ASF dual-hosted git repository.
koushiro pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 40acb6beb refactor(services/webhdfs): split webhdfs service into
separate crate (#7067)
40acb6beb is described below
commit 40acb6bebc6fa3214a512fea4e197fb660534262
Author: Piyush मिश्र <[email protected]>
AuthorDate: Fri Dec 19 19:29:55 2025 +0530
refactor(services/webhdfs): split webhdfs service into separate crate
(#7067)
---
core/Cargo.lock | 18 +++++++++
core/Cargo.toml | 3 +-
core/core/src/services/mod.rs | 5 ---
core/services/webhdfs/Cargo.toml | 46 ++++++++++++++++++++++
.../webhdfs => services/webhdfs/src}/backend.rs | 5 ++-
.../webhdfs => services/webhdfs/src}/config.rs | 8 ++--
.../webhdfs => services/webhdfs/src}/core.rs | 4 +-
.../webhdfs => services/webhdfs/src}/deleter.rs | 5 ++-
.../webhdfs => services/webhdfs/src}/docs.md | 6 +--
.../webhdfs => services/webhdfs/src}/error.rs | 4 +-
.../webhdfs/mod.rs => services/webhdfs/src/lib.rs} | 2 +-
.../webhdfs => services/webhdfs/src}/lister.rs | 5 ++-
.../webhdfs => services/webhdfs/src}/message.rs | 0
.../webhdfs => services/webhdfs/src}/writer.rs | 7 ++--
core/src/lib.rs | 2 +
15 files changed, 92 insertions(+), 28 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index e5f0ef985..c1863cdc0 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -5606,6 +5606,7 @@ dependencies = [
"opendal-service-upyun",
"opendal-service-vercel-blob",
"opendal-service-webdav",
+ "opendal-service-webhdfs",
"opendal-service-yandex-disk",
"opendal-testkit",
"opentelemetry",
@@ -6697,6 +6698,23 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "opendal-service-webhdfs"
+version = "0.55.0"
+dependencies = [
+ "anyhow",
+ "bytes",
+ "ctor",
+ "http 1.4.0",
+ "log",
+ "mea",
+ "opendal-core",
+ "serde",
+ "serde_json",
+ "tokio",
+ "uuid",
+]
+
[[package]]
name = "opendal-service-yandex-disk"
version = "0.55.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index d1db38515..559817cf5 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -172,7 +172,7 @@ services-upyun = ["dep:opendal-service-upyun"]
services-vercel-artifacts = ["opendal-core/services-vercel-artifacts"]
services-vercel-blob = ["dep:opendal-service-vercel-blob"]
services-webdav = ["dep:opendal-service-webdav"]
-services-webhdfs = ["opendal-core/services-webhdfs"]
+services-webhdfs = ["dep:opendal-service-webhdfs"]
services-yandex-disk = ["dep:opendal-service-yandex-disk"]
tests = ["opendal-testkit"]
@@ -267,6 +267,7 @@ opendal-service-tikv = { path = "services/tikv", version =
"0.55.0", optional =
opendal-service-upyun = { path = "services/upyun", version = "0.55.0",
optional = true, default-features = false }
opendal-service-vercel-blob = { path = "services/vercel-blob", version =
"0.55.0", optional = true, default-features = false }
opendal-service-webdav = { path = "services/webdav", version = "0.55.0",
optional = true, default-features = false }
+opendal-service-webhdfs = { path = "services/webhdfs", version = "0.55.0",
optional = true, default-features = false }
opendal-service-yandex-disk = { path = "services/yandex-disk", version =
"0.55.0", optional = true, default-features = false }
opendal-testkit = { path = "testkit", version = "0.55.0", optional = true }
diff --git a/core/core/src/services/mod.rs b/core/core/src/services/mod.rs
index 186f7f0ff..05a0b7a3b 100644
--- a/core/core/src/services/mod.rs
+++ b/core/core/src/services/mod.rs
@@ -53,8 +53,3 @@ pub use self::rocksdb::*;
mod sftp;
#[cfg(feature = "services-sftp")]
pub use sftp::*;
-
-#[cfg(feature = "services-webhdfs")]
-mod webhdfs;
-#[cfg(feature = "services-webhdfs")]
-pub use webhdfs::*;
diff --git a/core/services/webhdfs/Cargo.toml b/core/services/webhdfs/Cargo.toml
new file mode 100644
index 000000000..157d692d4
--- /dev/null
+++ b/core/services/webhdfs/Cargo.toml
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL WebHDFS service implementation"
+name = "opendal-service-webhdfs"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+anyhow = { version = "1.0.100", features = ["std"] }
+bytes = { workspace = true }
+ctor = { workspace = true }
+http = { workspace = true }
+log = { workspace = true }
+mea = "0.5.1"
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+uuid = { workspace = true, features = ["serde", "v4"] }
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/core/core/src/services/webhdfs/backend.rs
b/core/services/webhdfs/src/backend.rs
similarity index 99%
rename from core/core/src/services/webhdfs/backend.rs
rename to core/services/webhdfs/src/backend.rs
index b0043cd6f..f463a4396 100644
--- a/core/core/src/services/webhdfs/backend.rs
+++ b/core/services/webhdfs/src/backend.rs
@@ -35,8 +35,9 @@ use super::message::FileStatusType;
use super::message::FileStatusWrapper;
use super::writer::WebhdfsWriter;
use super::writer::WebhdfsWriters;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::oio;
+use opendal_core::raw::*;
+use opendal_core::*;
const WEBHDFS_DEFAULT_ENDPOINT: &str = "http://127.0.0.1:9870";;
diff --git a/core/core/src/services/webhdfs/config.rs
b/core/services/webhdfs/src/config.rs
similarity index 94%
rename from core/core/src/services/webhdfs/config.rs
rename to core/services/webhdfs/src/config.rs
index 76761c25c..f4024128f 100644
--- a/core/core/src/services/webhdfs/config.rs
+++ b/core/services/webhdfs/src/config.rs
@@ -53,10 +53,10 @@ impl Debug for WebhdfsConfig {
}
}
-impl crate::Configurator for WebhdfsConfig {
+impl opendal_core::Configurator for WebhdfsConfig {
type Builder = WebhdfsBuilder;
- fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> {
+ fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result<Self>
{
let mut map = uri.options().clone();
if let Some(authority) = uri.authority() {
map.insert("endpoint".to_string(), format!("http://{authority}";));
@@ -79,8 +79,8 @@ impl crate::Configurator for WebhdfsConfig {
#[cfg(test)]
mod tests {
use super::*;
- use crate::Configurator;
- use crate::types::OperatorUri;
+ use opendal_core::Configurator;
+ use opendal_core::OperatorUri;
#[test]
fn from_uri_sets_endpoint_and_root() {
diff --git a/core/core/src/services/webhdfs/core.rs
b/core/services/webhdfs/src/core.rs
similarity index 99%
rename from core/core/src/services/webhdfs/core.rs
rename to core/services/webhdfs/src/core.rs
index 4fef4886b..c9f289f45 100644
--- a/core/core/src/services/webhdfs/core.rs
+++ b/core/services/webhdfs/src/core.rs
@@ -28,8 +28,8 @@ use mea::once::OnceCell;
use serde::Deserialize;
use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct WebhdfsCore {
pub info: Arc<AccessorInfo>,
diff --git a/core/core/src/services/webhdfs/deleter.rs
b/core/services/webhdfs/src/deleter.rs
similarity index 94%
rename from core/core/src/services/webhdfs/deleter.rs
rename to core/services/webhdfs/src/deleter.rs
index e066cbde6..81a5856c6 100644
--- a/core/core/src/services/webhdfs/deleter.rs
+++ b/core/services/webhdfs/src/deleter.rs
@@ -21,8 +21,9 @@ use http::StatusCode;
use super::core::WebhdfsCore;
use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::oio;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct WebhdfsDeleter {
core: Arc<WebhdfsCore>,
diff --git a/core/core/src/services/webhdfs/docs.md
b/core/services/webhdfs/src/docs.md
similarity index 98%
rename from core/core/src/services/webhdfs/docs.md
rename to core/services/webhdfs/src/docs.md
index b2dd5342c..923e31e58 100644
--- a/core/core/src/services/webhdfs/docs.md
+++ b/core/services/webhdfs/src/docs.md
@@ -26,7 +26,7 @@ This service can be used to:
### File Creation and Write
For [File creation and
write](https://hadoop.apache.org/docs/r3.1.3/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Create_and_Write_to_a_File)
operations,
-OpenDAL WebHDFS is optimized for Hadoop Distributed File System (HDFS)
versions 2.9 and later.
+OpenDAL WebHDFS is optimized for Hadoop Distributed File System (HDFS)
versions 2.9 and later.
This involves two API calls in webhdfs, where the initial `put` call to the
namenode is redirected to the datanode handling the file data.
The optional `noredirect` flag can be set to prevent redirection. If used, the
API response body contains the datanode URL, which is then utilized for the
subsequent `put` call with the actual file data.
OpenDAL automatically sets the `noredirect` flag with the first `put` call.
This feature is supported starting from HDFS version 2.9.
@@ -41,8 +41,6 @@ This issue, identified as
[HDFS-6641](https://issues.apache.org/jira/browse/HDFS
In summary, OpenDAL WebHDFS is designed for optimal compatibility with HDFS,
specifically versions 2.9 and later.
-
-
## Configurations
- `root`: The root path of the WebHDFS service.
@@ -60,7 +58,7 @@ Refer to [`Builder`]'s public API docs for more information.
use std::sync::Arc;
use anyhow::Result;
-use opendal_core::services::Webhdfs;
+use opendal_service_webhdfs::Webhdfs;
use opendal_core::Operator;
#[tokio::main]
diff --git a/core/core/src/services/webhdfs/error.rs
b/core/services/webhdfs/src/error.rs
similarity index 98%
rename from core/core/src/services/webhdfs/error.rs
rename to core/services/webhdfs/src/error.rs
index 057ada360..7c06aa4e6 100644
--- a/core/core/src/services/webhdfs/error.rs
+++ b/core/services/webhdfs/src/error.rs
@@ -20,8 +20,8 @@ use http::StatusCode;
use http::response::Parts;
use serde::Deserialize;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
#[derive(Debug, Deserialize)]
#[serde(rename_all = "PascalCase")]
diff --git a/core/core/src/services/webhdfs/mod.rs
b/core/services/webhdfs/src/lib.rs
similarity index 96%
rename from core/core/src/services/webhdfs/mod.rs
rename to core/services/webhdfs/src/lib.rs
index 7dd99edc8..90fe4ba69 100644
--- a/core/core/src/services/webhdfs/mod.rs
+++ b/core/services/webhdfs/src/lib.rs
@@ -18,7 +18,7 @@
/// Default scheme for webhdfs service.
pub const WEBHDFS_SCHEME: &str = "webhdfs";
-use crate::types::DEFAULT_OPERATOR_REGISTRY;
+use opendal_core::DEFAULT_OPERATOR_REGISTRY;
mod backend;
mod config;
diff --git a/core/core/src/services/webhdfs/lister.rs
b/core/services/webhdfs/src/lister.rs
similarity index 98%
rename from core/core/src/services/webhdfs/lister.rs
rename to core/services/webhdfs/src/lister.rs
index 4fa0e2e71..021d234da 100644
--- a/core/core/src/services/webhdfs/lister.rs
+++ b/core/services/webhdfs/src/lister.rs
@@ -23,8 +23,9 @@ use http::StatusCode;
use super::core::WebhdfsCore;
use super::error::parse_error;
use super::message::*;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::oio;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct WebhdfsLister {
core: Arc<WebhdfsCore>,
diff --git a/core/core/src/services/webhdfs/message.rs
b/core/services/webhdfs/src/message.rs
similarity index 100%
rename from core/core/src/services/webhdfs/message.rs
rename to core/services/webhdfs/src/message.rs
diff --git a/core/core/src/services/webhdfs/writer.rs
b/core/services/webhdfs/src/writer.rs
similarity index 98%
rename from core/core/src/services/webhdfs/writer.rs
rename to core/services/webhdfs/src/writer.rs
index b64c2e7b3..56b691dca 100644
--- a/core/core/src/services/webhdfs/writer.rs
+++ b/core/services/webhdfs/src/writer.rs
@@ -23,9 +23,10 @@ use uuid::Uuid;
use super::core::WebhdfsCore;
use super::error::parse_error;
-use crate::raw::*;
-use crate::services::webhdfs::message::FileStatusWrapper;
-use crate::*;
+use crate::message::FileStatusWrapper;
+use opendal_core::raw::oio;
+use opendal_core::raw::*;
+use opendal_core::*;
pub type WebhdfsWriters =
TwoWays<oio::BlockWriter<WebhdfsWriter>, oio::AppendWriter<WebhdfsWriter>>;
diff --git a/core/src/lib.rs b/core/src/lib.rs
index dd823fb56..ccc4d0b51 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -134,6 +134,8 @@ pub mod services {
pub use opendal_service_vercel_blob::*;
#[cfg(feature = "services-webdav")]
pub use opendal_service_webdav::*;
+ #[cfg(feature = "services-webhdfs")]
+ pub use opendal_service_webhdfs::*;
#[cfg(feature = "services-yandex-disk")]
pub use opendal_service_yandex_disk::*;
}
