This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new 343a0999 remove support for HDFS ... (#1073)
343a0999 is described below
commit 343a0999d2c8713e9a38948c1c07d87250e0e283
Author: Marko Milenković <[email protected]>
AuthorDate: Sat Oct 12 22:50:27 2024 +0100
remove support for HDFS ... (#1073)
---
README.md | 2 +-
ballista/client/Cargo.toml | 2 --
ballista/core/Cargo.toml | 4 ----
ballista/core/src/object_store_registry/mod.rs | 9 ---------
4 files changed, 1 insertion(+), 16 deletions(-)
diff --git a/README.md b/README.md
index 80c10817..f614a16e 100644
--- a/README.md
+++ b/README.md
@@ -66,7 +66,7 @@ See the [architecture
guide](docs/source/contributors-guide/architecture.md) for
## Features
-- Supports HDFS as well as cloud object stores. S3 is supported today and GCS
and Azure support is planned.
+- Supports cloud object stores. S3 is supported today and GCS and Azure
support is planned.
- DataFrame and SQL APIs available from Python and Rust.
- Clients can connect to a Ballista cluster using [Flight SQL][flight-sql].
- JDBC support via Arrow Flight SQL JDBC Driver
diff --git a/ballista/client/Cargo.toml b/ballista/client/Cargo.toml
index 75bb7b48..2fb0c3d1 100644
--- a/ballista/client/Cargo.toml
+++ b/ballista/client/Cargo.toml
@@ -43,7 +43,5 @@ tokio = "1.0"
[features]
azure = ["ballista-core/azure"]
default = []
-hdfs = ["ballista-core/hdfs"]
-hdfs3 = ["ballista-core/hdfs3"]
s3 = ["ballista-core/s3"]
standalone = ["ballista-executor", "ballista-scheduler"]
diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml
index 884b3f60..519b3d9f 100644
--- a/ballista/core/Cargo.toml
+++ b/ballista/core/Cargo.toml
@@ -39,9 +39,6 @@ docsrs = []
# Used for testing ONLY: causes all values to hash to the same value (test for
collisions)
force_hash_collisions = ["datafusion/force_hash_collisions"]
gcs = ["object_store/gcp"]
-# Used to enable hdfs to be registered in the ObjectStoreRegistry by default
-hdfs = ["datafusion-objectstore-hdfs/hdfs"]
-hdfs3 = ["datafusion-objectstore-hdfs/hdfs3"]
s3 = ["object_store/aws"]
[dependencies]
@@ -52,7 +49,6 @@ bytes = "1.0"
chrono = { version = "0.4", default-features = false }
clap = { workspace = true }
datafusion = { workspace = true }
-datafusion-objectstore-hdfs = { version = "0.1.4", default-features = false,
optional = true }
datafusion-proto = { workspace = true }
datafusion-proto-common = { workspace = true }
futures = "0.3"
diff --git a/ballista/core/src/object_store_registry/mod.rs
b/ballista/core/src/object_store_registry/mod.rs
index 2e394a7c..aedccc5e 100644
--- a/ballista/core/src/object_store_registry/mod.rs
+++ b/ballista/core/src/object_store_registry/mod.rs
@@ -20,8 +20,6 @@ use datafusion::datasource::object_store::{
DefaultObjectStoreRegistry, ObjectStoreRegistry,
};
use datafusion::execution::runtime_env::RuntimeConfig;
-#[cfg(any(feature = "hdfs", feature = "hdfs3"))]
-use datafusion_objectstore_hdfs::object_store::hdfs::HadoopFileSystem;
#[cfg(feature = "s3")]
use object_store::aws::AmazonS3Builder;
#[cfg(feature = "azure")]
@@ -54,13 +52,6 @@ impl BallistaObjectStoreRegistry {
&self,
url: &Url,
) -> datafusion::error::Result<Arc<dyn ObjectStore>> {
- #[cfg(any(feature = "hdfs", feature = "hdfs3"))]
- {
- if let Some(store) = HadoopFileSystem::new(url.as_str()) {
- return Ok(Arc::new(store));
- }
- }
-
#[cfg(feature = "s3")]
{
if url.as_str().starts_with("s3://") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
