This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-ballista.git
The following commit(s) were added to refs/heads/master by this push:
new eec5c18d Add a feature for hdfs3 (#439)
eec5c18d is described below
commit eec5c18d4b995c64bbb28b5baee2934a6b4930a1
Author: yahoNanJing <[email protected]>
AuthorDate: Tue Oct 25 01:35:43 2022 +0800
Add a feature for hdfs3 (#439)
* Add a feature for hdfs3
* Update version for dependency of datafusion-objectstore-hdfs
Co-authored-by: yangzhong <[email protected]>
---
ballista/client/Cargo.toml | 1 +
ballista/core/Cargo.toml | 5 +++--
ballista/core/src/utils.rs | 4 ++--
3 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/ballista/client/Cargo.toml b/ballista/client/Cargo.toml
index 63503624..8c030b99 100644
--- a/ballista/client/Cargo.toml
+++ b/ballista/client/Cargo.toml
@@ -43,5 +43,6 @@ tokio = "1.0"
[features]
default = []
hdfs = ["ballista-core/hdfs"]
+hdfs3 = ["ballista-core/hdfs3"]
s3 = ["ballista-core/s3"]
standalone = ["ballista-executor", "ballista-scheduler"]
diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml
index 8da8a778..955af54e 100644
--- a/ballista/core/Cargo.toml
+++ b/ballista/core/Cargo.toml
@@ -34,7 +34,8 @@ rustc-args = ["--cfg", "docsrs"]
# Used for testing ONLY: causes all values to hash to the same value (test for
collisions)
force_hash_collisions = ["datafusion/force_hash_collisions"]
# Used to enable hdfs to be registered in the ObjectStoreRegistry by default
-hdfs = ["datafusion-objectstore-hdfs"]
+hdfs = ["datafusion-objectstore-hdfs/hdfs"]
+hdfs3 = ["datafusion-objectstore-hdfs/hdfs3"]
s3 = ["object_store/aws"]
simd = ["datafusion/simd"]
@@ -46,7 +47,7 @@ async-trait = "0.1.41"
chrono = { version = "0.4", default-features = false }
clap = { version = "3", features = ["derive", "cargo"] }
datafusion = "13.0.0"
-datafusion-objectstore-hdfs = { version = "0.1.0", optional = true }
+datafusion-objectstore-hdfs = { version = "0.1.1", default-features = false,
optional = true }
datafusion-proto = "13.0.0"
futures = "0.3"
hashbrown = "0.12"
diff --git a/ballista/core/src/utils.rs b/ballista/core/src/utils.rs
index 21626c55..33047992 100644
--- a/ballista/core/src/utils.rs
+++ b/ballista/core/src/utils.rs
@@ -43,7 +43,7 @@ use datafusion::physical_plan::metrics::MetricsSet;
use datafusion::physical_plan::projection::ProjectionExec;
use datafusion::physical_plan::sorts::sort::SortExec;
use datafusion::physical_plan::{metrics, ExecutionPlan, RecordBatchStream};
-#[cfg(feature = "hdfs")]
+#[cfg(any(feature = "hdfs", feature = "hdfs3"))]
use datafusion_objectstore_hdfs::object_store::hdfs::HadoopFileSystem;
use datafusion_proto::logical_plan::{
AsLogicalPlan, DefaultLogicalExtensionCodec, LogicalExtensionCodec,
@@ -89,7 +89,7 @@ impl ObjectStoreProvider for FeatureBasedObjectStoreProvider {
/// Return the key and object store
#[allow(unused_variables)]
fn get_by_url(&self, url: &Url) -> datafusion::error::Result<Arc<dyn
ObjectStore>> {
- #[cfg(feature = "hdfs")]
+ #[cfg(any(feature = "hdfs", feature = "hdfs3"))]
{
let store = HadoopFileSystem::new(url.as_str());
if let Some(store) = store {
