This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 345f727 Replace Datafusion Error with Generic Error for Object store
(#1541)
345f727 is described below
commit 345f72728bf0c3f95a3c994b6d7d5b7de2e3f35e
Author: Matthew Turner <[email protected]>
AuthorDate: Wed Jan 19 15:38:19 2022 -0500
Replace Datafusion Error with Generic Error for Object store (#1541)
* Rebase
* Update tests
* Add GenericError
* Cleanup
* Remove object store trait updates
* Fix clippy
* Cleanup code review comments
---
datafusion/src/datasource/object_store/local.rs | 3 +--
datafusion/src/datasource/object_store/mod.rs | 2 +-
datafusion/src/error.rs | 15 +++++++++++++++
datafusion/src/execution/context.rs | 1 +
4 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/datafusion/src/datasource/object_store/local.rs
b/datafusion/src/datasource/object_store/local.rs
index 0e857c8..d46be4a 100644
--- a/datafusion/src/datasource/object_store/local.rs
+++ b/datafusion/src/datasource/object_store/local.rs
@@ -28,8 +28,7 @@ use crate::datasource::object_store::{
FileMeta, FileMetaStream, ListEntryStream, ObjectReader, ObjectStore,
};
use crate::datasource::PartitionedFile;
-use crate::error::DataFusionError;
-use crate::error::Result;
+use crate::error::{DataFusionError, Result};
use super::{ObjectReaderStream, SizedFile};
diff --git a/datafusion/src/datasource/object_store/mod.rs
b/datafusion/src/datasource/object_store/mod.rs
index 77ca1ef..c774896 100644
--- a/datafusion/src/datasource/object_store/mod.rs
+++ b/datafusion/src/datasource/object_store/mod.rs
@@ -122,7 +122,7 @@ pub type ListEntryStream =
/// Stream readers opened on a given object store
pub type ObjectReaderStream =
- Pin<Box<dyn Stream<Item = Result<Arc<dyn ObjectReader>>> + Send + Sync +
'static>>;
+ Pin<Box<dyn Stream<Item = Result<Arc<dyn ObjectReader>>> + Send + Sync>>;
/// A ObjectStore abstracts access to an underlying file/object storage.
/// It maps strings (e.g. URLs, filesystem paths, etc) to sources of bytes
diff --git a/datafusion/src/error.rs b/datafusion/src/error.rs
index 5e94e14..bf9df50 100644
--- a/datafusion/src/error.rs
+++ b/datafusion/src/error.rs
@@ -31,6 +31,9 @@ use sqlparser::parser::ParserError;
/// Result type for operations that could result in an [DataFusionError]
pub type Result<T> = result::Result<T, DataFusionError>;
+/// Error type for generic operations that could result in
DataFusionError::External
+pub type GenericError = Box<dyn error::Error + Send + Sync>;
+
/// DataFusion error
#[derive(Debug)]
#[allow(missing_docs)]
@@ -64,6 +67,9 @@ pub enum DataFusionError {
/// This error is thrown when a consumer cannot acquire memory from the
Memory Manager
/// we can just cancel the execution of the partition.
ResourcesExhausted(String),
+ /// Errors originating from outside DataFusion's core codebase.
+ /// For example, a custom S3Error from the crate datafusion-objectstore-s3
+ External(GenericError),
}
impl DataFusionError {
@@ -104,6 +110,12 @@ impl From<ParserError> for DataFusionError {
}
}
+impl From<GenericError> for DataFusionError {
+ fn from(err: GenericError) -> Self {
+ DataFusionError::External(err)
+ }
+}
+
impl Display for DataFusionError {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
match *self {
@@ -135,6 +147,9 @@ impl Display for DataFusionError {
DataFusionError::ResourcesExhausted(ref desc) => {
write!(f, "Resources exhausted: {}", desc)
}
+ DataFusionError::External(ref desc) => {
+ write!(f, "External error: {}", desc)
+ }
}
}
}
diff --git a/datafusion/src/execution/context.rs
b/datafusion/src/execution/context.rs
index 5cc0aef..a3ca29f 100644
--- a/datafusion/src/execution/context.rs
+++ b/datafusion/src/execution/context.rs
@@ -576,6 +576,7 @@ impl ExecutionContext {
.unwrap()
.object_store_registry
.get_by_uri(uri)
+ .map_err(DataFusionError::from)
}
/// Registers a table using a custom `TableProvider` so that
