This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 33aea7e640 Remove `file_type()` from `FileFormat` (#10499)
33aea7e640 is described below
commit 33aea7e6407406a9ec2cbc27323e6f499ebbcd76
Author: Jeffrey Vo <[email protected]>
AuthorDate: Thu May 16 03:23:21 2024 +1000
Remove `file_type()` from `FileFormat` (#10499)
---
datafusion/core/src/datasource/file_format/arrow.rs | 6 +-----
datafusion/core/src/datasource/file_format/avro.rs | 5 -----
datafusion/core/src/datasource/file_format/csv.rs | 9 +++------
datafusion/core/src/datasource/file_format/json.rs | 6 +-----
datafusion/core/src/datasource/file_format/mod.rs | 5 +----
datafusion/core/src/datasource/file_format/parquet.rs | 6 +-----
6 files changed, 7 insertions(+), 30 deletions(-)
diff --git a/datafusion/core/src/datasource/file_format/arrow.rs
b/datafusion/core/src/datasource/file_format/arrow.rs
index 9d58465191..8c67905415 100644
--- a/datafusion/core/src/datasource/file_format/arrow.rs
+++ b/datafusion/core/src/datasource/file_format/arrow.rs
@@ -40,7 +40,7 @@ use arrow::ipc::reader::FileReader;
use arrow::ipc::writer::IpcWriteOptions;
use arrow::ipc::{root_as_message, CompressionType};
use arrow_schema::{ArrowError, Schema, SchemaRef};
-use datafusion_common::{not_impl_err, DataFusionError, FileType, Statistics};
+use datafusion_common::{not_impl_err, DataFusionError, Statistics};
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_physical_expr::{PhysicalExpr, PhysicalSortRequirement};
use datafusion_physical_plan::insert::{DataSink, DataSinkExec};
@@ -136,10 +136,6 @@ impl FileFormat for ArrowFormat {
order_requirements,
)) as _)
}
-
- fn file_type(&self) -> FileType {
- FileType::ARROW
- }
}
/// Implements [`DataSink`] for writing to arrow_ipc files
diff --git a/datafusion/core/src/datasource/file_format/avro.rs
b/datafusion/core/src/datasource/file_format/avro.rs
index 132dae14c6..7b2c26a2c4 100644
--- a/datafusion/core/src/datasource/file_format/avro.rs
+++ b/datafusion/core/src/datasource/file_format/avro.rs
@@ -23,7 +23,6 @@ use std::sync::Arc;
use arrow::datatypes::Schema;
use arrow::datatypes::SchemaRef;
use async_trait::async_trait;
-use datafusion_common::FileType;
use datafusion_physical_expr::PhysicalExpr;
use object_store::{GetResultPayload, ObjectMeta, ObjectStore};
@@ -89,10 +88,6 @@ impl FileFormat for AvroFormat {
let exec = AvroExec::new(conf);
Ok(Arc::new(exec))
}
-
- fn file_type(&self) -> FileType {
- FileType::AVRO
- }
}
#[cfg(test)]
diff --git a/datafusion/core/src/datasource/file_format/csv.rs
b/datafusion/core/src/datasource/file_format/csv.rs
index 17bc7aafce..ae5ac52025 100644
--- a/datafusion/core/src/datasource/file_format/csv.rs
+++ b/datafusion/core/src/datasource/file_format/csv.rs
@@ -42,7 +42,7 @@ use arrow::datatypes::SchemaRef;
use arrow::datatypes::{DataType, Field, Fields, Schema};
use datafusion_common::config::CsvOptions;
use datafusion_common::file_options::csv_writer::CsvWriterOptions;
-use datafusion_common::{exec_err, not_impl_err, DataFusionError, FileType};
+use datafusion_common::{exec_err, not_impl_err, DataFusionError};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::{PhysicalExpr, PhysicalSortRequirement};
use datafusion_physical_plan::metrics::MetricsSet;
@@ -280,10 +280,6 @@ impl FileFormat for CsvFormat {
order_requirements,
)) as _)
}
-
- fn file_type(&self) -> FileType {
- FileType::CSV
- }
}
impl CsvFormat {
@@ -549,8 +545,9 @@ mod tests {
use arrow::compute::concat_batches;
use datafusion_common::cast::as_string_array;
+ use datafusion_common::internal_err;
use datafusion_common::stats::Precision;
- use datafusion_common::{internal_err, GetExt};
+ use datafusion_common::{FileType, GetExt};
use datafusion_execution::runtime_env::{RuntimeConfig, RuntimeEnv};
use datafusion_expr::{col, lit};
diff --git a/datafusion/core/src/datasource/file_format/json.rs
b/datafusion/core/src/datasource/file_format/json.rs
index 9f526e1c87..6e6c798485 100644
--- a/datafusion/core/src/datasource/file_format/json.rs
+++ b/datafusion/core/src/datasource/file_format/json.rs
@@ -43,7 +43,7 @@ use arrow::json::reader::{infer_json_schema_from_iterator,
ValueIter};
use arrow_array::RecordBatch;
use datafusion_common::config::JsonOptions;
use datafusion_common::file_options::json_writer::JsonWriterOptions;
-use datafusion_common::{not_impl_err, FileType};
+use datafusion_common::not_impl_err;
use datafusion_execution::TaskContext;
use datafusion_physical_expr::{PhysicalExpr, PhysicalSortRequirement};
use datafusion_physical_plan::metrics::MetricsSet;
@@ -184,10 +184,6 @@ impl FileFormat for JsonFormat {
order_requirements,
)) as _)
}
-
- fn file_type(&self) -> FileType {
- FileType::JSON
- }
}
impl Default for JsonSerializer {
diff --git a/datafusion/core/src/datasource/file_format/mod.rs
b/datafusion/core/src/datasource/file_format/mod.rs
index fdb89a2649..243a91b743 100644
--- a/datafusion/core/src/datasource/file_format/mod.rs
+++ b/datafusion/core/src/datasource/file_format/mod.rs
@@ -41,7 +41,7 @@ use crate::error::Result;
use crate::execution::context::SessionState;
use crate::physical_plan::{ExecutionPlan, Statistics};
-use datafusion_common::{not_impl_err, FileType};
+use datafusion_common::not_impl_err;
use datafusion_physical_expr::{PhysicalExpr, PhysicalSortRequirement};
use async_trait::async_trait;
@@ -104,9 +104,6 @@ pub trait FileFormat: Send + Sync + fmt::Debug {
) -> Result<Arc<dyn ExecutionPlan>> {
not_impl_err!("Writer not implemented for this format")
}
-
- /// Returns the FileType corresponding to this FileFormat
- fn file_type(&self) -> FileType;
}
#[cfg(test)]
diff --git a/datafusion/core/src/datasource/file_format/parquet.rs
b/datafusion/core/src/datasource/file_format/parquet.rs
index fa379eb5b4..8182ced6f2 100644
--- a/datafusion/core/src/datasource/file_format/parquet.rs
+++ b/datafusion/core/src/datasource/file_format/parquet.rs
@@ -47,7 +47,7 @@ use datafusion_common::config::TableParquetOptions;
use datafusion_common::file_options::parquet_writer::ParquetWriterOptions;
use datafusion_common::stats::Precision;
use datafusion_common::{
- exec_err, internal_datafusion_err, not_impl_err, DataFusionError, FileType,
+ exec_err, internal_datafusion_err, not_impl_err, DataFusionError,
};
use datafusion_common_runtime::SpawnedTask;
use datafusion_execution::TaskContext;
@@ -286,10 +286,6 @@ impl FileFormat for ParquetFormat {
order_requirements,
)) as _)
}
-
- fn file_type(&self) -> FileType {
- FileType::PARQUET
- }
}
fn summarize_min_max(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
