matthewmturner commented on a change in pull request #1715:
URL: https://github.com/apache/arrow-datafusion/pull/1715#discussion_r803179007
##########
File path: datafusion/src/datasource/listing/table.rs
##########
@@ -40,7 +44,123 @@ use crate::datasource::{
use super::helpers::{expr_applicable_for_cols, pruned_partition_list,
split_files};
+/// Configuration for creating a 'ListingTable'
+pub struct ListingTableConfig {
+ /// `ObjectStore` that contains the files for the `ListingTable`.
+ pub object_store: Arc<dyn ObjectStore>,
+ /// Path on the `ObjectStore` for creating `ListingTable`.
+ pub table_path: String,
+ /// Optional `SchemaRef` for the to be created `ListingTable`.
+ pub file_schema: Option<SchemaRef>,
+ /// Optional `ListingOptions` for the to be created `ListingTable`.
+ pub options: Option<ListingOptions>,
+}
+
+impl ListingTableConfig {
+ /// Creates new `ListingTableConfig`. The `SchemaRef` and
`ListingOptions` are inferred based on the suffix of the provided `table_path`.
+ pub fn new(
+ object_store: Arc<dyn ObjectStore>,
+ table_path: impl Into<String>,
+ ) -> Self {
+ Self {
+ object_store,
+ table_path: table_path.into(),
+ file_schema: None,
+ options: None,
+ }
+ }
+ /// Add `schema` to `ListingTableConfig`
+ pub fn with_schema(self, schema: SchemaRef) -> Self {
+ Self {
+ object_store: self.object_store,
+ table_path: self.table_path,
+ file_schema: Some(schema),
+ options: self.options,
+ }
+ }
+
+ /// Add `listing_options` to `ListingTableConfig`
+ pub fn with_listing_options(self, listing_options: ListingOptions) -> Self
{
+ Self {
+ object_store: self.object_store,
+ table_path: self.table_path,
+ file_schema: self.file_schema,
+ options: Some(listing_options),
+ }
+ }
+
+ fn infer_format(suffix: &str) -> Result<Arc<dyn FileFormat>> {
+ match suffix {
+ "avro" => Ok(Arc::new(AvroFormat::default())),
+ "csv" => Ok(Arc::new(CsvFormat::default())),
+ "json" => Ok(Arc::new(JsonFormat::default())),
+ "parquet" => Ok(Arc::new(ParquetFormat::default())),
+ _ => Err(DataFusionError::Internal(
+ "Unable to infer file type".into(),
Review comment:
Good idea. I'll update.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
