jychen7 commented on a change in pull request #2099:
URL: https://github.com/apache/arrow-datafusion/pull/2099#discussion_r835968916
##########
File path: datafusion/src/execution/context.rs
##########
@@ -549,20 +558,19 @@ impl SessionContext {
/// Registers a Parquet data source so that it can be referenced from SQL
statements
/// executed against this context.
- pub async fn register_parquet(&mut self, name: &str, uri: &str) ->
Result<()> {
- let (target_partitions, enable_pruning) = {
+ pub async fn register_parquet(
+ &mut self,
+ name: &str,
+ uri: &str,
+ options: ParquetReadOptions<'_>,
+ ) -> Result<()> {
+ let (target_partitions, parquet_pruning) = {
let conf = self.copied_config();
(conf.target_partitions, conf.parquet_pruning)
};
- let file_format =
ParquetFormat::default().with_enable_pruning(enable_pruning);
-
- let listing_options = ListingOptions {
- format: Arc::new(file_format),
- collect_stat: true,
- file_extension: DEFAULT_PARQUET_EXTENSION.to_owned(),
- target_partitions,
- table_partition_cols: vec![],
- };
+ let listing_options = options
+ .parquet_pruning(parquet_pruning)
Review comment:
this is for following test use
https://github.com/apache/arrow-datafusion/blob/a09e1aeb5fa279e2a14554c3dad9dfb17d9326e7/datafusion/tests/parquet_pruning.rs#L163-L174
disable parquet pruning has one use case from
https://github.com/apache/arrow-datafusion/issues/723
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
