tustvold commented on code in PR #3885:
URL: https://github.com/apache/arrow-datafusion/pull/3885#discussion_r1002828791
##########
datafusion/core/src/execution/options.rs:
##########
@@ -168,56 +170,31 @@ pub struct ParquetReadOptions<'a> {
pub file_extension: &'a str,
/// Partition Columns
pub table_partition_cols: Vec<String>,
- /// Should DataFusion parquet reader use the predicate to prune data,
- /// overridden by value on execution::context::SessionConfig
- // TODO move this into ConfigOptions
- pub parquet_pruning: bool,
- /// Tell the parquet reader to skip any metadata that may be in
- /// the file Schema. This can help avoid schema conflicts due to
- /// metadata. Defaults to true.
- // TODO move this into ConfigOptions
- pub skip_metadata: bool,
}
impl<'a> Default for ParquetReadOptions<'a> {
fn default() -> Self {
- let format_default = ParquetFormat::default();
-
Self {
file_extension: DEFAULT_PARQUET_EXTENSION,
table_partition_cols: vec![],
- parquet_pruning: format_default.enable_pruning(),
- skip_metadata: format_default.skip_metadata(),
}
}
}
impl<'a> ParquetReadOptions<'a> {
- /// Specify parquet_pruning
- pub fn parquet_pruning(mut self, parquet_pruning: bool) -> Self {
Review Comment:
Doesn't this represent a functionality loss, previously you could configure
these options on per-datasource basis. Now they are located on ConfigOptions
which is session global?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
