This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new dff2f3c3c6 Minor: Clarify which parquet options are used for
reading/writing (#11511)
dff2f3c3c6 is described below
commit dff2f3c3c637fd5c3b30ed0cf26fac75c22973ac
Author: Andrew Lamb <[email protected]>
AuthorDate: Thu Jul 18 06:03:39 2024 -0400
Minor: Clarify which parquet options are used for reading/writing (#11511)
---
datafusion/common/src/config.rs | 63 ++++++++++++----------
.../common/src/file_options/parquet_writer.rs | 1 +
.../sqllogictest/test_files/information_schema.slt | 52 +++++++++---------
docs/source/user-guide/configs.md | 52 +++++++++---------
4 files changed, 87 insertions(+), 81 deletions(-)
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs
index 880f0119ce..b46b002baa 100644
--- a/datafusion/common/src/config.rs
+++ b/datafusion/common/src/config.rs
@@ -315,93 +315,96 @@ config_namespace! {
}
config_namespace! {
- /// Options related to parquet files
+ /// Options for reading and writing parquet files
///
/// See also: [`SessionConfig`]
///
/// [`SessionConfig`]:
https://docs.rs/datafusion/latest/datafusion/prelude/struct.SessionConfig.html
pub struct ParquetOptions {
- /// If true, reads the Parquet data page level metadata (the
+ // The following options affect reading parquet files
+
+ /// (reading) If true, reads the Parquet data page level metadata (the
/// Page Index), if present, to reduce the I/O and number of
/// rows decoded.
pub enable_page_index: bool, default = true
- /// If true, the parquet reader attempts to skip entire row groups
based
+ /// (reading) If true, the parquet reader attempts to skip entire row
groups based
/// on the predicate in the query and the metadata (min/max values)
stored in
/// the parquet file
pub pruning: bool, default = true
- /// If true, the parquet reader skip the optional embedded metadata
that may be in
+ /// (reading) If true, the parquet reader skip the optional embedded
metadata that may be in
/// the file Schema. This setting can help avoid schema conflicts when
querying
/// multiple parquet files with schemas containing compatible types
but different metadata
pub skip_metadata: bool, default = true
- /// If specified, the parquet reader will try and fetch the last
`size_hint`
+ /// (reading) If specified, the parquet reader will try and fetch the
last `size_hint`
/// bytes of the parquet file optimistically. If not specified, two
reads are required:
/// One read to fetch the 8-byte parquet footer and
/// another to fetch the metadata length encoded in the footer
pub metadata_size_hint: Option<usize>, default = None
- /// If true, filter expressions are be applied during the parquet
decoding operation to
+ /// (reading) If true, filter expressions are be applied during the
parquet decoding operation to
/// reduce the number of rows decoded. This optimization is sometimes
called "late materialization".
pub pushdown_filters: bool, default = false
- /// If true, filter expressions evaluated during the parquet decoding
operation
+ /// (reading) If true, filter expressions evaluated during the parquet
decoding operation
/// will be reordered heuristically to minimize the cost of
evaluation. If false,
/// the filters are applied in the same order as written in the query
pub reorder_filters: bool, default = false
- // The following map to parquet::file::properties::WriterProperties
+ // The following options affect writing to parquet files
+ // and map to parquet::file::properties::WriterProperties
- /// Sets best effort maximum size of data page in bytes
+ /// (writing) Sets best effort maximum size of data page in bytes
pub data_pagesize_limit: usize, default = 1024 * 1024
- /// Sets write_batch_size in bytes
+ /// (writing) Sets write_batch_size in bytes
pub write_batch_size: usize, default = 1024
- /// Sets parquet writer version
+ /// (writing) Sets parquet writer version
/// valid values are "1.0" and "2.0"
pub writer_version: String, default = "1.0".into()
- /// Sets default parquet compression codec
+ /// (writing) Sets default parquet compression codec.
/// Valid values are: uncompressed, snappy, gzip(level),
/// lzo, brotli(level), lz4, zstd(level), and lz4_raw.
/// These values are not case sensitive. If NULL, uses
/// default parquet writer setting
pub compression: Option<String>, default = Some("zstd(3)".into())
- /// Sets if dictionary encoding is enabled. If NULL, uses
+ /// (writing) Sets if dictionary encoding is enabled. If NULL, uses
/// default parquet writer setting
pub dictionary_enabled: Option<bool>, default = None
- /// Sets best effort maximum dictionary page size, in bytes
+ /// (writing) Sets best effort maximum dictionary page size, in bytes
pub dictionary_page_size_limit: usize, default = 1024 * 1024
- /// Sets if statistics are enabled for any column
+ /// (writing) Sets if statistics are enabled for any column
/// Valid values are: "none", "chunk", and "page"
/// These values are not case sensitive. If NULL, uses
/// default parquet writer setting
pub statistics_enabled: Option<String>, default = None
- /// Sets max statistics size for any column. If NULL, uses
+ /// (writing) Sets max statistics size for any column. If NULL, uses
/// default parquet writer setting
pub max_statistics_size: Option<usize>, default = None
- /// Target maximum number of rows in each row group (defaults to 1M
+ /// (writing) Target maximum number of rows in each row group
(defaults to 1M
/// rows). Writing larger row groups requires more memory to write, but
/// can get better compression and be faster to read.
pub max_row_group_size: usize, default = 1024 * 1024
- /// Sets "created by" property
+ /// (writing) Sets "created by" property
pub created_by: String, default = concat!("datafusion version ",
env!("CARGO_PKG_VERSION")).into()
- /// Sets column index truncate length
+ /// (writing) Sets column index truncate length
pub column_index_truncate_length: Option<usize>, default = None
- /// Sets best effort maximum number of rows in data page
+ /// (writing) Sets best effort maximum number of rows in data page
pub data_page_row_count_limit: usize, default = usize::MAX
- /// Sets default encoding for any column
+ /// (writing) Sets default encoding for any column.
/// Valid values are: plain, plain_dictionary, rle,
/// bit_packed, delta_binary_packed, delta_length_byte_array,
/// delta_byte_array, rle_dictionary, and byte_stream_split.
@@ -409,27 +412,27 @@ config_namespace! {
/// default parquet writer setting
pub encoding: Option<String>, default = None
- /// Use any available bloom filters when reading parquet files
+ /// (writing) Use any available bloom filters when reading parquet
files
pub bloom_filter_on_read: bool, default = true
- /// Write bloom filters for all columns when creating parquet files
+ /// (writing) Write bloom filters for all columns when creating
parquet files
pub bloom_filter_on_write: bool, default = false
- /// Sets bloom filter false positive probability. If NULL, uses
+ /// (writing) Sets bloom filter false positive probability. If NULL,
uses
/// default parquet writer setting
pub bloom_filter_fpp: Option<f64>, default = None
- /// Sets bloom filter number of distinct values. If NULL, uses
+ /// (writing) Sets bloom filter number of distinct values. If NULL,
uses
/// default parquet writer setting
pub bloom_filter_ndv: Option<u64>, default = None
- /// Controls whether DataFusion will attempt to speed up writing
+ /// (writing) Controls whether DataFusion will attempt to speed up
writing
/// parquet files by serializing them in parallel. Each column
/// in each row group in each output file are serialized in parallel
/// leveraging a maximum possible core count of
n_files*n_row_groups*n_columns.
pub allow_single_file_parallelism: bool, default = true
- /// By default parallel parquet writer is tuned for minimum
+ /// (writing) By default parallel parquet writer is tuned for minimum
/// memory usage in a streaming execution plan. You may see
/// a performance benefit when writing large parquet files
/// by increasing maximum_parallel_row_group_writers and
@@ -440,7 +443,7 @@ config_namespace! {
/// data frame.
pub maximum_parallel_row_group_writers: usize, default = 1
- /// By default parallel parquet writer is tuned for minimum
+ /// (writing) By default parallel parquet writer is tuned for minimum
/// memory usage in a streaming execution plan. You may see
/// a performance benefit when writing large parquet files
/// by increasing maximum_parallel_row_group_writers and
@@ -450,7 +453,6 @@ config_namespace! {
/// writing out already in-memory data, such as from a cached
/// data frame.
pub maximum_buffered_record_batches_per_stream: usize, default = 2
-
}
}
@@ -1534,6 +1536,9 @@ macro_rules! config_namespace_with_hashmap {
}
config_namespace_with_hashmap! {
+ /// Options controlling parquet format for individual columns.
+ ///
+ /// See [`ParquetOptions`] for more details
pub struct ColumnOptions {
/// Sets if bloom filter is enabled for the column path.
pub bloom_filter_enabled: Option<bool>, default = None
diff --git a/datafusion/common/src/file_options/parquet_writer.rs
b/datafusion/common/src/file_options/parquet_writer.rs
index dd4bb8ce50..abe7db2009 100644
--- a/datafusion/common/src/file_options/parquet_writer.rs
+++ b/datafusion/common/src/file_options/parquet_writer.rs
@@ -35,6 +35,7 @@ use parquet::{
/// Options for writing parquet files
#[derive(Clone, Debug)]
pub struct ParquetWriterOptions {
+ /// parquet-rs writer properties
pub writer_options: WriterProperties,
}
diff --git a/datafusion/sqllogictest/test_files/information_schema.slt
b/datafusion/sqllogictest/test_files/information_schema.slt
index 95bea1223a..f7b755b019 100644
--- a/datafusion/sqllogictest/test_files/information_schema.slt
+++ b/datafusion/sqllogictest/test_files/information_schema.slt
@@ -262,32 +262,32 @@ datafusion.execution.listing_table_ignore_subdirectory
true Should sub directori
datafusion.execution.max_buffered_batches_per_output_file 2 This is the
maximum number of RecordBatches buffered for each output file being worked.
Higher values can potentially give faster write performance at the cost of
higher peak memory consumption
datafusion.execution.meta_fetch_concurrency 32 Number of files to read in
parallel when inferring schema and statistics
datafusion.execution.minimum_parallel_output_files 4 Guarantees a minimum
level of output files running in parallel. RecordBatches will be distributed in
round robin fashion to each parallel writer. Each writer is closed and a new
file opened once soft_max_rows_per_output_file is reached.
-datafusion.execution.parquet.allow_single_file_parallelism true Controls
whether DataFusion will attempt to speed up writing parquet files by
serializing them in parallel. Each column in each row group in each output file
are serialized in parallel leveraging a maximum possible core count of
n_files*n_row_groups*n_columns.
-datafusion.execution.parquet.bloom_filter_fpp NULL Sets bloom filter false
positive probability. If NULL, uses default parquet writer setting
-datafusion.execution.parquet.bloom_filter_ndv NULL Sets bloom filter number of
distinct values. If NULL, uses default parquet writer setting
-datafusion.execution.parquet.bloom_filter_on_read true Use any available bloom
filters when reading parquet files
-datafusion.execution.parquet.bloom_filter_on_write false Write bloom filters
for all columns when creating parquet files
-datafusion.execution.parquet.column_index_truncate_length NULL Sets column
index truncate length
-datafusion.execution.parquet.compression zstd(3) Sets default parquet
compression codec Valid values are: uncompressed, snappy, gzip(level), lzo,
brotli(level), lz4, zstd(level), and lz4_raw. These values are not case
sensitive. If NULL, uses default parquet writer setting
-datafusion.execution.parquet.created_by datafusion Sets "created by" property
-datafusion.execution.parquet.data_page_row_count_limit 18446744073709551615
Sets best effort maximum number of rows in data page
-datafusion.execution.parquet.data_pagesize_limit 1048576 Sets best effort
maximum size of data page in bytes
-datafusion.execution.parquet.dictionary_enabled NULL Sets if dictionary
encoding is enabled. If NULL, uses default parquet writer setting
-datafusion.execution.parquet.dictionary_page_size_limit 1048576 Sets best
effort maximum dictionary page size, in bytes
-datafusion.execution.parquet.enable_page_index true If true, reads the Parquet
data page level metadata (the Page Index), if present, to reduce the I/O and
number of rows decoded.
-datafusion.execution.parquet.encoding NULL Sets default encoding for any
column Valid values are: plain, plain_dictionary, rle, bit_packed,
delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary,
and byte_stream_split. These values are not case sensitive. If NULL, uses
default parquet writer setting
-datafusion.execution.parquet.max_row_group_size 1048576 Target maximum number
of rows in each row group (defaults to 1M rows). Writing larger row groups
requires more memory to write, but can get better compression and be faster to
read.
-datafusion.execution.parquet.max_statistics_size NULL Sets max statistics size
for any column. If NULL, uses default parquet writer setting
-datafusion.execution.parquet.maximum_buffered_record_batches_per_stream 2 By
default parallel parquet writer is tuned for minimum memory usage in a
streaming execution plan. You may see a performance benefit when writing large
parquet files by increasing maximum_parallel_row_group_writers and
maximum_buffered_record_batches_per_stream if your system has idle cores and
can tolerate additional memory usage. Boosting these values is likely
worthwhile when writing out already in-memory data, [...]
-datafusion.execution.parquet.maximum_parallel_row_group_writers 1 By default
parallel parquet writer is tuned for minimum memory usage in a streaming
execution plan. You may see a performance benefit when writing large parquet
files by increasing maximum_parallel_row_group_writers and
maximum_buffered_record_batches_per_stream if your system has idle cores and
can tolerate additional memory usage. Boosting these values is likely
worthwhile when writing out already in-memory data, such as [...]
-datafusion.execution.parquet.metadata_size_hint NULL If specified, the parquet
reader will try and fetch the last `size_hint` bytes of the parquet file
optimistically. If not specified, two reads are required: One read to fetch the
8-byte parquet footer and another to fetch the metadata length encoded in the
footer
-datafusion.execution.parquet.pruning true If true, the parquet reader attempts
to skip entire row groups based on the predicate in the query and the metadata
(min/max values) stored in the parquet file
-datafusion.execution.parquet.pushdown_filters false If true, filter
expressions are be applied during the parquet decoding operation to reduce the
number of rows decoded. This optimization is sometimes called "late
materialization".
-datafusion.execution.parquet.reorder_filters false If true, filter expressions
evaluated during the parquet decoding operation will be reordered heuristically
to minimize the cost of evaluation. If false, the filters are applied in the
same order as written in the query
-datafusion.execution.parquet.skip_metadata true If true, the parquet reader
skip the optional embedded metadata that may be in the file Schema. This
setting can help avoid schema conflicts when querying multiple parquet files
with schemas containing compatible types but different metadata
-datafusion.execution.parquet.statistics_enabled NULL Sets if statistics are
enabled for any column Valid values are: "none", "chunk", and "page" These
values are not case sensitive. If NULL, uses default parquet writer setting
-datafusion.execution.parquet.write_batch_size 1024 Sets write_batch_size in
bytes
-datafusion.execution.parquet.writer_version 1.0 Sets parquet writer version
valid values are "1.0" and "2.0"
+datafusion.execution.parquet.allow_single_file_parallelism true (writing)
Controls whether DataFusion will attempt to speed up writing parquet files by
serializing them in parallel. Each column in each row group in each output file
are serialized in parallel leveraging a maximum possible core count of
n_files*n_row_groups*n_columns.
+datafusion.execution.parquet.bloom_filter_fpp NULL (writing) Sets bloom filter
false positive probability. If NULL, uses default parquet writer setting
+datafusion.execution.parquet.bloom_filter_ndv NULL (writing) Sets bloom filter
number of distinct values. If NULL, uses default parquet writer setting
+datafusion.execution.parquet.bloom_filter_on_read true (writing) Use any
available bloom filters when reading parquet files
+datafusion.execution.parquet.bloom_filter_on_write false (writing) Write bloom
filters for all columns when creating parquet files
+datafusion.execution.parquet.column_index_truncate_length NULL (writing) Sets
column index truncate length
+datafusion.execution.parquet.compression zstd(3) (writing) Sets default
parquet compression codec. Valid values are: uncompressed, snappy, gzip(level),
lzo, brotli(level), lz4, zstd(level), and lz4_raw. These values are not case
sensitive. If NULL, uses default parquet writer setting
+datafusion.execution.parquet.created_by datafusion (writing) Sets "created by"
property
+datafusion.execution.parquet.data_page_row_count_limit 18446744073709551615
(writing) Sets best effort maximum number of rows in data page
+datafusion.execution.parquet.data_pagesize_limit 1048576 (writing) Sets best
effort maximum size of data page in bytes
+datafusion.execution.parquet.dictionary_enabled NULL (writing) Sets if
dictionary encoding is enabled. If NULL, uses default parquet writer setting
+datafusion.execution.parquet.dictionary_page_size_limit 1048576 (writing) Sets
best effort maximum dictionary page size, in bytes
+datafusion.execution.parquet.enable_page_index true (reading) If true, reads
the Parquet data page level metadata (the Page Index), if present, to reduce
the I/O and number of rows decoded.
+datafusion.execution.parquet.encoding NULL (writing) Sets default encoding
for any column. Valid values are: plain, plain_dictionary, rle, bit_packed,
delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary,
and byte_stream_split. These values are not case sensitive. If NULL, uses
default parquet writer setting
+datafusion.execution.parquet.max_row_group_size 1048576 (writing) Target
maximum number of rows in each row group (defaults to 1M rows). Writing larger
row groups requires more memory to write, but can get better compression and be
faster to read.
+datafusion.execution.parquet.max_statistics_size NULL (writing) Sets max
statistics size for any column. If NULL, uses default parquet writer setting
+datafusion.execution.parquet.maximum_buffered_record_batches_per_stream 2
(writing) By default parallel parquet writer is tuned for minimum memory usage
in a streaming execution plan. You may see a performance benefit when writing
large parquet files by increasing maximum_parallel_row_group_writers and
maximum_buffered_record_batches_per_stream if your system has idle cores and
can tolerate additional memory usage. Boosting these values is likely
worthwhile when writing out already in-me [...]
+datafusion.execution.parquet.maximum_parallel_row_group_writers 1 (writing) By
default parallel parquet writer is tuned for minimum memory usage in a
streaming execution plan. You may see a performance benefit when writing large
parquet files by increasing maximum_parallel_row_group_writers and
maximum_buffered_record_batches_per_stream if your system has idle cores and
can tolerate additional memory usage. Boosting these values is likely
worthwhile when writing out already in-memory dat [...]
+datafusion.execution.parquet.metadata_size_hint NULL (reading) If specified,
the parquet reader will try and fetch the last `size_hint` bytes of the parquet
file optimistically. If not specified, two reads are required: One read to
fetch the 8-byte parquet footer and another to fetch the metadata length
encoded in the footer
+datafusion.execution.parquet.pruning true (reading) If true, the parquet
reader attempts to skip entire row groups based on the predicate in the query
and the metadata (min/max values) stored in the parquet file
+datafusion.execution.parquet.pushdown_filters false (reading) If true, filter
expressions are be applied during the parquet decoding operation to reduce the
number of rows decoded. This optimization is sometimes called "late
materialization".
+datafusion.execution.parquet.reorder_filters false (reading) If true, filter
expressions evaluated during the parquet decoding operation will be reordered
heuristically to minimize the cost of evaluation. If false, the filters are
applied in the same order as written in the query
+datafusion.execution.parquet.skip_metadata true (reading) If true, the parquet
reader skip the optional embedded metadata that may be in the file Schema. This
setting can help avoid schema conflicts when querying multiple parquet files
with schemas containing compatible types but different metadata
+datafusion.execution.parquet.statistics_enabled NULL (writing) Sets if
statistics are enabled for any column Valid values are: "none", "chunk", and
"page" These values are not case sensitive. If NULL, uses default parquet
writer setting
+datafusion.execution.parquet.write_batch_size 1024 (writing) Sets
write_batch_size in bytes
+datafusion.execution.parquet.writer_version 1.0 (writing) Sets parquet writer
version valid values are "1.0" and "2.0"
datafusion.execution.planning_concurrency 13 Fan-out during initial physical
planning. This is mostly use to plan `UNION` children in parallel. Defaults to
the number of CPU cores on the system
datafusion.execution.soft_max_rows_per_output_file 50000000 Target number of
rows in output files when writing multiple. This is a soft max, so it can be
exceeded slightly. There also will be one file smaller than the limit if the
total number of rows written is not roughly divisible by the soft max
datafusion.execution.sort_in_place_threshold_bytes 1048576 When sorting, below
what size should data be concatenated and sorted in a single RecordBatch rather
than sorted in batches and merged.
diff --git a/docs/source/user-guide/configs.md
b/docs/source/user-guide/configs.md
index 5130b0a56d..8d3ecbc985 100644
--- a/docs/source/user-guide/configs.md
+++ b/docs/source/user-guide/configs.md
@@ -49,32 +49,32 @@ Environment variables are read during `SessionConfig`
initialisation so they mus
| datafusion.execution.collect_statistics |
false | Should DataFusion collect statistics after listing
files
[...]
| datafusion.execution.target_partitions | 0
| Number of partitions for query execution. Increasing
partitions can increase concurrency. Defaults to the number of CPU cores on the
system
[...]
| datafusion.execution.time_zone |
+00:00 | The default time zone Some functions, e.g.
`EXTRACT(HOUR from SOME_TIME)`, shift the underlying datetime according to this
time zone, and then extract the hour
[...]
-| datafusion.execution.parquet.enable_page_index |
true | If true, reads the Parquet data page level metadata
(the Page Index), if present, to reduce the I/O and number of rows decoded.
[...]
-| datafusion.execution.parquet.pruning |
true | If true, the parquet reader attempts to skip entire
row groups based on the predicate in the query and the metadata (min/max
values) stored in the parquet file
[...]
-| datafusion.execution.parquet.skip_metadata |
true | If true, the parquet reader skip the optional
embedded metadata that may be in the file Schema. This setting can help avoid
schema conflicts when querying multiple parquet files with schemas containing
compatible types but different metadata
[...]
-| datafusion.execution.parquet.metadata_size_hint |
NULL | If specified, the parquet reader will try and fetch
the last `size_hint` bytes of the parquet file optimistically. If not
specified, two reads are required: One read to fetch the 8-byte parquet footer
and another to fetch the metadata length encoded in the footer
[...]
-| datafusion.execution.parquet.pushdown_filters |
false | If true, filter expressions are be applied during
the parquet decoding operation to reduce the number of rows decoded. This
optimization is sometimes called "late materialization".
[...]
-| datafusion.execution.parquet.reorder_filters |
false | If true, filter expressions evaluated during the
parquet decoding operation will be reordered heuristically to minimize the cost
of evaluation. If false, the filters are applied in the same order as written
in the query
[...]
-| datafusion.execution.parquet.data_pagesize_limit |
1048576 | Sets best effort maximum size of data page in bytes
[...]
-| datafusion.execution.parquet.write_batch_size |
1024 | Sets write_batch_size in bytes
[...]
-| datafusion.execution.parquet.writer_version |
1.0 | Sets parquet writer version valid values are "1.0"
and "2.0"
[...]
-| datafusion.execution.parquet.compression |
zstd(3) | Sets default parquet compression codec Valid values
are: uncompressed, snappy, gzip(level), lzo, brotli(level), lz4, zstd(level),
and lz4_raw. These values are not case sensitive. If NULL, uses default parquet
writer setting
[...]
-| datafusion.execution.parquet.dictionary_enabled |
NULL | Sets if dictionary encoding is enabled. If NULL,
uses default parquet writer setting
[...]
-| datafusion.execution.parquet.dictionary_page_size_limit |
1048576 | Sets best effort maximum dictionary page size, in
bytes
[...]
-| datafusion.execution.parquet.statistics_enabled |
NULL | Sets if statistics are enabled for any column Valid
values are: "none", "chunk", and "page" These values are not case sensitive. If
NULL, uses default parquet writer setting
[...]
-| datafusion.execution.parquet.max_statistics_size |
NULL | Sets max statistics size for any column. If NULL,
uses default parquet writer setting
[...]
-| datafusion.execution.parquet.max_row_group_size |
1048576 | Target maximum number of rows in each row group
(defaults to 1M rows). Writing larger row groups requires more memory to write,
but can get better compression and be faster to read.
[...]
-| datafusion.execution.parquet.created_by |
datafusion version 40.0.0 | Sets "created by" property
[...]
-| datafusion.execution.parquet.column_index_truncate_length |
NULL | Sets column index truncate length
[...]
-| datafusion.execution.parquet.data_page_row_count_limit |
18446744073709551615 | Sets best effort maximum number of rows in data
page
[...]
-| datafusion.execution.parquet.encoding |
NULL | Sets default encoding for any column Valid values
are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed,
delta_length_byte_array, delta_byte_array, rle_dictionary, and
byte_stream_split. These values are not case sensitive. If NULL, uses default
parquet writer setting
[...]
-| datafusion.execution.parquet.bloom_filter_on_read |
true | Use any available bloom filters when reading
parquet files
[...]
-| datafusion.execution.parquet.bloom_filter_on_write |
false | Write bloom filters for all columns when creating
parquet files
[...]
-| datafusion.execution.parquet.bloom_filter_fpp |
NULL | Sets bloom filter false positive probability. If
NULL, uses default parquet writer setting
[...]
-| datafusion.execution.parquet.bloom_filter_ndv |
NULL | Sets bloom filter number of distinct values. If
NULL, uses default parquet writer setting
[...]
-| datafusion.execution.parquet.allow_single_file_parallelism |
true | Controls whether DataFusion will attempt to speed
up writing parquet files by serializing them in parallel. Each column in each
row group in each output file are serialized in parallel leveraging a maximum
possible core count of n_files*n_row_groups*n_columns.
[...]
-| datafusion.execution.parquet.maximum_parallel_row_group_writers | 1
| By default parallel parquet writer is tuned for
minimum memory usage in a streaming execution plan. You may see a performance
benefit when writing large parquet files by increasing
maximum_parallel_row_group_writers and
maximum_buffered_record_batches_per_stream if your system has idle cores and
can tolerate additional memory usage. Boosting these values is likely
worthwhile when writi [...]
-| datafusion.execution.parquet.maximum_buffered_record_batches_per_stream | 2
| By default parallel parquet writer is tuned for
minimum memory usage in a streaming execution plan. You may see a performance
benefit when writing large parquet files by increasing
maximum_parallel_row_group_writers and
maximum_buffered_record_batches_per_stream if your system has idle cores and
can tolerate additional memory usage. Boosting these values is likely
worthwhile when writi [...]
+| datafusion.execution.parquet.enable_page_index |
true | (reading) If true, reads the Parquet data page
level metadata (the Page Index), if present, to reduce the I/O and number of
rows decoded.
[...]
+| datafusion.execution.parquet.pruning |
true | (reading) If true, the parquet reader attempts to
skip entire row groups based on the predicate in the query and the metadata
(min/max values) stored in the parquet file
[...]
+| datafusion.execution.parquet.skip_metadata |
true | (reading) If true, the parquet reader skip the
optional embedded metadata that may be in the file Schema. This setting can
help avoid schema conflicts when querying multiple parquet files with schemas
containing compatible types but different metadata
[...]
+| datafusion.execution.parquet.metadata_size_hint |
NULL | (reading) If specified, the parquet reader will try
and fetch the last `size_hint` bytes of the parquet file optimistically. If not
specified, two reads are required: One read to fetch the 8-byte parquet footer
and another to fetch the metadata length encoded in the footer
[...]
+| datafusion.execution.parquet.pushdown_filters |
false | (reading) If true, filter expressions are be
applied during the parquet decoding operation to reduce the number of rows
decoded. This optimization is sometimes called "late materialization".
[...]
+| datafusion.execution.parquet.reorder_filters |
false | (reading) If true, filter expressions evaluated
during the parquet decoding operation will be reordered heuristically to
minimize the cost of evaluation. If false, the filters are applied in the same
order as written in the query
[...]
+| datafusion.execution.parquet.data_pagesize_limit |
1048576 | (writing) Sets best effort maximum size of data
page in bytes
[...]
+| datafusion.execution.parquet.write_batch_size |
1024 | (writing) Sets write_batch_size in bytes
[...]
+| datafusion.execution.parquet.writer_version |
1.0 | (writing) Sets parquet writer version valid values
are "1.0" and "2.0"
[...]
+| datafusion.execution.parquet.compression |
zstd(3) | (writing) Sets default parquet compression codec.
Valid values are: uncompressed, snappy, gzip(level), lzo, brotli(level), lz4,
zstd(level), and lz4_raw. These values are not case sensitive. If NULL, uses
default parquet writer setting
[...]
+| datafusion.execution.parquet.dictionary_enabled |
NULL | (writing) Sets if dictionary encoding is enabled.
If NULL, uses default parquet writer setting
[...]
+| datafusion.execution.parquet.dictionary_page_size_limit |
1048576 | (writing) Sets best effort maximum dictionary page
size, in bytes
[...]
+| datafusion.execution.parquet.statistics_enabled |
NULL | (writing) Sets if statistics are enabled for any
column Valid values are: "none", "chunk", and "page" These values are not case
sensitive. If NULL, uses default parquet writer setting
[...]
+| datafusion.execution.parquet.max_statistics_size |
NULL | (writing) Sets max statistics size for any column.
If NULL, uses default parquet writer setting
[...]
+| datafusion.execution.parquet.max_row_group_size |
1048576 | (writing) Target maximum number of rows in each row
group (defaults to 1M rows). Writing larger row groups requires more memory to
write, but can get better compression and be faster to read.
[...]
+| datafusion.execution.parquet.created_by |
datafusion version 40.0.0 | (writing) Sets "created by" property
[...]
+| datafusion.execution.parquet.column_index_truncate_length |
NULL | (writing) Sets column index truncate length
[...]
+| datafusion.execution.parquet.data_page_row_count_limit |
18446744073709551615 | (writing) Sets best effort maximum number of rows
in data page
[...]
+| datafusion.execution.parquet.encoding |
NULL | (writing) Sets default encoding for any column.
Valid values are: plain, plain_dictionary, rle, bit_packed,
delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary,
and byte_stream_split. These values are not case sensitive. If NULL, uses
default parquet writer setting
[...]
+| datafusion.execution.parquet.bloom_filter_on_read |
true | (writing) Use any available bloom filters when
reading parquet files
[...]
+| datafusion.execution.parquet.bloom_filter_on_write |
false | (writing) Write bloom filters for all columns when
creating parquet files
[...]
+| datafusion.execution.parquet.bloom_filter_fpp |
NULL | (writing) Sets bloom filter false positive
probability. If NULL, uses default parquet writer setting
[...]
+| datafusion.execution.parquet.bloom_filter_ndv |
NULL | (writing) Sets bloom filter number of distinct
values. If NULL, uses default parquet writer setting
[...]
+| datafusion.execution.parquet.allow_single_file_parallelism |
true | (writing) Controls whether DataFusion will attempt
to speed up writing parquet files by serializing them in parallel. Each column
in each row group in each output file are serialized in parallel leveraging a
maximum possible core count of n_files*n_row_groups*n_columns.
[...]
+| datafusion.execution.parquet.maximum_parallel_row_group_writers | 1
| (writing) By default parallel parquet writer is tuned
for minimum memory usage in a streaming execution plan. You may see a
performance benefit when writing large parquet files by increasing
maximum_parallel_row_group_writers and
maximum_buffered_record_batches_per_stream if your system has idle cores and
can tolerate additional memory usage. Boosting these values is likely
worthwhile [...]
+| datafusion.execution.parquet.maximum_buffered_record_batches_per_stream | 2
| (writing) By default parallel parquet writer is tuned
for minimum memory usage in a streaming execution plan. You may see a
performance benefit when writing large parquet files by increasing
maximum_parallel_row_group_writers and
maximum_buffered_record_batches_per_stream if your system has idle cores and
can tolerate additional memory usage. Boosting these values is likely
worthwhile [...]
| datafusion.execution.aggregate.scalar_update_factor | 10
| Specifies the threshold for using `ScalarValue`s to
update accumulators during high-cardinality aggregations for each input batch.
The aggregation is considered high-cardinality if the number of affected groups
is greater than or equal to `batch_size / scalar_update_factor`. In such cases,
`ScalarValue`s are utilized for updating accumulators, rather than the default
batch-slice approa [...]
| datafusion.execution.planning_concurrency | 0
| Fan-out during initial physical planning. This is
mostly use to plan `UNION` children in parallel. Defaults to the number of CPU
cores on the system
[...]
| datafusion.execution.sort_spill_reservation_bytes |
10485760 | Specifies the reserved memory for each spillable
sort operation to facilitate an in-memory merge. When a sort operation spills
to disk, the in-memory data must be sorted and merged before being written to a
file. This setting reserves a specific amount of memory for that in-memory
sort/merge process. Note: This setting is irrelevant if the sort operation
cannot spill (i.e., if there's [...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
