This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 5be0557909 [Parquet] Add writer option to skip bloom filters for
column chunks whose data pages are all dictionary encoded (#10963)
5be0557909 is described below
commit 5be055790968b1e96c3642604f40b418b8f2f230
Author: ranflarion <[email protected]>
AuthorDate: Wed Sep 9 10:38:10 2026 -0400
[Parquet] Add writer option to skip bloom filters for column chunks whose
data pages are all dictionary encoded (#10963)
# Which issue does this PR close?
- Closes #10962.
# Rationale for this change
A column chunk whose data pages are all dictionary encoded carries its
exact set of distinct values in the dictionary page, so a bloom filter
for it adds nothing a reader cannot already get exactly, while every
value is still hashed into the filter during the write and the filter is
serialized after the chunk. parquet-java stopped writing these in
PARQUET-2251 (apache/parquet-java#1033, 1.13.0), so files from Spark,
Hive and Iceberg never have a bloom filter on a dictionary-only chunk,
and there was no way to get the same output from this crate. Details in
#10962.
# What changes are included in this PR?
- `WriterProperties::bloom_filter_for_dictionary_encoded_chunks` with
`WriterPropertiesBuilder::set_bloom_filter_for_dictionary_encoded_chunks`
and `DEFAULT_BLOOM_FILTER_FOR_DICTIONARY_ENCODED_CHUNKS = true`, so the
default output is unchanged.
- In `GenericColumnWriter::close`, when the option is `false`, the bloom
filter is dropped unless `encoding_stats` records at least one
`DATA_PAGE`/`DATA_PAGE_V2` whose encoding is not `PLAIN_DICTIONARY` or
`RLE_DICTIONARY`, the same test `ParquetFileWriter.writeColumnChunk`
applies in parquet-java. `flush_bloom_filter` is still called so the
encoder state is reset as before.
# Are these changes tested?
Yes, `test_bloom_filter_for_dictionary_encoded_chunks` writes a small
dictionary-friendly Int32 column across the dictionary on/off × option
on/off matrix and asserts a filter is present in every case except
dictionary on with the option off.
# Are there any user-facing changes?
One new writer property, opt-in, documented on the setter. No breaking
changes.
---------
Co-authored-by: Andrew Lamb <[email protected]>
---
parquet/src/column/writer/mod.rs | 43 +++++++++++++++++++++++++++++++++++++++-
parquet/src/file/properties.rs | 28 ++++++++++++++++++++++++++
2 files changed, 70 insertions(+), 1 deletion(-)
diff --git a/parquet/src/column/writer/mod.rs b/parquet/src/column/writer/mod.rs
index 01c812051f..18def152f8 100644
--- a/parquet/src/column/writer/mod.rs
+++ b/parquet/src/column/writer/mod.rs
@@ -826,6 +826,13 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E>
{
let metadata = self.build_column_metadata()?;
self.page_writer.close()?;
+ let write_bloom_filter =
self.props.bloom_filter_for_dictionary_encoded_chunks()
+ || self.has_non_dictionary_data_page();
+ let bloom_filter = self
+ .encoder
+ .flush_bloom_filter()
+ .filter(|_| write_bloom_filter);
+
let boundary_order = match (
self.data_page_boundary_ascending,
self.data_page_boundary_descending,
@@ -848,7 +855,7 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> {
Ok(ColumnCloseResult {
bytes_written: self.column_metrics.total_bytes_written,
rows_written: self.column_metrics.total_rows_written,
- bloom_filter: self.encoder.flush_bloom_filter(),
+ bloom_filter,
metadata,
column_index,
offset_index,
@@ -934,6 +941,18 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E>
{
Ok(values_consumed)
}
+ fn has_non_dictionary_data_page(&self) -> bool {
+ self.encoding_stats.iter().any(|stats| {
+ matches!(
+ stats.page_type,
+ PageType::DATA_PAGE | PageType::DATA_PAGE_V2
+ ) && !matches!(
+ stats.encoding,
+ Encoding::PLAIN_DICTIONARY | Encoding::RLE_DICTIONARY
+ )
+ })
+ }
+
/// Index one past the last level of a sub-batch window that starts at
/// `start` and covers at most `max_values` values, clamped to
/// `chunk_size`.
@@ -2228,6 +2247,28 @@ mod tests {
assert_eq!(metadata.dictionary_page_offset(), None);
}
+ #[test]
+ fn test_bloom_filter_for_dictionary_encoded_chunks() {
+ fn bloom_filter_written(dictionary_enabled: bool,
for_dictionary_chunks: bool) -> bool {
+ let props = Arc::new(
+ WriterProperties::builder()
+ .set_dictionary_enabled(dictionary_enabled)
+ .set_bloom_filter_enabled(true)
+
.set_bloom_filter_for_dictionary_encoded_chunks(for_dictionary_chunks)
+ .build(),
+ );
+ let mut writer =
+ get_test_column_writer::<Int32Type>(get_test_page_writer(), 0,
0, props);
+ writer.write_batch(&[1, 2, 1, 2], None, None).unwrap();
+ writer.close().unwrap().bloom_filter.is_some()
+ }
+
+ assert!(bloom_filter_written(true, true));
+ assert!(bloom_filter_written(false, true));
+ assert!(bloom_filter_written(false, false));
+ assert!(!bloom_filter_written(true, false));
+ }
+
#[test]
fn test_column_writer_default_encoding_support_bool() {
check_encoding_write_support::<BoolType>(
diff --git a/parquet/src/file/properties.rs b/parquet/src/file/properties.rs
index a03fb0d9cc..3de032d4f9 100644
--- a/parquet/src/file/properties.rs
+++ b/parquet/src/file/properties.rs
@@ -46,6 +46,8 @@ pub const DEFAULT_STATISTICS_ENABLED: EnabledStatistics =
EnabledStatistics::Pag
pub const DEFAULT_WRITE_PAGE_HEADER_STATISTICS: bool = false;
/// Default value for [`WriterProperties::max_row_group_row_count`]
pub const DEFAULT_MAX_ROW_GROUP_ROW_COUNT: usize = 1024 * 1024;
+/// Default value for
[`WriterProperties::bloom_filter_for_dictionary_encoded_chunks`]
+pub const DEFAULT_BLOOM_FILTER_FOR_DICTIONARY_ENCODED_CHUNKS: bool = true;
/// Default value for [`WriterProperties::bloom_filter_position`]
pub const DEFAULT_BLOOM_FILTER_POSITION: BloomFilterPosition =
BloomFilterPosition::AfterRowGroup;
/// Default value for [`WriterProperties::created_by`]
@@ -246,6 +248,7 @@ pub struct WriterProperties {
max_row_group_row_count: Option<usize>,
max_row_group_bytes: Option<usize>,
bloom_filter_position: BloomFilterPosition,
+ bloom_filter_for_dictionary_encoded_chunks: bool,
writer_version: WriterVersion,
created_by: String,
offset_index_setting: OffsetIndexSetting,
@@ -369,6 +372,14 @@ impl WriterProperties {
self.bloom_filter_position
}
+ /// Returns whether a column chunk whose data pages are all dictionary
encoded gets a
+ /// bloom filter.
+ ///
+ /// For more details see
[`WriterPropertiesBuilder::set_bloom_filter_for_dictionary_encoded_chunks`]
+ pub fn bloom_filter_for_dictionary_encoded_chunks(&self) -> bool {
+ self.bloom_filter_for_dictionary_encoded_chunks
+ }
+
/// Returns configured writer version.
///
/// For more details see [`WriterPropertiesBuilder::set_writer_version`]
@@ -599,6 +610,7 @@ pub struct WriterPropertiesBuilder {
max_row_group_row_count: Option<usize>,
max_row_group_bytes: Option<usize>,
bloom_filter_position: BloomFilterPosition,
+ bloom_filter_for_dictionary_encoded_chunks: bool,
writer_version: WriterVersion,
created_by: String,
offset_index_disabled: bool,
@@ -625,6 +637,8 @@ impl Default for WriterPropertiesBuilder {
max_row_group_row_count: Some(DEFAULT_MAX_ROW_GROUP_ROW_COUNT),
max_row_group_bytes: None,
bloom_filter_position: DEFAULT_BLOOM_FILTER_POSITION,
+ bloom_filter_for_dictionary_encoded_chunks:
+ DEFAULT_BLOOM_FILTER_FOR_DICTIONARY_ENCODED_CHUNKS,
writer_version: DEFAULT_WRITER_VERSION,
created_by: DEFAULT_CREATED_BY.to_string(),
offset_index_disabled: DEFAULT_OFFSET_INDEX_DISABLED,
@@ -681,6 +695,8 @@ impl WriterPropertiesBuilder {
max_row_group_row_count: self.max_row_group_row_count,
max_row_group_bytes: self.max_row_group_bytes,
bloom_filter_position: self.bloom_filter_position,
+ bloom_filter_for_dictionary_encoded_chunks: self
+ .bloom_filter_for_dictionary_encoded_chunks,
writer_version: self.writer_version,
created_by: self.created_by,
offset_index_setting,
@@ -788,6 +804,16 @@ impl WriterPropertiesBuilder {
self
}
+ /// Sets whether a column chunk whose data pages are all dictionary
encoded gets a bloom
+ /// filter (defaults to `true` via
[`DEFAULT_BLOOM_FILTER_FOR_DICTIONARY_ENCODED_CHUNKS`]).
+ ///
+ /// The dictionary page of such a chunk already holds every distinct
value, so its bloom
+ /// filter is redundant; set this to `false` to skip writing it and save
the space.
+ pub fn set_bloom_filter_for_dictionary_encoded_chunks(mut self, value:
bool) -> Self {
+ self.bloom_filter_for_dictionary_encoded_chunks = value;
+ self
+ }
+
/// Sets "created by" property (defaults to `parquet-rs version <VERSION>`
via
/// [`DEFAULT_CREATED_BY`]).
///
@@ -1374,6 +1400,8 @@ impl From<WriterProperties> for WriterPropertiesBuilder {
max_row_group_row_count: props.max_row_group_row_count,
max_row_group_bytes: props.max_row_group_bytes,
bloom_filter_position: props.bloom_filter_position,
+ bloom_filter_for_dictionary_encoded_chunks: props
+ .bloom_filter_for_dictionary_encoded_chunks,
writer_version: props.writer_version,
created_by: props.created_by,
offset_index_disabled: !matches!(
