This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20671-f08b933057542b35954ea92233b6696c5d22c102 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit fdb64abd7d7cc4bf27b733c3ed83f80ab92114e3 Author: Shuo Cheng <[email protected]> AuthorDate: Tue Nov 17 13:04:23 2026 +0000 fix(common): disable writing bloom filters for native log files (#20671) * fix(common): disable writing bloom filters for native log files * refactor(common): share native log bloom filter policy --- .../row/HoodieRowDataFileWriterFactory.java | 4 +- .../row/TestHoodieRowDataFileWriterFactory.java | 71 +++++++++++++++++++ .../io/storage/HoodieSparkFileWriterFactory.java | 6 +- .../storage/TestHoodieAvroFileWriterFactory.java | 70 +++++++++++++++---- .../hudi/common/config/HoodieStorageConfig.java | 3 +- .../core/io/storage/HoodieFileWriterFactory.java | 7 +- .../hadoop/HoodieAvroFileWriterFactory.java | 7 +- .../hudi/io/hadoop/TestHoodieOrcReaderWriter.java | 31 +++++++++ .../storage/TestHoodieSparkFileWriterFactory.java | 79 ++++++++++++++++++++++ 9 files changed, 253 insertions(+), 25 deletions(-) diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java index 1f7323604d1c..f13ac3a40405 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java @@ -102,7 +102,7 @@ public class HoodieRowDataFileWriterFactory extends HoodieFileWriterFactory { HoodieConfig hoodieConfig = injectedConfigs.getRight(); Configuration conf = (Configuration) storageConfiguration.unwrapAs(Configuration.class); - BloomFilter filter = enableBloomFilter(metaFieldsMode, hoodieConfig) ? createBloomFilter(hoodieConfig) : null; + BloomFilter filter = enableBloomFilter(metaFieldsMode, hoodieConfig, storagePath) ? createBloomFilter(hoodieConfig) : null; HoodieRowDataParquetWriteSupport writeSupport = (HoodieRowDataParquetWriteSupport) ReflectionUtils.loadClass( hoodieConfig.getStringOrDefault(HoodieStorageConfig.HOODIE_PARQUET_FLINK_ROW_DATA_WRITE_SUPPORT_CLASS), new Class<?>[] {Configuration.class, HoodieSchema.class, BloomFilter.class}, @@ -121,7 +121,7 @@ public class HoodieRowDataFileWriterFactory extends HoodieFileWriterFactory { TaskContextSupplier taskContextSupplier) { MetaFieldsMode metaFieldsMode = MetaFieldsMode.resolve(config); boolean withOperation = config.getBooleanOrDefault(HoodieWriteConfig.ALLOW_OPERATION_METADATA_FIELD); - Option<org.apache.hudi.common.bloom.BloomFilter> bloomFilter = enableBloomFilter(metaFieldsMode, config) + Option<org.apache.hudi.common.bloom.BloomFilter> bloomFilter = enableBloomFilter(metaFieldsMode, config, path) ? Option.of(createBloomFilter(config)) : Option.empty(); return new HoodieRowDataLanceWriter( path, diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowDataFileWriterFactory.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowDataFileWriterFactory.java new file mode 100644 index 000000000000..0b906a0a3062 --- /dev/null +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowDataFileWriterFactory.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.io.storage.row; + +import org.apache.hudi.common.avro.HoodieBloomFilterWriteSupport; +import org.apache.hudi.common.engine.LocalTaskContextSupplier; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.util.ParquetUtils; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.StringData; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests for {@link HoodieRowDataFileWriterFactory}. + */ +class TestHoodieRowDataFileWriterFactory { + + @TempDir + Path tmpDir; + + @Test + void testNativeLogBloomPolicy() throws Exception { + HoodieSchema schema = HoodieSchema.parse("{\"type\":\"record\",\"name\":\"test\",\"fields\":[" + + "{\"name\":\"_hoodie_record_key\",\"type\":\"string\"}]}"); + HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(tmpDir.toString()).build(); + config.setValue("hoodie.index.type", "GLOBAL_BLOOM"); + HoodieStorage storage = HoodieTestUtils.getStorage(tmpDir.toString()); + HoodieRowDataFileWriterFactory factory = new HoodieRowDataFileWriterFactory(storage); + for (String fileName : Arrays.asList("file-1_1-0-1_001.parquet", + "file-1_1-0-1_002_1.log.parquet", "file-1_1-0-1_002_1.deletes.parquet")) { + StoragePath path = new StoragePath(tmpDir.toString(), fileName); + try (HoodieRowDataParquetWriter writer = (HoodieRowDataParquetWriter) factory.newParquetFileWriter( + "002", path, config, schema, new LocalTaskContextSupplier())) { + writer.writeRow("key-1", GenericRowData.of(StringData.fromString("key-1"))); + } + Map<String, String> footer = ParquetUtils.readMetadata(storage, path).getFileMetaData().getKeyValueMetaData(); + boolean expectedBloom = fileName.equals("file-1_1-0-1_001.parquet"); + assertEquals(expectedBloom, footer.containsKey(HoodieBloomFilterWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY), fileName); + assertEquals(expectedBloom, footer.containsKey(HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER), fileName); + assertEquals(expectedBloom, footer.containsKey(HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER), fileName); + } + } +} diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java index ae9ad730d456..c27ee52a611f 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java @@ -113,7 +113,7 @@ public class HoodieSparkFileWriterFactory extends HoodieFileWriterFactory { } HoodieRowParquetWriteSupport writeSupport = getHoodieRowParquetWriteSupport(storageConfiguration, schema, - hoodieConfig, enableBloomFilter(metaFieldsMode, hoodieConfig)); + hoodieConfig, enableBloomFilter(metaFieldsMode, hoodieConfig, path)); HoodieRowParquetConfig parquetConfig = new HoodieRowParquetConfig(writeSupport, CompressionCodecName.fromConf(compressionCodecName), hoodieConfig.getIntOrDefault(HoodieStorageConfig.PARQUET_BLOCK_SIZE), @@ -166,7 +166,7 @@ public class HoodieSparkFileWriterFactory extends HoodieFileWriterFactory { MetaFieldsMode metaFieldsMode = MetaFieldsMode.resolve(config); boolean populateMetaFields = metaFieldsMode.toLegacyPopulateMetaFields(); StructType structType = HoodieInternalRowUtils.getCachedSchema(schema); - boolean enableBloomFilter = enableBloomFilter(metaFieldsMode, config); + boolean enableBloomFilter = enableBloomFilter(metaFieldsMode, config, path); Option<BloomFilter> bloomFilter = enableBloomFilter ? Option.of(createBloomFilter(config)) : Option.empty(); long maxFileSize = config.getLongOrDefault(HoodieStorageConfig.LANCE_MAX_FILE_SIZE); long allocatorSize = config.getLongOrDefault(HoodieStorageConfig.LANCE_WRITE_ALLOCATOR_SIZE_BYTES); @@ -192,7 +192,7 @@ public class HoodieSparkFileWriterFactory extends HoodieFileWriterFactory { MetaFieldsMode metaFieldsMode = MetaFieldsMode.resolve(config); boolean populateMetaFields = metaFieldsMode.toLegacyPopulateMetaFields(); StructType structType = HoodieInternalRowUtils.getCachedSchema(schema); - boolean enableBloomFilter = enableBloomFilter(metaFieldsMode, config); + boolean enableBloomFilter = enableBloomFilter(metaFieldsMode, config, path); Option<BloomFilter> bloomFilter = enableBloomFilter ? Option.of(createBloomFilter(config)) : Option.empty(); long maxFileSize = config.getLongOrDefault(HoodieStorageConfig.VORTEX_MAX_FILE_SIZE); long allocatorSize = config.getLongOrDefault(HoodieStorageConfig.VORTEX_WRITE_ALLOCATOR_SIZE_BYTES); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java index bf84ec4f9e29..859758f097bc 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java @@ -19,9 +19,13 @@ package org.apache.hudi.io.storage; import org.apache.hudi.client.SparkTaskContextSupplier; +import org.apache.hudi.common.avro.HoodieBloomFilterWriteSupport; import org.apache.hudi.common.config.HoodieStorageConfig; +import org.apache.hudi.common.engine.LocalTaskContextSupplier; import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; import org.apache.hudi.common.model.MetaFieldsMode; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.util.ParquetUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.core.io.storage.HoodieFileWriter; import org.apache.hudi.core.io.storage.HoodieFileWriterFactory; @@ -34,11 +38,19 @@ import org.apache.hudi.table.HoodieSparkTable; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestBase; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.util.Arrays; +import java.util.Map; import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.HOODIE_SCHEMA; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -88,34 +100,64 @@ public class TestHoodieAvroFileWriterFactory extends HoodieClientTestBase { assertTrue(thrown.getMessage().contains("format not supported yet.")); } - @Test - public void testEnableBloomFilter() { - HoodieWriteConfig config = getConfig(IndexType.BLOOM); - assertTrue(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config)); - assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.NONE, config)); - assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.COMMIT_TIME_ONLY, config)); - assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.FILE_NAME_ONLY, config)); - assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME, config)); + @ParameterizedTest + @CsvSource({"SIMPLE, true", "GLOBAL_BLOOM, false"}) + void testNativeLogBloomPolicy(String indexType, boolean baseBloomEnabled) throws Exception { + HoodieSchema schema = HoodieSchema.parse("{\"type\":\"record\",\"name\":\"test\",\"fields\":[" + + "{\"name\":\"_hoodie_record_key\",\"type\":\"string\"}]}"); + HoodieWriteConfig config = getConfigBuilder().withSchema(schema.toString()) + .withStorageConfig(HoodieStorageConfig.newBuilder().parquetBloomFilterEnable(baseBloomEnabled).build()).build(); + config.setValue("hoodie.index.type", indexType); + GenericRecord record = new GenericData.Record(schema.toAvroSchema()); + record.put("_hoodie_record_key", "key-1"); + for (String fileName : Arrays.asList("file-1_1-0-1_001.parquet", + "file-1_1-0-1_002_1.log.parquet", "file-1_1-0-1_002_1.deletes.parquet")) { + StoragePath path = new StoragePath(basePath, fileName); + try (HoodieFileWriter writer = HoodieFileWriterFactory.getFileWriter( + "002", path, storage, config, schema, new LocalTaskContextSupplier(), HoodieRecordType.AVRO)) { + writer.writeRow("key-1", record); + } + Map<String, String> footer = ParquetUtils.readMetadata(storage, path).getFileMetaData().getKeyValueMetaData(); + boolean expectedBloom = fileName.equals("file-1_1-0-1_001.parquet"); + assertEquals(expectedBloom, footer.containsKey(HoodieBloomFilterWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY), fileName); + assertEquals(expectedBloom, footer.containsKey(HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER), fileName); + assertEquals(expectedBloom, footer.containsKey(HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER), fileName); + } + } - config = getConfig(IndexType.SIMPLE); - assertTrue(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config)); + @ParameterizedTest + @ValueSource(strings = {"parquet", "lance", "vortex"}) + public void testEnableBloomFilter(String format) { + StoragePath path = new StoragePath(basePath, "file-1_1-0-1_001." + format); + StoragePath dataLogPath = new StoragePath(basePath, "file-1_1-0-1_002_1.log." + format); + StoragePath deleteLogPath = new StoragePath(basePath, "file-1_1-0-1_002_1.deletes." + format); + HoodieWriteConfig config = getConfig(IndexType.BLOOM); + assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config, dataLogPath)); + assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config, deleteLogPath)); + assertTrue(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config, path)); + assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.NONE, config, path)); + assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.COMMIT_TIME_ONLY, config, path)); + assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.FILE_NAME_ONLY, config, path)); + assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME, config, path)); config = getConfig(IndexType.SIMPLE); - assertTrue(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config)); + assertTrue(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config, path)); + assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config, dataLogPath)); + assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config, deleteLogPath)); config = getConfigBuilder(IndexType.BLOOM) .withStorageConfig(HoodieStorageConfig.newBuilder() .parquetBloomFilterEnable(false).build()).build(); - assertTrue(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config)); + assertTrue(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config, path)); config = getConfigBuilder(IndexType.SIMPLE) .withStorageConfig(HoodieStorageConfig.newBuilder() .parquetBloomFilterEnable(true).build()).build(); - assertTrue(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config)); + assertTrue(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config, path)); config = getConfigBuilder(IndexType.SIMPLE) .withStorageConfig(HoodieStorageConfig.newBuilder() .parquetBloomFilterEnable(false).build()).build(); - assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config)); + assertFalse(HoodieFileWriterFactory.enableBloomFilter(MetaFieldsMode.ALL, config, path)); } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java index c5b14c6ffe50..003bc125462c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java @@ -271,7 +271,8 @@ public class HoodieStorageConfig extends HoodieConfig { .markAdvanced() .sinceVersion("0.15.0") .withDocumentation("Control whether to write bloom filter or not. Default true. " - + "We can set to false in non bloom index cases for CPU resource saving."); + + "We can set to false in non bloom index cases for CPU resource saving. " + + "Native Parquet log files do not write record-key bloom filters."); public static final ConfigProperty<Boolean> PARQUET_VARIANT_WRITE_SHREDDING_ENABLED = ConfigProperty .key("hoodie.parquet.variant.write.shredding.enabled") diff --git a/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieFileWriterFactory.java b/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieFileWriterFactory.java index 7b1fc9859106..e29cc5ec1ff3 100644 --- a/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieFileWriterFactory.java +++ b/hudi-common/src/main/java/org/apache/hudi/core/io/storage/HoodieFileWriterFactory.java @@ -137,10 +137,11 @@ public class HoodieFileWriterFactory { } /** - * Check if need to enable bloom filter. + * Check whether to enable Hudi record-key bloom filters, excluding native logs since bloom indexes only probe base files. */ - public static boolean enableBloomFilter(MetaFieldsMode metaFieldsMode, HoodieConfig config) { - return metaFieldsMode.isRecordKeyPopulated() + public static boolean enableBloomFilter(MetaFieldsMode metaFieldsMode, HoodieConfig config, StoragePath path) { + return !FSUtils.isNativeLogFile(path.getName()) + && metaFieldsMode.isRecordKeyPopulated() && (config.getBooleanOrDefault(HoodieStorageConfig.PARQUET_WITH_BLOOM_FILTER_ENABLED) // HoodieIndexConfig is located in the package hudi-client-common, and the package hudi-client-common depends on the package hudi-common, // so the class HoodieIndexConfig cannot be accessed in hudi-common, otherwise there will be a circular dependency problem diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java index af5abdbb191c..8e3655927038 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java @@ -30,6 +30,7 @@ import org.apache.hudi.common.config.HoodieParquetConfig; import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.engine.TaskContextSupplier; +import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableConfig; @@ -116,7 +117,8 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory { StorageConfiguration storageConfiguration = injectedConfigs.getLeft(); HoodieConfig hoodieConfig = injectedConfigs.getRight(); - HoodieAvroWriteSupport writeSupport = getHoodieAvroWriteSupport(schema, hoodieConfig, storageConfiguration, enableBloomFilter(metaFieldsMode, hoodieConfig)); + HoodieAvroWriteSupport writeSupport = getHoodieAvroWriteSupport(schema, hoodieConfig, storageConfiguration, + enableBloomFilter(metaFieldsMode, hoodieConfig, path)); String compressionCodecName = hoodieConfig.getString(HoodieStorageConfig.PARQUET_COMPRESSION_CODEC_NAME); // Support PARQUET_COMPRESSION_CODEC_NAME is "" @@ -170,7 +172,8 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory { protected HoodieFileWriter newOrcFileWriter( String instantTime, StoragePath path, HoodieConfig config, HoodieSchema schema, TaskContextSupplier taskContextSupplier) throws IOException { - BloomFilter filter = createBloomFilter(config); + // bloom indexes only consume base files; native ORC log readers do not use this filter. + BloomFilter filter = FSUtils.isNativeLogFile(path.getName()) ? null : createBloomFilter(config); HoodieOrcConfig orcConfig = new HoodieOrcConfig(storage.getConf(), CompressionKind.valueOf(config.getString(HoodieStorageConfig.ORC_COMPRESSION_CODEC_NAME)), config.getInt(HoodieStorageConfig.ORC_STRIPE_SIZE), diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java index d27789634dd2..5e5bcfabd782 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java @@ -25,12 +25,14 @@ import org.apache.hudi.common.bloom.BloomFilterFactory; import org.apache.hudi.common.bloom.BloomFilterTypeCode; import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.engine.LocalTaskContextSupplier; +import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.core.io.storage.HoodieAvroFileReader; import org.apache.hudi.core.io.storage.HoodieAvroFileWriter; +import org.apache.hudi.core.io.storage.HoodieFileWriterFactory; import org.apache.hudi.core.io.storage.HoodieIOFactory; import org.apache.hudi.core.io.storage.HoodieOrcConfig; import org.apache.hudi.io.storage.hadoop.HoodieAvroOrcWriter; @@ -38,12 +40,16 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.storage.StoragePath; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.orc.CompressionKind; import org.apache.orc.OrcFile; import org.apache.orc.Reader; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import java.io.IOException; import java.util.Collections; @@ -109,6 +115,31 @@ public class TestHoodieOrcReaderWriter extends TestHoodieReaderWriterBase { } } + @ParameterizedTest + @CsvSource({"log, false", "deletes, false", "base, true"}) + void testNativeLogBloomPolicy(String fileType, boolean expectedBloom) throws Exception { + HoodieSchema schema = HoodieSchema.parse("{\"type\":\"record\",\"name\":\"test\",\"fields\":[" + + "{\"name\":\"_hoodie_record_key\",\"type\":\"string\"}]}"); + HoodieStorageConfig config = HoodieStorageConfig.newBuilder().build(); + config.setValue("hoodie.index.type", "GLOBAL_BLOOM"); + String fileName = "base".equals(fileType) ? "file-1_1-0-1_001.orc" + : FSUtils.makeNativeLogFileName("file-1", "1-0-1", "002", 1, fileType, HoodieFileFormat.ORC); + StoragePath path = new StoragePath(tempDir.toString(), fileName); + HoodieStorage storage = HoodieTestUtils.getStorage(path); + GenericRecord record = new GenericData.Record(schema.toAvroSchema()); + record.put("_hoodie_record_key", "key-1"); + try (HoodieAvroFileWriter writer = (HoodieAvroFileWriter) HoodieFileWriterFactory.getFileWriter( + "002", path, storage, config, schema, new LocalTaskContextSupplier(), HoodieRecordType.AVRO)) { + writer.writeAvro("key-1", record); + } + try (Reader reader = OrcFile.createReader(new Path(path.toUri()), OrcFile.readerOptions(storage.getConf().unwrapAs(Configuration.class)))) { + assertEquals(expectedBloom, reader.hasMetadataValue(HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY)); + assertEquals(expectedBloom, reader.hasMetadataValue(HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER)); + assertEquals(expectedBloom, reader.hasMetadataValue(HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER)); + assertEquals(1L, reader.getNumberOfRows()); + } + } + @Test public void testAddFooterMetadata() throws Exception { HoodieSchema schema = getHoodieSchemaFromResource(TestHoodieOrcReaderWriter.class, "/exampleSchema.avsc"); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkFileWriterFactory.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkFileWriterFactory.java new file mode 100644 index 000000000000..22c81aadbe68 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkFileWriterFactory.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.io.storage; + +import org.apache.hudi.common.avro.HoodieBloomFilterWriteSupport; +import org.apache.hudi.common.config.HoodieStorageConfig; +import org.apache.hudi.common.engine.LocalTaskContextSupplier; +import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.util.ParquetUtils; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.core.io.storage.HoodieFileWriter; +import org.apache.hudi.core.io.storage.HoodieFileWriterFactory; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; +import org.apache.spark.unsafe.types.UTF8String; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests for {@link HoodieSparkFileWriterFactory}. + */ +class TestHoodieSparkFileWriterFactory { + + @TempDir + Path tmpDir; + + @ParameterizedTest + @ValueSource(strings = {"BLOOM", "SIMPLE"}) + void testNativeLogBloomPolicy(String indexType) throws Exception { + HoodieSchema schema = HoodieSchema.parse("{\"type\":\"record\",\"name\":\"test\",\"fields\":[" + + "{\"name\":\"_hoodie_record_key\",\"type\":\"string\"}]}"); + HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(tmpDir.toString()).withSchema(schema.toString()) + .withStorageConfig(HoodieStorageConfig.newBuilder().parquetBloomFilterEnable(false).build()).build(); + config.setValue("hoodie.index.type", indexType); + HoodieStorage storage = HoodieTestUtils.getStorage(tmpDir.toString()); + InternalRow row = new GenericInternalRow(new Object[] {UTF8String.fromString("key-1")}); + for (String fileName : Arrays.asList("file-1_1-0-1_001.parquet", + "file-1_1-0-1_002_1.log.parquet", "file-1_1-0-1_002_1.deletes.parquet")) { + StoragePath path = new StoragePath(tmpDir.toString(), fileName); + try (HoodieFileWriter writer = HoodieFileWriterFactory.getFileWriter( + "002", path, storage, config, schema, new LocalTaskContextSupplier(), HoodieRecordType.SPARK)) { + writer.writeRow("key-1", row); + } + Map<String, String> footer = ParquetUtils.readMetadata(storage, path).getFileMetaData().getKeyValueMetaData(); + boolean expectedBloom = fileName.equals("file-1_1-0-1_001.parquet") && indexType.contains("BLOOM"); + assertEquals(expectedBloom, footer.containsKey(HoodieBloomFilterWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY), fileName); + assertEquals(expectedBloom, footer.containsKey(HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER), fileName); + assertEquals(expectedBloom, footer.containsKey(HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER), fileName); + } + } +}
