This is an automated email from the ASF dual-hosted git repository. yuzhaojing pushed a commit to branch release-0.12.1-rc1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 070a395c5eb51ae2edacdbb932bebe6d86c59409 Author: Manu <[email protected]> AuthorDate: Tue Sep 27 07:45:12 2022 +0800 [HUDI-4902] Set default partitioner for SIMPLE BUCKET index (#6759) --- .../java/org/apache/hudi/config/HoodieLayoutConfig.java | 11 ++++++++++- .../org/apache/hudi/config/TestHoodieWriteConfig.java | 17 +++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLayoutConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLayoutConfig.java index 1d0810696d..0579ee3d62 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLayoutConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLayoutConfig.java @@ -48,6 +48,9 @@ public class HoodieLayoutConfig extends HoodieConfig { .noDefaultValue() .withDocumentation("Partitioner class, it is used to distribute data in a specific way."); + public static final String SIMPLE_BUCKET_LAYOUT_PARTITIONER_CLASS_NAME = + "org.apache.hudi.table.action.commit.SparkBucketIndexPartitioner"; + private HoodieLayoutConfig() { super(); } @@ -80,8 +83,14 @@ public class HoodieLayoutConfig extends HoodieConfig { } private void setDefault() { - if (layoutConfig.contains(HoodieIndexConfig.INDEX_TYPE.key()) && layoutConfig.getString(HoodieIndexConfig.INDEX_TYPE.key()).equals(HoodieIndex.IndexType.BUCKET.name())) { + if (layoutConfig.contains(HoodieIndexConfig.INDEX_TYPE.key()) + && layoutConfig.getString(HoodieIndexConfig.INDEX_TYPE.key()).equals(HoodieIndex.IndexType.BUCKET.name())) { layoutConfig.setDefaultValue(LAYOUT_TYPE, HoodieStorageLayout.LayoutType.BUCKET.name()); + + // Currently, the partitioner of the SIMPLE bucket index is supported by SparkBucketIndexPartitioner only. + if ("SIMPLE".equals(layoutConfig.getString(HoodieIndexConfig.BUCKET_INDEX_ENGINE_TYPE))) { + layoutConfig.setDefaultValue(LAYOUT_PARTITIONER_CLASS_NAME, SIMPLE_BUCKET_LAYOUT_PARTITIONER_CLASS_NAME); + } } layoutConfig.setDefaultValue(LAYOUT_TYPE, LAYOUT_TYPE.defaultValue()); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java index 0adbb998a0..e956668d0c 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java @@ -371,6 +371,23 @@ public class TestHoodieWriteConfig { HoodieFailedWritesCleaningPolicy.LAZY, FileSystemBasedLockProviderTestClass.class.getName()); } + @Test + public void testSimpleBucketIndexPartitionerConfig() { + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath("/tmp") + .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BUCKET) + .withBucketIndexEngineType(HoodieIndex.BucketIndexEngineType.SIMPLE).build()) + .build(); + assertEquals(HoodieLayoutConfig.SIMPLE_BUCKET_LAYOUT_PARTITIONER_CLASS_NAME, writeConfig.getString(HoodieLayoutConfig.LAYOUT_PARTITIONER_CLASS_NAME)); + + HoodieWriteConfig overwritePartitioner = HoodieWriteConfig.newBuilder().withPath("/tmp") + .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BUCKET) + .withBucketIndexEngineType(HoodieIndex.BucketIndexEngineType.SIMPLE) + .build()) + .withLayoutConfig(HoodieLayoutConfig.newBuilder().withLayoutPartitioner("org.apache.hudi.table.action.commit.UpsertPartitioner").build()) + .build(); + assertEquals("org.apache.hudi.table.action.commit.UpsertPartitioner", overwritePartitioner.getString(HoodieLayoutConfig.LAYOUT_PARTITIONER_CLASS_NAME)); + } + private HoodieWriteConfig createWriteConfig(Map<String, String> configs) { final Properties properties = new Properties(); configs.forEach(properties::setProperty);
