This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 2e9e1ab1d1d7 feat: Change the config for record index max file group
size to be a long (#17461)
2e9e1ab1d1d7 is described below
commit 2e9e1ab1d1d73ec92c4fb2037d4af747fea05a43
Author: Prashant Wason <[email protected]>
AuthorDate: Tue Dec 2 22:49:58 2025 -0800
feat: Change the config for record index max file group size to be a long
(#17461)
---
.../org/apache/hudi/config/HoodieWriteConfig.java | 2 +-
.../hudi/common/config/HoodieMetadataConfig.java | 8 ++---
.../hudi/metadata/HoodieTableMetadataUtil.java | 2 +-
.../common/config/TestHoodieMetadataConfig.java | 34 ++++++++++++++++++++++
4 files changed, 40 insertions(+), 6 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 81d10c059001..f6bc28b53e4f 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -2703,7 +2703,7 @@ public class HoodieWriteConfig extends HoodieConfig {
return metadataConfig.getRecordIndexGrowthFactor();
}
- public int getRecordIndexMaxFileGroupSizeBytes() {
+ public long getRecordIndexMaxFileGroupSizeBytes() {
return metadataConfig.getRecordIndexMaxFileGroupSizeBytes();
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
index 9e161afa983b..ba5152440467 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
@@ -322,9 +322,9 @@ public final class HoodieMetadataConfig extends
HoodieConfig {
.sinceVersion("1.1.0")
.withDocumentation("Maximum number of file groups to use for Partitioned
Record Index.");
- public static final ConfigProperty<Integer>
RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP = ConfigProperty
+ public static final ConfigProperty<Long>
RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP = ConfigProperty
.key(METADATA_PREFIX + ".record.index.max.filegroup.size")
- .defaultValue(1024 * 1024 * 1024)
+ .defaultValue(1024 * 1024 * 1024L)
.markAdvanced()
.sinceVersion("0.14.0")
.withDocumentation("Maximum size in bytes of a single file group. Large
file group takes longer to compact.");
@@ -708,8 +708,8 @@ public final class HoodieMetadataConfig extends
HoodieConfig {
return getFloat(RECORD_INDEX_GROWTH_FACTOR_PROP);
}
- public int getRecordIndexMaxFileGroupSizeBytes() {
- return getInt(RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP);
+ public long getRecordIndexMaxFileGroupSizeBytes() {
+ return getLong(RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP);
}
public String getSplliableMapDir() {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
index 6f45e5fa3f7c..48fe18d91c2c 100644
---
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
+++
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
@@ -2374,7 +2374,7 @@ public class HoodieTableMetadataUtil {
* @return The estimated number of file groups.
*/
public static int estimateFileGroupCount(MetadataPartitionType
partitionType, Supplier<Long> recordCountSupplier, int averageRecordSize, int
minFileGroupCount,
- int maxFileGroupCount, float
growthFactor, int maxFileGroupSizeBytes) {
+ int maxFileGroupCount, float
growthFactor, long maxFileGroupSizeBytes) {
int fileGroupCount;
long recordCount = -1;
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java
b/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java
index fd3e0fdb961e..89a1d0761828 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java
@@ -106,4 +106,38 @@ class TestHoodieMetadataConfig {
.build();
assertTrue(configWithCustomValue.isGlobalRecordLevelIndexEnabled());
}
+
+ @Test
+ void testRecordIndexMaxFileGroupSizeBytes() {
+ // Test default value (1GB)
+ HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().build();
+ assertEquals(1024L * 1024L * 1024L,
config.getRecordIndexMaxFileGroupSizeBytes());
+
+ // Test custom value using builder method
+ long customSize = 2L * 1024L * 1024L * 1024L; // 2GB
+ HoodieMetadataConfig configWithBuilder = HoodieMetadataConfig.newBuilder()
+ .withRecordIndexMaxFileGroupSizeBytes(customSize)
+ .build();
+ assertEquals(customSize,
configWithBuilder.getRecordIndexMaxFileGroupSizeBytes());
+
+ // Test custom value via Properties
+ Properties props = new Properties();
+
props.put(HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key(),
String.valueOf(customSize));
+ HoodieMetadataConfig configWithProperties =
HoodieMetadataConfig.newBuilder()
+ .fromProperties(props)
+ .build();
+ assertEquals(customSize,
configWithProperties.getRecordIndexMaxFileGroupSizeBytes());
+
+ // Test value larger than Integer.MAX_VALUE to ensure long is properly
handled
+ long largeSize = 3L * 1024L * 1024L * 1024L; // 3GB (exceeds
Integer.MAX_VALUE which is ~2.1GB)
+ Properties propsLarge = new Properties();
+
propsLarge.put(HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key(),
String.valueOf(largeSize));
+ HoodieMetadataConfig configWithLargeValue =
HoodieMetadataConfig.newBuilder()
+ .fromProperties(propsLarge)
+ .build();
+ assertEquals(largeSize,
configWithLargeValue.getRecordIndexMaxFileGroupSizeBytes());
+
+ // Verify that the value is indeed larger than Integer.MAX_VALUE
+ assertTrue(largeSize > Integer.MAX_VALUE, "Test value should exceed
Integer.MAX_VALUE to validate long type");
+ }
}
