This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch release-1.1.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 256523affc3e2a3269488ac289dec8566d033aae Author: Trivedhi <[email protected]> AuthorDate: Mon Oct 27 08:49:43 2025 +0530 fix: Fixed the recovering method for the older versions where checksum is not present (#14148) --- .../org/apache/hudi/common/util/ConfigUtils.java | 15 ++++++++++-- .../apache/hudi/common/util/TestConfigUtils.java | 27 ++++++++++++++++++++++ .../hudi/common/table/TestHoodieTableConfig.java | 13 ++++++++--- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java index 0aa9e979e9da..d480d56d83dc 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java @@ -59,6 +59,7 @@ import static org.apache.hudi.common.config.HoodieCommonConfig.DISK_MAP_BITCASK_ import static org.apache.hudi.common.config.HoodieCommonConfig.SPILLABLE_DISK_MAP_TYPE; import static org.apache.hudi.common.config.HoodieMemoryConfig.MAX_MEMORY_FOR_MERGE; import static org.apache.hudi.common.config.HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH; +import static org.apache.hudi.common.table.HoodieTableConfig.NAME; import static org.apache.hudi.common.table.HoodieTableConfig.TABLE_CHECKSUM; import static org.apache.hudi.keygen.constant.KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED; @@ -705,6 +706,16 @@ public class ConfigUtils { } } + public static boolean isPropertiesInvalid(TypedProperties props) { + // For older versions if checksum is not present, table name needs to be present to generate the checksum + if (!props.containsKey(TABLE_CHECKSUM.key())) { + return !props.containsKey(NAME.key()); + } + + // If checkSum is present we need to validate + return !HoodieTableConfig.validateChecksum(props); + } + public static void recoverIfNeeded(HoodieStorage storage, StoragePath cfgPath, StoragePath backupCfgPath) throws IOException { boolean needCopy = false; @@ -714,7 +725,7 @@ public class ConfigUtils { TypedProperties props = new TypedProperties(); try (InputStream in = storage.open(cfgPath)) { props.load(in); - if (!props.containsKey(TABLE_CHECKSUM.key()) || !HoodieTableConfig.validateChecksum(props)) { + if (isPropertiesInvalid(props)) { // the cfg file is invalid storage.deleteFile(cfgPath); needCopy = true; @@ -727,7 +738,7 @@ public class ConfigUtils { try (InputStream backupStream = new ByteArrayInputStream(bytes)) { TypedProperties backupProps = new TypedProperties(); backupProps.load(backupStream); - if (!backupProps.containsKey(TABLE_CHECKSUM.key()) || !HoodieTableConfig.validateChecksum(backupProps)) { + if (isPropertiesInvalid(backupProps)) { // need to delete the backup as anyway reads will also fail // subsequent writes will recover and update storage.deleteFile(backupCfgPath); diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestConfigUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestConfigUtils.java index 5bf1dd2ac958..ea2b4a1ef85f 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestConfigUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestConfigUtils.java @@ -427,4 +427,31 @@ public class TestConfigUtils { assertEquals(1, props.size()); assertEquals("overwrite", props.get("strategy")); } + + @Test + void testIsPropertiesInvalid() { + TypedProperties props = new TypedProperties(); + // Case-1 : Empty properties - should be invalid + assertTrue(ConfigUtils.isPropertiesInvalid(props)); + + // Case-2 : Valid properties with valid checksum + props.setProperty(HoodieTableConfig.NAME.key(), "test_db.test_table"); + props.setProperty(HoodieTableConfig.TYPE.key(), "COPY_ON_WRITE"); + props.setProperty(HoodieTableConfig.VERSION.key(), "6"); + props.setProperty(HoodieTableConfig.TABLE_CHECKSUM.key(), String.valueOf(HoodieTableConfig.generateChecksum(props))); + + assertFalse(ConfigUtils.isPropertiesInvalid(props)); + + // Case-3 : Invalid checksum + props.setProperty(HoodieTableConfig.TABLE_CHECKSUM.key(), "0"); + assertTrue(ConfigUtils.isPropertiesInvalid(props)); + + // Case-4: without checksum property, valid properties + props.remove(HoodieTableConfig.TABLE_CHECKSUM.key()); + assertFalse(ConfigUtils.isPropertiesInvalid(props)); + + // Case-5: without checksum property, invalid properties + props.remove(HoodieTableConfig.NAME.key()); + assertTrue(ConfigUtils.isPropertiesInvalid(props)); + } } \ No newline at end of file diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java index a59ae8c5c3f2..b240bb2dfabd 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java @@ -21,6 +21,7 @@ package org.apache.hudi.common.table; import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.config.HoodieConfig; import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.AWSDmsAvroPayload; import org.apache.hudi.common.model.DefaultHoodieRecordPayload; import org.apache.hudi.common.model.EventTimeAvroPayload; @@ -50,6 +51,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.util.Arrays; import java.util.Collections; @@ -257,10 +259,15 @@ class TestHoodieTableConfig extends HoodieCommonTestHarness { assertEquals(7, config.getProps().size()); // 2. Backup properties file is also invalid + try (InputStream in = storage.open(cfgPath); + OutputStream out = storage.create(backupCfgPath)) { + TypedProperties props = new TypedProperties(); + props.load(in); + // Keeping the invalid checksum + props.setProperty(TABLE_CHECKSUM.key(), "000000000"); + props.store(out, ""); + } storage.deleteFile(cfgPath); - // create the empty files - storage.create(cfgPath); - storage.create(backupCfgPath); assertThrows(IOException.class, () -> recoverIfNeeded(storage, cfgPath, backupCfgPath)); assertFalse(storage.exists(backupCfgPath)); assertFalse(storage.exists(cfgPath));
