This is an automated email from the ASF dual-hosted git repository.
vbalaji pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 3ca1fb39564 [HUDI-7262] Validate checksum only if it exists (#10417)
3ca1fb39564 is described below
commit 3ca1fb395649d5c7ed2bb44c66d52f8d4168b291
Author: Jing Zhang <[email protected]>
AuthorDate: Thu Dec 28 05:58:07 2023 +0800
[HUDI-7262] Validate checksum only if it exists (#10417)
---
.../org/apache/hudi/common/util/ConfigUtils.java | 6 +++-
.../hudi/common/table/TestHoodieTableConfig.java | 6 ++--
.../TestUpgradeOrDowngradeProcedure.scala | 36 ++++++++++++++++++++--
3 files changed, 43 insertions(+), 5 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java
index 518ff5e2c8e..c5bdb42ef3c 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java
@@ -45,6 +45,8 @@ import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
+import static org.apache.hudi.common.table.HoodieTableConfig.TABLE_CHECKSUM;
+
public class ConfigUtils {
public static final String STREAMER_CONFIG_PREFIX = "hoodie.streamer.";
@Deprecated
@@ -565,7 +567,9 @@ public class ConfigUtils {
props.clear();
props.load(is);
found = true;
-
ValidationUtils.checkArgument(HoodieTableConfig.validateChecksum(props));
+ if (props.containsKey(TABLE_CHECKSUM.key())) {
+
ValidationUtils.checkArgument(HoodieTableConfig.validateChecksum(props));
+ }
return props;
} catch (IOException e) {
LOG.warn(String.format("Could not read properties from %s: %s",
path, e));
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
index 4eb6d4b7b7b..040ad775474 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
@@ -40,6 +40,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import static org.apache.hudi.common.table.HoodieTableConfig.TABLE_CHECKSUM;
import static org.apache.hudi.common.util.ConfigUtils.recoverIfNeeded;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -161,14 +162,15 @@ public class TestHoodieTableConfig extends
HoodieCommonTestHarness {
// Should return backup config if hoodie.properties is corrupted
Properties props = new Properties();
+ props.put(TABLE_CHECKSUM.key(), "0");
try (FSDataOutputStream out = fs.create(cfgPath)) {
- props.store(out, "No checksum in file so is invalid");
+ props.store(out, "Wrong checksum in file so is invalid");
}
new HoodieTableConfig(fs, metaPath.toString(), null, null);
// Should throw exception if both hoodie.properties and backup are
corrupted
try (FSDataOutputStream out = fs.create(backupCfgPath)) {
- props.store(out, "No checksum in file so is invalid");
+ props.store(out, "Wrong checksum in file so is invalid");
}
assertThrows(IllegalArgumentException.class, () -> new
HoodieTableConfig(fs, metaPath.toString(), null, null));
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
index 1bd29cabc40..20fcd820337 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
@@ -20,9 +20,11 @@ package org.apache.spark.sql.hudi.procedure
import org.apache.hadoop.fs.Path
import org.apache.hudi.common.config.HoodieConfig
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient,
HoodieTableVersion}
+import org.apache.hudi.common.util.{BinaryUtil, ConfigUtils, StringUtils}
import org.apache.spark.api.java.JavaSparkContext
import java.io.IOException
+import java.time.Instant
class TestUpgradeOrDowngradeProcedure extends HoodieSparkProcedureTestBase {
@@ -104,8 +106,38 @@ class TestUpgradeOrDowngradeProcedure extends
HoodieSparkProcedureTestBase {
// downgrade table to THREE
checkAnswer(s"""call downgrade_table(table => '$tableName', to_version
=> 'THREE')""")(Seq(true))
- // upgrade table to FOUR
- checkAnswer(s"""call upgrade_table(table => '$tableName', to_version =>
'FOUR')""")(Seq(true))
+ var metaClient = HoodieTableMetaClient.builder
+ .setConf(new
JavaSparkContext(spark.sparkContext).hadoopConfiguration())
+ .setBasePath(tablePath)
+ .build
+ // verify hoodie.table.version of the table is THREE
+ assertResult(HoodieTableVersion.THREE.versionCode) {
+ metaClient.getTableConfig.getTableVersion.versionCode()
+ }
+ val metaPathDir = new Path(metaClient.getBasePath,
HoodieTableMetaClient.METAFOLDER_NAME)
+ // delete checksum from hoodie.properties
+ val props = ConfigUtils.fetchConfigs(metaClient.getFs,
metaPathDir.toString, HoodieTableConfig.HOODIE_PROPERTIES_FILE,
HoodieTableConfig.HOODIE_PROPERTIES_FILE_BACKUP, 1, 1000)
+ props.remove(HoodieTableConfig.TABLE_CHECKSUM.key)
+ try {
+ val outputStream = metaClient.getFs.create(new Path(metaPathDir,
HoodieTableConfig.HOODIE_PROPERTIES_FILE))
+ props.store(outputStream, "Updated at " + Instant.now)
+ outputStream.close()
+ } catch {
+ case e: Exception => fail(e)
+ }
+ // verify hoodie.table.checksum is deleted from hoodie.properties
+ metaClient = HoodieTableMetaClient.reload(metaClient)
+ assertResult(false)
{metaClient.getTableConfig.contains(HoodieTableConfig.TABLE_CHECKSUM)}
+ // upgrade table to SIX
+ checkAnswer(s"""call upgrade_table(table => '$tableName', to_version =>
'SIX')""")(Seq(true))
+ metaClient = HoodieTableMetaClient.reload(metaClient)
+ assertResult(HoodieTableVersion.SIX.versionCode) {
+ metaClient.getTableConfig.getTableVersion.versionCode()
+ }
+ val expectedCheckSum =
BinaryUtil.generateChecksum(StringUtils.getUTF8Bytes(tableName))
+ assertResult(expectedCheckSum) {
+ metaClient.getTableConfig.getLong(HoodieTableConfig.TABLE_CHECKSUM)
+ }
}
}
