This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new d2e9865d0c8 [HUDI-6257] Upgrade table version partition path check
should consider hive style partitioning (#8794)
d2e9865d0c8 is described below
commit d2e9865d0c88fb8504626f8ff1e398128c2d30cb
Author: KnightChess <[email protected]>
AuthorDate: Tue May 30 10:03:50 2023 +0800
[HUDI-6257] Upgrade table version partition path check should consider hive
style partitioning (#8794)
---
.../table/upgrade/FourToFiveUpgradeHandler.java | 23 +++++++++++--
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 38 ++++++++++++++++++----
2 files changed, 53 insertions(+), 8 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FourToFiveUpgradeHandler.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FourToFiveUpgradeHandler.java
index d803dd9ac22..01183357e7a 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FourToFiveUpgradeHandler.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FourToFiveUpgradeHandler.java
@@ -21,8 +21,10 @@ package org.apache.hudi.table.upgrade;
import org.apache.hudi.common.config.ConfigProperty;
import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.table.HoodieTable;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -46,8 +48,9 @@ public class FourToFiveUpgradeHandler implements
UpgradeHandler {
@Override
public Map<ConfigProperty, String> upgrade(HoodieWriteConfig config,
HoodieEngineContext context, String instantTime, SupportsUpgradeDowngrade
upgradeDowngradeHelper) {
try {
- FileSystem fs = new
Path(config.getBasePath()).getFileSystem(context.getHadoopConf().get());
- if (!config.doSkipDefaultPartitionValidation() && fs.exists(new
Path(config.getBasePath() + "/" + DEPRECATED_DEFAULT_PARTITION_PATH))) {
+ HoodieTable table = upgradeDowngradeHelper.getTable(config, context);
+
+ if (!config.doSkipDefaultPartitionValidation() &&
hasDefaultPartitionPath(config, table)) {
LOG.error(String.format("\"%s\" partition detected. From 0.12, we are
changing the default partition in hudi to %s "
+ " Please read and write back the data in \"%s\" partition in
hudi to new partition path \"%s\". \"\n"
+ " Sample spark command to use to re-write the data: \n\n"
@@ -71,4 +74,20 @@ public class FourToFiveUpgradeHandler implements
UpgradeHandler {
}
return new HashMap<>();
}
+
+ private boolean hasDefaultPartitionPath(HoodieWriteConfig config,
HoodieTable table) throws IOException {
+ HoodieTableConfig tableConfig = table.getMetaClient().getTableConfig();
+ if (!tableConfig.getPartitionFields().isPresent()) {
+ return false;
+ }
+ String checkPartitionPath = DEPRECATED_DEFAULT_PARTITION_PATH;
+ boolean hiveStylePartitioningEnable =
Boolean.parseBoolean(tableConfig.getHiveStylePartitioningEnable());
+ // dt=default/ht=default, only need check dt=default
+ if (hiveStylePartitioningEnable) {
+ String[] partitions = tableConfig.getPartitionFields().get();
+ checkPartitionPath = partitions[0] + "=" +
DEPRECATED_DEFAULT_PARTITION_PATH;
+ }
+ FileSystem fs = new
Path(config.getBasePath()).getFileSystem(table.getHadoopConf());
+ return fs.exists(new Path(config.getBasePath() + "/" +
checkPartitionPath));
+ }
}
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
index 8dae47247a6..9d837565363 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
@@ -83,6 +83,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.hudi.common.table.HoodieTableConfig.BASE_FILE_FORMAT;
+import static
org.apache.hudi.common.table.HoodieTableConfig.HIVE_STYLE_PARTITIONING_ENABLE;
import static org.apache.hudi.common.table.HoodieTableConfig.TYPE;
import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
import static
org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
@@ -338,20 +339,30 @@ public class TestUpgradeDowngrade extends
HoodieClientTestBase {
@Test
public void testUpgradeFourtoFive() throws Exception {
- testUpgradeFourToFiveInternal(false, false);
+ testUpgradeFourToFiveInternal(false, false, false);
}
@Test
public void testUpgradeFourtoFiveWithDefaultPartition() throws Exception {
- testUpgradeFourToFiveInternal(true, false);
+ testUpgradeFourToFiveInternal(true, false, false);
}
@Test
public void testUpgradeFourtoFiveWithDefaultPartitionWithSkipValidation()
throws Exception {
- testUpgradeFourToFiveInternal(true, true);
+ testUpgradeFourToFiveInternal(true, true, false);
}
- private void testUpgradeFourToFiveInternal(boolean assertDefaultPartition,
boolean skipDefaultPartitionValidation) throws Exception {
+ @Test
+ public void testUpgradeFourtoFiveWithHiveStyleDefaultPartition() throws
Exception {
+ testUpgradeFourToFiveInternal(true, false, true);
+ }
+
+ @Test
+ public void
testUpgradeFourtoFiveWithHiveStyleDefaultPartitionWithSkipValidation() throws
Exception {
+ testUpgradeFourToFiveInternal(true, true, true);
+ }
+
+ private void testUpgradeFourToFiveInternal(boolean assertDefaultPartition,
boolean skipDefaultPartitionValidation, boolean isHiveStyle) throws Exception {
String tableName = metaClient.getTableConfig().getTableName();
// clean up and re instantiate meta client w/ right table props
cleanUp();
@@ -366,14 +377,19 @@ public class TestUpgradeDowngrade extends
HoodieClientTestBase {
initMetaClient(getTableType(), properties);
// init config, table and client.
- HoodieWriteConfig cfg =
getConfigBuilder().withAutoCommit(false).withRollbackUsingMarkers(false)
+ HoodieWriteConfig cfg =
getConfigBuilder().withAutoCommit(true).withRollbackUsingMarkers(false)
.doSkipDefaultPartitionValidation(skipDefaultPartitionValidation).withProps(params).build();
SparkRDDWriteClient client = getHoodieWriteClient(cfg);
// Write inserts
doInsert(client);
if (assertDefaultPartition) {
- doInsertWithDefaultPartition(client);
+ if (isHiveStyle) {
+ doInsertWithDefaultHiveStylePartition(client);
+ cfg.setValue(HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
+ } else {
+ doInsertWithDefaultPartition(client);
+ }
}
// downgrade table props
@@ -429,6 +445,16 @@ public class TestUpgradeDowngrade extends
HoodieClientTestBase {
client.insert(writeRecords, commit1).collect();
}
+ private void doInsertWithDefaultHiveStylePartition(SparkRDDWriteClient
client) {
+ // Write 1 (only inserts)
+ dataGen = new HoodieTestDataGenerator(new String[]{"partition_path=" +
DEPRECATED_DEFAULT_PARTITION_PATH});
+ String commit1 = "005";
+ client.startCommitWithTime(commit1);
+ List<HoodieRecord> records = dataGen.generateInserts(commit1, 100);
+ JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1);
+ client.insert(writeRecords, commit1).collect();
+ }
+
private void downgradeTableConfigsFromTwoToOne(HoodieWriteConfig cfg) throws
IOException {
Properties properties = new Properties(cfg.getProps());
properties.remove(HoodieTableConfig.RECORDKEY_FIELDS.key());
