This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new bcfd3545e60 [HUDI-6236] Write hive_style_partitioning_enable to table
config in DeltaStreamer entrypoints (#8752)
bcfd3545e60 is described below
commit bcfd3545e6034adfe5a1196e71c19339fd00ef08
Author: voonhous <[email protected]>
AuthorDate: Sat May 20 10:21:34 2023 +0800
[HUDI-6236] Write hive_style_partitioning_enable to table config in
DeltaStreamer entrypoints (#8752)
---
.../hudi/utilities/deltastreamer/DeltaSync.java | 10 ++++++++++
.../deltastreamer/TestHoodieDeltaStreamer.java | 19 +++++++++++++++++++
2 files changed, 29 insertions(+)
diff --git
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
index cbd19305e41..936320a648e 100644
---
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
+++
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
@@ -141,6 +141,8 @@ import scala.collection.JavaConversions;
import static org.apache.hudi.avro.AvroSchemaUtils.getAvroRecordQualifiedName;
import static org.apache.hudi.common.table.HoodieTableConfig.ARCHIVELOG_FOLDER;
import static
org.apache.hudi.common.table.HoodieTableConfig.DROP_PARTITION_COLUMNS;
+import static
org.apache.hudi.common.table.HoodieTableConfig.HIVE_STYLE_PARTITIONING_ENABLE;
+import static
org.apache.hudi.common.table.HoodieTableConfig.URL_ENCODE_PARTITIONING;
import static
org.apache.hudi.config.HoodieClusteringConfig.ASYNC_CLUSTERING_ENABLE;
import static org.apache.hudi.config.HoodieClusteringConfig.INLINE_CLUSTERING;
import static org.apache.hudi.config.HoodieCompactionConfig.INLINE_COMPACT;
@@ -387,6 +389,10 @@ public class DeltaSync implements Serializable, Closeable {
.setCDCSupplementalLoggingMode(props.getString(HoodieTableConfig.CDC_SUPPLEMENTAL_LOGGING_MODE.key(),
HoodieTableConfig.CDC_SUPPLEMENTAL_LOGGING_MODE.defaultValue()))
.setShouldDropPartitionColumns(isDropPartitionColumns())
+
.setHiveStylePartitioningEnable(props.getBoolean(HIVE_STYLE_PARTITIONING_ENABLE.key(),
+
Boolean.parseBoolean(HIVE_STYLE_PARTITIONING_ENABLE.defaultValue())))
+
.setUrlEncodePartitioning(props.getBoolean(URL_ENCODE_PARTITIONING.key(),
+ Boolean.parseBoolean(URL_ENCODE_PARTITIONING.defaultValue())))
.initTable(new Configuration(jssc.hadoopConfiguration()),
cfg.targetBasePath);
}
@@ -497,6 +503,10 @@ public class DeltaSync implements Serializable, Closeable {
.setPartitionMetafileUseBaseFormat(props.getBoolean(HoodieTableConfig.PARTITION_METAFILE_USE_BASE_FORMAT.key(),
HoodieTableConfig.PARTITION_METAFILE_USE_BASE_FORMAT.defaultValue()))
.setShouldDropPartitionColumns(isDropPartitionColumns())
+
.setHiveStylePartitioningEnable(props.getBoolean(HIVE_STYLE_PARTITIONING_ENABLE.key(),
+
Boolean.parseBoolean(HIVE_STYLE_PARTITIONING_ENABLE.defaultValue())))
+
.setUrlEncodePartitioning(props.getBoolean(URL_ENCODE_PARTITIONING.key(),
+ Boolean.parseBoolean(URL_ENCODE_PARTITIONING.defaultValue())))
.initTable(new Configuration(jssc.hadoopConfiguration()),
cfg.targetBasePath);
}
diff --git
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index 504730e3d33..053480e7fc4 100644
---
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -144,6 +144,8 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import static
org.apache.hudi.common.table.HoodieTableConfig.HIVE_STYLE_PARTITIONING_ENABLE;
+import static
org.apache.hudi.common.table.HoodieTableConfig.URL_ENCODE_PARTITIONING;
import static
org.apache.hudi.common.table.HoodieTableMetaClient.METAFOLDER_NAME;
import static
org.apache.hudi.config.metrics.HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE;
import static
org.apache.hudi.config.metrics.HoodieMetricsConfig.TURN_METRICS_ON;
@@ -656,6 +658,23 @@ public class TestHoodieDeltaStreamer extends
HoodieDeltaStreamerTestBase {
LOG.debug("Expected error during table creation", e);
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testTableCreationContainsHiveStylePartitioningEnable(boolean
configFlag) throws Exception {
+ String tablePath = basePath +
"/url_encode_and_hive_style_partitioning_enable_" + configFlag;
+ HoodieDeltaStreamer.Config cfg = TestHelpers.makeConfig(tablePath,
WriteOperationType.INSERT);
+ // Update DeltaStreamer configs for with parameterized test input
+ cfg.configs.add(HIVE_STYLE_PARTITIONING_ENABLE.key() + "=" + configFlag);
+ cfg.configs.add(URL_ENCODE_PARTITIONING.key() + "=" + configFlag);
+ HoodieDeltaStreamer deltaStreamer = new HoodieDeltaStreamer(cfg, jsc);
+ deltaStreamer.getIngestionService().ingestOnce();
+ // Create new metaClient from tablePath
+ HoodieTableMetaClient metaClient =
HoodieTableMetaClient.builder().setConf(context.getHadoopConf().get())
+ .setBasePath(tablePath).build();
+ assertEquals(configFlag,
Boolean.parseBoolean(metaClient.getTableConfig().getHiveStylePartitioningEnable()));
+ assertEquals(configFlag,
Boolean.parseBoolean(metaClient.getTableConfig().getUrlEncodePartitioning()));
+ }
+
@ParameterizedTest
@EnumSource(value = HoodieRecordType.class, names = {"AVRO", "SPARK"})
public void testBulkInsertsAndUpsertsWithBootstrap(HoodieRecordType
recordType) throws Exception {
