[hudi] branch master updated: [HUDI-1448] Hudi dla sync support skip rt table syncing (#2324)

[leesf]

This is an automated email from the ASF dual-hosted git repository.

leesf pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new facde4c  [HUDI-1448]  Hudi dla sync support skip rt table syncing 
(#2324)
facde4c is described below

commit facde4c16f9a61e1c62ed268c869493696aa6a06
Author: lw0090 <lw309637...@gmail.com>
AuthorDate: Mon Dec 14 23:25:10 2020 +0800

    [HUDI-1448]  Hudi dla sync support skip rt table syncing (#2324)
---
 .../src/main/java/org/apache/hudi/dla/DLASyncConfig.java              | 4 ++++
 .../hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncTool.java  | 4 +++-
 .../hudi-dla-sync/src/main/java/org/apache/hudi/dla/util/Utils.java   | 2 ++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git 
a/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncConfig.java 
b/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncConfig.java
index b04cc0a..8a92de6 100644
--- 
a/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncConfig.java
+++ 
b/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncConfig.java
@@ -62,6 +62,9 @@ public class DLASyncConfig implements Serializable {
   @Parameter(names = {"--skip-ro-suffix"}, description = "Skip the `_ro` 
suffix for Read optimized table, when registering")
   public Boolean skipROSuffix = false;
 
+  @Parameter(names = {"--skip-rt-sync"}, description = "Skip the RT table 
syncing")
+  public Boolean skipRTSync = false;
+
   @Parameter(names = {"--hive-style-partitioning"}, description = "Use DLA 
hive style partitioning, true if like the following style: 
field1=value1/field2=value2")
   public Boolean useDLASyncHiveStylePartitioning = false;
 
@@ -83,6 +86,7 @@ public class DLASyncConfig implements Serializable {
     newConfig.partitionValueExtractorClass = cfg.partitionValueExtractorClass;
     newConfig.assumeDatePartitioning = cfg.assumeDatePartitioning;
     newConfig.skipROSuffix = cfg.skipROSuffix;
+    newConfig.skipRTSync = cfg.skipRTSync;
     newConfig.useDLASyncHiveStylePartitioning = 
cfg.useDLASyncHiveStylePartitioning;
     newConfig.supportTimestamp = cfg.supportTimestamp;
     return newConfig;
diff --git 
a/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncTool.java 
b/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncTool.java
index 347bb62..25d52c5 100644
--- a/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncTool.java
+++ b/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncTool.java
@@ -93,7 +93,9 @@ public class DLASyncTool extends AbstractSyncTool {
           // sync a RO table for MOR
           syncHoodieTable(roTableTableName.get(), false);
           // sync a RT table for MOR
-          syncHoodieTable(snapshotTableName, true);
+          if (!cfg.skipRTSync) {
+            syncHoodieTable(snapshotTableName, true);
+          }
           break;
         default:
           LOG.error("Unknown table type " + hoodieDLAClient.getTableType());
diff --git 
a/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/util/Utils.java 
b/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/util/Utils.java
index 636fd0b..ad47b71 100644
--- a/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/util/Utils.java
+++ b/hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/util/Utils.java
@@ -36,6 +36,7 @@ public class Utils {
   public static String DLA_PARTITION_EXTRACTOR_CLASS_OPT_KEY = 
"hoodie.datasource.dla_sync.partition_extractor_class";
   public static String DLA_ASSUME_DATE_PARTITIONING = 
"hoodie.datasource.dla_sync.assume_date_partitioning";
   public static String DLA_SKIP_RO_SUFFIX = 
"hoodie.datasource.dla_sync.skip_ro_suffix";
+  public static String DLA_SKIP_RT_SYNC = 
"hoodie.datasource.dla_sync.skip_rt_sync";
   public static String DLA_SYNC_HIVE_STYLE_PARTITIONING = 
"hoodie.datasource.dla_sync.hive.style.partitioning";
 
   public static Properties configToProperties(DLASyncConfig cfg) {
@@ -69,6 +70,7 @@ public class Utils {
     config.partitionValueExtractorClass = 
properties.getProperty(DLA_PARTITION_EXTRACTOR_CLASS_OPT_KEY);
     config.assumeDatePartitioning = 
Boolean.parseBoolean(properties.getProperty(DLA_ASSUME_DATE_PARTITIONING, 
"false"));
     config.skipROSuffix = 
Boolean.parseBoolean(properties.getProperty(DLA_SKIP_RO_SUFFIX, "false"));
+    config.skipRTSync = 
Boolean.parseBoolean(properties.getProperty(DLA_SKIP_RT_SYNC, "false"));
     config.useDLASyncHiveStylePartitioning = 
Boolean.parseBoolean(properties.getProperty(DLA_SYNC_HIVE_STYLE_PARTITIONING, 
"false"));
     return config;
   }