vinothchandar commented on a change in pull request #1810:
URL: https://github.com/apache/hudi/pull/1810#discussion_r465228726
##########
File path: hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
##########
@@ -261,6 +268,44 @@ private[hudi] object HoodieSparkSqlWriter {
hiveSyncConfig
}
+ private def metaSync(parameters: Map[String, String],
+ basePath: Path,
+ hadoopConf: Configuration): Boolean = {
+ val hiveSyncEnabled = parameters.get(HIVE_SYNC_ENABLED_OPT_KEY).exists(r
=> r.toBoolean)
+ var metaSyncEnabled = parameters.get(HUDI_SYNC_ENABLED_OPT_KEY).exists(r
=> r.toBoolean)
+ var syncClientToolClass = parameters(SYNC_CLIENT_TOOL_CLASS)
+ // for backward compatibility
+ if (hiveSyncEnabled) {
+ metaSyncEnabled = true
+ syncClientToolClass = String.format("%s,%s", syncClientToolClass,
"org.apache.hudi.hive.HiveSyncTool")
Review comment:
if someone does `hiveSyncEnabled == true && metaSyncEnabled == true &&
syncClientToolClass = org.apache.hudi.hive.HiveSyncTool`, we will sync two
times? can we just a set to hold the classes.
##########
File path: hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
##########
@@ -261,6 +268,44 @@ private[hudi] object HoodieSparkSqlWriter {
hiveSyncConfig
}
+ private def metaSync(parameters: Map[String, String],
+ basePath: Path,
+ hadoopConf: Configuration): Boolean = {
+ val hiveSyncEnabled = parameters.get(HIVE_SYNC_ENABLED_OPT_KEY).exists(r
=> r.toBoolean)
+ var metaSyncEnabled = parameters.get(HUDI_SYNC_ENABLED_OPT_KEY).exists(r
=> r.toBoolean)
Review comment:
actually like META_SYNC better here. it was more meaningful. wdyt?
##########
File path:
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
##########
@@ -267,9 +267,16 @@ public Operation convert(String value) throws
ParameterException {
description = "Should duplicate records from source be
dropped/filtered out before insert/bulk-insert")
public Boolean filterDupes = false;
+ //will abandon in the future version, recommended use --enable-sync
@Parameter(names = {"--enable-hive-sync"}, description = "Enable syncing
to hive")
public Boolean enableHiveSync = false;
+ @Parameter(names = {"--enable-sync"}, description = "Enable syncing meta")
+ public Boolean enableMetaSync = false;
+
+ @Parameter(names = {"--sync-tool-classes"}, description = "Meta sync
client tool, using comma to separate multi tools")
+ public String syncClientToolClass = "org.apache.hudi.hive.HiveSyncTool";
Review comment:
lets do `HiveSyncTool.class.getName` or soemthing?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
