(hudi) branch master updated: [HUDI-8363] Support to provide custom spark app name for HoodieStreamer (#12094)

[danny0405]

This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 957103416a7 [HUDI-8363] Support to provide custom spark app name for 
HoodieStreamer (#12094)
957103416a7 is described below

commit 957103416a787cf07b025d6419b9fe4de66df01c
Author: Prathit malik <53890994+prathi...@users.noreply.github.com>
AuthorDate: Tue Oct 15 07:10:07 2024 +0530

    [HUDI-8363] Support to provide custom spark app name for HoodieStreamer 
(#12094)
---
 .../apache/hudi/utilities/streamer/HoodieStreamer.java    | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
index e10582f52a8..22b9071d0b6 100644
--- 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
+++ 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
@@ -434,6 +434,11 @@ public class HoodieStreamer implements Serializable {
         + " committed checkpoint, and rely on other configs to pick the 
starting offsets).")
     public String ignoreCheckpoint = null;
 
+    @Parameter(names = {"--spark-app-name"},
+            description = "spark app name to use while creating spark context."
+                    + " If not defined then defaults to 
streamer-{cfg.targetTableName}.")
+    public String sparkAppName = "";
+
     public boolean isAsyncCompactionEnabled() {
       return continuousMode && !forceDisableCompaction
           && 
HoodieTableType.MERGE_ON_READ.equals(HoodieTableType.valueOf(tableType));
@@ -593,10 +598,16 @@ public class HoodieStreamer implements Serializable {
     final Config cfg = getConfig(args);
     Map<String, String> additionalSparkConfigs = 
SchedulerConfGenerator.getSparkSchedulingConfigs(cfg);
     JavaSparkContext jssc = null;
+    String sparkAppName;
+    if (!StringUtils.isNullOrEmpty(cfg.sparkAppName)) {
+      sparkAppName = cfg.sparkAppName;
+    } else {
+      sparkAppName = "streamer-" + cfg.targetTableName;
+    }
     if (StringUtils.isNullOrEmpty(cfg.sparkMaster)) {
-      jssc = UtilHelpers.buildSparkContext("streamer-" + cfg.targetTableName, 
additionalSparkConfigs);
+      jssc = UtilHelpers.buildSparkContext(sparkAppName, 
additionalSparkConfigs);
     } else {
-      jssc = UtilHelpers.buildSparkContext("streamer-" + cfg.targetTableName, 
cfg.sparkMaster, additionalSparkConfigs);
+      jssc = UtilHelpers.buildSparkContext(sparkAppName, cfg.sparkMaster, 
additionalSparkConfigs);
     }
     if (cfg.enableHiveSync) {
       LOG.warn("--enable-hive-sync will be deprecated in a future release; 
please use --enable-sync instead for Hive syncing");