(hudi) branch master updated: [MINOR] Do not force setting spark conf in UtilHelpers (#11166)

[danny0405]

This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 113233f97ba [MINOR] Do not force setting spark conf in UtilHelpers 
(#11166)
113233f97ba is described below

commit 113233f97ba1de5230f1e4607fc62a6d58c6a4cd
Author: Zouxxyy <zouxinyu....@alibaba-inc.com>
AuthorDate: Tue May 7 18:19:48 2024 +0800

    [MINOR] Do not force setting spark conf in UtilHelpers (#11166)
---
 .../org/apache/hudi/utilities/UtilHelpers.java     | 38 +++++++++++-----------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
index 724a36e6b18..124abeb059f 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
@@ -333,19 +333,19 @@ public class UtilHelpers {
     String master = sparkConf.get("spark.master", defaultMaster);
     sparkConf.setMaster(master);
     if (master.startsWith("yarn")) {
-      sparkConf.set("spark.eventLog.overwrite", "true");
-      sparkConf.set("spark.eventLog.enabled", "true");
+      sparkConf.setIfMissing("spark.eventLog.overwrite", "true");
+      sparkConf.setIfMissing("spark.eventLog.enabled", "true");
     }
-    sparkConf.set("spark.ui.port", "8090");
+    sparkConf.setIfMissing("spark.ui.port", "8090");
     sparkConf.setIfMissing("spark.driver.maxResultSize", "2g");
-    sparkConf.set("spark.serializer", 
"org.apache.spark.serializer.KryoSerializer");
-    sparkConf.set("spark.kryo.registrator", 
"org.apache.spark.HoodieSparkKryoRegistrar");
-    sparkConf.set("spark.sql.extensions", 
"org.apache.spark.sql.hudi.HoodieSparkSessionExtension");
-    sparkConf.set("spark.hadoop.mapred.output.compress", "true");
-    sparkConf.set("spark.hadoop.mapred.output.compression.codec", "true");
-    sparkConf.set("spark.hadoop.mapred.output.compression.codec", 
"org.apache.hadoop.io.compress.GzipCodec");
-    sparkConf.set("spark.hadoop.mapred.output.compression.type", "BLOCK");
-    sparkConf.set("spark.driver.allowMultipleContexts", "true");
+    sparkConf.setIfMissing("spark.serializer", 
"org.apache.spark.serializer.KryoSerializer");
+    sparkConf.setIfMissing("spark.kryo.registrator", 
"org.apache.spark.HoodieSparkKryoRegistrar");
+    sparkConf.setIfMissing("spark.sql.extensions", 
"org.apache.spark.sql.hudi.HoodieSparkSessionExtension");
+    sparkConf.setIfMissing("spark.hadoop.mapred.output.compress", "true");
+    sparkConf.setIfMissing("spark.hadoop.mapred.output.compression.codec", 
"true");
+    sparkConf.setIfMissing("spark.hadoop.mapred.output.compression.codec", 
"org.apache.hadoop.io.compress.GzipCodec");
+    sparkConf.setIfMissing("spark.hadoop.mapred.output.compression.type", 
"BLOCK");
+    sparkConf.setIfMissing("spark.driver.allowMultipleContexts", "true");
 
     additionalConfigs.forEach(sparkConf::set);
     return sparkConf;
@@ -353,15 +353,15 @@ public class UtilHelpers {
 
   private static SparkConf buildSparkConf(String appName, Map<String, String> 
additionalConfigs) {
     final SparkConf sparkConf = new SparkConf().setAppName(appName);
-    sparkConf.set("spark.ui.port", "8090");
+    sparkConf.setIfMissing("spark.ui.port", "8090");
     sparkConf.setIfMissing("spark.driver.maxResultSize", "2g");
-    sparkConf.set("spark.serializer", 
"org.apache.spark.serializer.KryoSerializer");
-    sparkConf.set("spark.kryo.registrator", 
"org.apache.spark.HoodieSparkKryoRegistrar");
-    sparkConf.set("spark.sql.extensions", 
"org.apache.spark.sql.hudi.HoodieSparkSessionExtension");
-    sparkConf.set("spark.hadoop.mapred.output.compress", "true");
-    sparkConf.set("spark.hadoop.mapred.output.compression.codec", "true");
-    sparkConf.set("spark.hadoop.mapred.output.compression.codec", 
"org.apache.hadoop.io.compress.GzipCodec");
-    sparkConf.set("spark.hadoop.mapred.output.compression.type", "BLOCK");
+    sparkConf.setIfMissing("spark.serializer", 
"org.apache.spark.serializer.KryoSerializer");
+    sparkConf.setIfMissing("spark.kryo.registrator", 
"org.apache.spark.HoodieSparkKryoRegistrar");
+    sparkConf.setIfMissing("spark.sql.extensions", 
"org.apache.spark.sql.hudi.HoodieSparkSessionExtension");
+    sparkConf.setIfMissing("spark.hadoop.mapred.output.compress", "true");
+    sparkConf.setIfMissing("spark.hadoop.mapred.output.compression.codec", 
"true");
+    sparkConf.setIfMissing("spark.hadoop.mapred.output.compression.codec", 
"org.apache.hadoop.io.compress.GzipCodec");
+    sparkConf.setIfMissing("spark.hadoop.mapred.output.compression.type", 
"BLOCK");
 
     additionalConfigs.forEach(sparkConf::set);
     return sparkConf;