[hudi] branch master updated: [HUDI-6360] spark-memory parmater should not be forced to be true (#8945)

[danny0405]

This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new aca385d70c5 [HUDI-6360] spark-memory parmater should not be forced to 
be true (#8945)
aca385d70c5 is described below

commit aca385d70c5b3ff06883d2cbf72082fc5d31894b
Author: ksmou <135721692+ks...@users.noreply.github.com>
AuthorDate: Thu Jun 15 11:12:51 2023 +0800

    [HUDI-6360] spark-memory parmater should not be forced to be true (#8945)
---
 .../src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java  | 2 +-
 .../src/main/java/org/apache/hudi/utilities/HoodieCompactor.java      | 2 +-
 .../src/main/java/org/apache/hudi/utilities/UtilHelpers.java          | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java
 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java
index f102e732f0e..3e636855c96 100644
--- 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java
+++ 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieClusteringJob.java
@@ -87,7 +87,7 @@ public class HoodieClusteringJob {
     public int parallelism = 1;
     @Parameter(names = {"--spark-master", "-ms"}, description = "Spark master")
     public String sparkMaster = null;
-    @Parameter(names = {"--spark-memory", "-sm"}, description = "spark memory 
to use", required = true)
+    @Parameter(names = {"--spark-memory", "-sm"}, description = "spark memory 
to use", required = false)
     public String sparkMemory = null;
     @Parameter(names = {"--retry", "-rt"}, description = "number of retries")
     public int retry = 0;
diff --git 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java
index 6fed2320105..7f463b32319 100644
--- 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java
+++ 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java
@@ -91,7 +91,7 @@ public class HoodieCompactor {
     public String schemaFile = null;
     @Parameter(names = {"--spark-master", "-ms"}, description = "Spark 
master", required = false)
     public String sparkMaster = null;
-    @Parameter(names = {"--spark-memory", "-sm"}, description = "spark memory 
to use", required = true)
+    @Parameter(names = {"--spark-memory", "-sm"}, description = "spark memory 
to use", required = false)
     public String sparkMemory = null;
     @Parameter(names = {"--retry", "-rt"}, description = "number of retries", 
required = false)
     public int retry = 0;
diff --git 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
index f4f095d73fb..f9b27bfdd5b 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java
@@ -336,7 +336,9 @@ public class UtilHelpers {
    */
   public static JavaSparkContext buildSparkContext(String appName, String 
sparkMaster, String sparkMemory) {
     SparkConf sparkConf = buildSparkConf(appName, sparkMaster);
-    sparkConf.set("spark.executor.memory", sparkMemory);
+    if (sparkMemory != null) {
+      sparkConf.set("spark.executor.memory", sparkMemory);
+    }
     return new JavaSparkContext(sparkConf);
   }