[hudi] branch master updated: [HUDI-6239] Fix clustering pool scheduler conf not take effect bug (#8763)

[danny0405]

This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 84b3dd33edc [HUDI-6239]  Fix clustering pool scheduler conf not take 
effect bug (#8763)
84b3dd33edc is described below

commit 84b3dd33edc774cc166ed0f3d26968645e6b5150
Author: kongwei <kong...@pku.edu.cn>
AuthorDate: Sat May 20 10:47:40 2023 +0800

    [HUDI-6239]  Fix clustering pool scheduler conf not take effect bug (#8763)
    
    Co-authored-by: wei.kong <wei.k...@shopee.com>
---
 .../deltastreamer/SchedulerConfGenerator.java      | 26 +++++++++++++++++-----
 .../deltastreamer/TestSchedulerConfGenerator.java  | 26 ++++++++++++++++++++++
 2 files changed, 46 insertions(+), 6 deletions(-)

diff --git 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java
 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java
index e23cb04ae70..038a64b4e2f 100644
--- 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java
+++ 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java
@@ -51,10 +51,24 @@ public class SchedulerConfGenerator {
   public static final String SPARK_SCHEDULER_FAIR_MODE = "FAIR";
 
   private static final String SPARK_SCHEDULING_PATTERN =
-      "<?xml version=\"1.0\"?>\n<allocations>\n  <pool name=\"%s\">\n"
-          + "    <schedulingMode>%s</schedulingMode>\n    
<weight>%s</weight>\n    <minShare>%s</minShare>\n"
-          + "  </pool>\n  <pool name=\"%s\">\n    
<schedulingMode>%s</schedulingMode>\n"
-          + "    <weight>%s</weight>\n    <minShare>%s</minShare>\n  
</pool>\n</allocations>";
+      "<?xml version=\"1.0\"?>\n"
+          + "<allocations>\n"
+          + "    <pool name=\"%s\">\n"
+          + "        <schedulingMode>%s</schedulingMode>\n"
+          + "        <weight>%s</weight>\n"
+          + "        <minShare>%s</minShare>\n"
+          + "    </pool>\n"
+          + "    <pool name=\"%s\">\n"
+          + "        <schedulingMode>%s</schedulingMode>\n"
+          + "        <weight>%s</weight>\n"
+          + "        <minShare>%s</minShare>\n"
+          + "    </pool>\n"
+          + "    <pool name=\"%s\">\n"
+          + "        <schedulingMode>%s</schedulingMode>\n"
+          + "        <weight>%s</weight>\n"
+          + "        <minShare>%s</minShare>\n"
+          + "    </pool>\n"
+          + "</allocations>";
 
   /**
    * Helper to generate spark scheduling configs in XML format with input 
params.
@@ -67,8 +81,8 @@ public class SchedulerConfGenerator {
    * @param clusteringWeight Minshare for clustering
    * @return Spark scheduling configs
    */
-  private static String generateConfig(Integer deltaSyncWeight, Integer 
compactionWeight, Integer deltaSyncMinShare,
-      Integer compactionMinShare, Integer clusteringWeight, Integer 
clusteringMinShare) {
+  static String generateConfig(Integer deltaSyncWeight, Integer 
compactionWeight, Integer deltaSyncMinShare,
+                               Integer compactionMinShare, Integer 
clusteringWeight, Integer clusteringMinShare) {
     return String.format(SPARK_SCHEDULING_PATTERN, DELTASYNC_POOL_NAME, 
SPARK_SCHEDULER_FAIR_MODE,
         deltaSyncWeight.toString(), deltaSyncMinShare.toString(), 
COMPACT_POOL_NAME, SPARK_SCHEDULER_FAIR_MODE,
         compactionWeight.toString(), compactionMinShare.toString(), 
CLUSTERING_POOL_NAME, SPARK_SCHEDULER_FAIR_MODE,
diff --git 
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSchedulerConfGenerator.java
 
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSchedulerConfGenerator.java
index 26b8bc1c885..5085261b9d1 100644
--- 
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSchedulerConfGenerator.java
+++ 
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSchedulerConfGenerator.java
@@ -25,6 +25,7 @@ import org.junit.jupiter.api.Test;
 
 import java.util.Map;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 
@@ -51,4 +52,29 @@ public class TestSchedulerConfGenerator {
     configs = SchedulerConfGenerator.getSparkSchedulingConfigs(cfg);
     
assertNotNull(configs.get(SparkConfigs.SPARK_SCHEDULER_ALLOCATION_FILE_KEY()), 
"all satisfies");
   }
+
+  @Test
+  public void testGenerateConfig() {
+    String targetConfig =
+        "<?xml version=\"1.0\"?>\n"
+            + "<allocations>\n"
+            + "    <pool name=\"hoodiedeltasync\">\n"
+            + "        <schedulingMode>FAIR</schedulingMode>\n"
+            + "        <weight>1</weight>\n"
+            + "        <minShare>2</minShare>\n"
+            + "    </pool>\n"
+            + "    <pool name=\"hoodiecompact\">\n"
+            + "        <schedulingMode>FAIR</schedulingMode>\n"
+            + "        <weight>3</weight>\n"
+            + "        <minShare>4</minShare>\n"
+            + "    </pool>\n"
+            + "    <pool name=\"hoodiecluster\">\n"
+            + "        <schedulingMode>FAIR</schedulingMode>\n"
+            + "        <weight>5</weight>\n"
+            + "        <minShare>6</minShare>\n"
+            + "    </pool>\n"
+            + "</allocations>";
+    String generatedConfig = SchedulerConfGenerator.generateConfig(1, 3, 2, 4, 
5, 6);
+    assertEquals(targetConfig, generatedConfig);
+  }
 }