This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9628aa8 YARN-10598. CS Flexible Auto Queue Creation: Modify RM
/scheduler endpoint to extend the creation type with additional information.
Contributed by Benjamin Teke
9628aa8 is described below
commit 9628aa87bf213c1752511679a31c3d483b93db34
Author: Szilard Nemeth <[email protected]>
AuthorDate: Wed Jan 27 18:14:06 2021 +0100
YARN-10598. CS Flexible Auto Queue Creation: Modify RM /scheduler endpoint
to extend the creation type with additional information. Contributed by
Benjamin Teke
---
.../webapp/dao/CapacitySchedulerInfo.java | 5 +
.../webapp/dao/CapacitySchedulerQueueInfo.java | 5 +
.../dao/helper/CapacitySchedulerInfoHelper.java | 71 +++++++++---
.../webapp/TestRMWebServicesCapacitySched.java | 6 +-
...estRMWebServicesCapacitySchedDynamicConfig.java | 122 +++++++++++++++------
.../TestRMWebServicesForCSWithPartitions.java | 2 +-
6 files changed, 159 insertions(+), 52 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index 548cfe8..9143817 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -58,6 +58,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected String orderingPolicyInfo;
protected String mode;
protected String queueType;
+ protected String creationMethod;
+ protected String autoCreationEligibility;
@XmlTransient
static final float EPSILON = 1e-8f;
@@ -107,6 +109,9 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
}
mode = CapacitySchedulerInfoHelper.getMode(parent);
queueType = CapacitySchedulerInfoHelper.getQueueType(parent);
+ creationMethod = CapacitySchedulerInfoHelper.getCreationMethod(parent);
+ autoCreationEligibility = CapacitySchedulerInfoHelper
+ .getAutoCreationEligibility(parent);
}
public float getCapacity() {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index dec666b..9126566 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -91,6 +91,8 @@ public class CapacitySchedulerQueueInfo {
protected LeafQueueTemplateInfo leafQueueTemplate;
protected String mode;
protected String queueType;
+ protected String creationMethod;
+ protected String autoCreationEligibility;
CapacitySchedulerQueueInfo() {
};
@@ -137,6 +139,9 @@ public class CapacitySchedulerQueueInfo {
mode = CapacitySchedulerInfoHelper.getMode(q);
queueType = CapacitySchedulerInfoHelper.getQueueType(q);
+ creationMethod = CapacitySchedulerInfoHelper.getCreationMethod(q);
+ autoCreationEligibility = CapacitySchedulerInfoHelper
+ .getAutoCreationEligibility(q);
ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
populateQueueResourceUsage(queueResourceUsage);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java
index 52a5a5c..355528d7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java
@@ -18,17 +18,44 @@ package
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
-import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractManagedParentQueue;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
+/**
+ * Helper class to describe a queue's type, its creation method and its
+ * eligibility of having auto created children.
+ *
+ * queueType: a queue can be a parent or a leaf.
+ *
+ * creationMethod: the creation method of the queue. Can be: static,
+ * dynamicLegacy or dynamicFlexible. When the legacy way of queue auto-creation
+ * (before YARN-10506) is used, a parent can only be static (ManagedParent)
+ * and a leaf queue can only be dynamicLegacy (no static child queues are
+ * allowed under ManagedParents). When the flexible auto queue creation is used
+ * both a parent and a leaf can be either static or dynamicFlexible.
+ *
+ * autoCreationEligibility: describes whether a queue can have dynamically
+ * created children. Can be: off, legacy or flexible. Every leaf will have this
+ * field with the value off, as they can't have children. When the legacy way
+ * of queue auto-creation (before YARN-10506) is used a ManagedParent will have
+ * the legacy value. When the flexible auto queue creation is used a static
+ * parent can have the value flexible if it is configured to allow auto queue
+ * creation, or off if it is not. A dynamic parent implicitly will have the
+ * value flexible, as a dynamically created parent cannot have static children.
+ */
public class CapacitySchedulerInfoHelper {
- private static final String AUTO_CREATED_LEAF = "autoCreatedLeaf";
- private static final String STATIC_LEAF = "staticLeaf";
- private static final String AUTO_CREATED_PARENT = "autoCreatedParent";
- private static final String STATIC_PARENT = "staticParent";
+ private static final String PARENT_QUEUE = "parent";
+ private static final String LEAF_QUEUE = "leaf";
private static final String UNKNOWN_QUEUE = "unknown";
+ private static final String STATIC_QUEUE = "static";
+ private static final String LEGACY_DYNAMIC_QUEUE = "dynamicLegacy";
+ private static final String FLEXIBLE_DYNAMIC_QUEUE = "dynamicFlexible";
+ private static final String AUTO_CREATION_OFF = "off";
+ private static final String AUTO_CREATION_LEGACY = "legacy";
+ private static final String AUTO_CREATION_FLEXIBLE = "flexible";
private CapacitySchedulerInfoHelper() {}
@@ -52,19 +79,31 @@ public class CapacitySchedulerInfoHelper {
public static String getQueueType(CSQueue queue) {
if (queue instanceof LeafQueue) {
- if (((AbstractCSQueue)queue).isDynamicQueue()) {
- return AUTO_CREATED_LEAF;
- } else {
- return STATIC_LEAF;
- }
+ return LEAF_QUEUE;
} else if (queue instanceof ParentQueue) {
- if (((AbstractCSQueue)queue).isDynamicQueue()) {
- return AUTO_CREATED_PARENT;
- } else {
- //A ParentQueue with isDynamic=false or an AbstractManagedParentQueue
- return STATIC_PARENT;
- }
+ return PARENT_QUEUE;
}
return UNKNOWN_QUEUE;
}
+
+ public static String getCreationMethod(CSQueue queue) {
+ if (queue instanceof AutoCreatedLeafQueue) {
+ return LEGACY_DYNAMIC_QUEUE;
+ } else if (((AbstractCSQueue)queue).isDynamicQueue()) {
+ return FLEXIBLE_DYNAMIC_QUEUE;
+ } else {
+ return STATIC_QUEUE;
+ }
+ }
+
+ public static String getAutoCreationEligibility(CSQueue queue) {
+ if (queue instanceof ManagedParentQueue) {
+ return AUTO_CREATION_LEGACY;
+ } else if (queue instanceof ParentQueue &&
+ ((ParentQueue)queue).isEligibleForAutoQueueCreation()) {
+ return AUTO_CREATION_FLEXIBLE;
+ } else {
+ return AUTO_CREATION_OFF;
+ }
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 61b4df7..9ad75d6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -362,7 +362,7 @@ public class TestRMWebServicesCapacitySched extends
JerseyTestBase {
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo");
- assertEquals("incorrect number of elements in: " + info, 16,
info.length());
+ assertEquals("incorrect number of elements in: " + info, 18,
info.length());
verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"),
@@ -413,10 +413,10 @@ public class TestRMWebServicesCapacitySched extends
JerseyTestBase {
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
- int numExpectedElements = 31;
+ int numExpectedElements = 33;
boolean isParentQueue = true;
if (!info.has("queues")) {
- numExpectedElements = 49;
+ numExpectedElements = 51;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements,
info.length());
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
index 118a0df..3ccb967 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils;
@@ -73,11 +75,14 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
private static final float EXP_ROOT_WEIGHT_IN_WEIGHT_MODE = 1.0F;
private static final float EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE = 1.0F;
private static final double DELTA = 0.00001;
- private static final String STATIC_PARENT = "staticParent";
- private static final String STATIC_LEAF = "staticLeaf";
+ private static final String PARENT_QUEUE = "parent";
+ private static final String LEAF_QUEUE = "leaf";
+ private static final String STATIC_QUEUE = "static";
+ private static final String FLEXIBLE_DYNAMIC_QUEUE = "dynamicFlexible";
+ private static final String AUTO_CREATION_OFF = "off";
+ private static final String AUTO_CREATION_LEGACY = "legacy";
+ private static final String AUTO_CREATION_FLEXIBLE = "flexible";
private static final int GB = 1024;
- private static final String AUTO_CREATED_LEAF = "autoCreatedLeaf";
- private static final String AUTO_CREATED_PARENT = "autoCreatedParent";
protected static MockRM RM;
private CapacitySchedulerAutoQueueHandler autoQueueHandler;
@@ -88,13 +93,18 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
public final float weight;
public final float normalizedWeight;
private String queueType;
+ private String creationMethod;
+ private String autoCreationEligibility;
public ExpectedQueueWithProperties(String path, float weight,
- float normalizedWeight, String queueType) {
+ float normalizedWeight, String queueType, String creationMethod,
+ String autoCreationEligibility) {
this.path = path;
this.weight = weight;
this.normalizedWeight = normalizedWeight;
this.queueType = queueType;
+ this.creationMethod = creationMethod;
+ this.autoCreationEligibility = autoCreationEligibility;
}
}
@@ -161,16 +171,41 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
validateSchedulerInfo(json, "percentage",
new ExpectedQueueWithProperties("root",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
- STATIC_PARENT),
+ PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.default",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
- STATIC_LEAF),
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test1",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
- STATIC_LEAF),
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test2",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
- STATIC_LEAF));
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
+ }
+
+ @Test
+ public void testSchedulerResponsePercentageModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = CSConfigGenerator
+ .createPercentageConfigLegacyAutoCreation();
+ config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
+
+ initResourceManager(config);
+ JSONObject json = sendRequestToSchedulerEndpoint();
+ validateSchedulerInfo(json, "percentage",
+ new ExpectedQueueWithProperties("root",
+ EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
+ PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
+ new ExpectedQueueWithProperties("root.default",
+ EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
+ new ExpectedQueueWithProperties("root.test1",
+ EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
+ new ExpectedQueueWithProperties("root.managedtest2",
+ EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
+ PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_LEGACY));
}
@Test
@@ -186,16 +221,16 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
validateSchedulerInfo(json, "absolute",
new ExpectedQueueWithProperties("root",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
- STATIC_PARENT),
+ PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.default",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
- STATIC_LEAF),
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test1",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
- STATIC_LEAF),
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test2",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
- STATIC_LEAF));
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
}
@Test
@@ -211,13 +246,13 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
validateSchedulerInfo(json, "weight",
new ExpectedQueueWithProperties("root",
EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
- STATIC_PARENT),
+ PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.default", 10.0f, 0.5f,
- STATIC_LEAF),
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test1", 4.0f, 0.2f,
- STATIC_LEAF),
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test2", 6.0f, 0.3f,
- STATIC_LEAF));
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
}
@Test
@@ -234,13 +269,13 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
validateSchedulerInfo(json, "weight",
new ExpectedQueueWithProperties("root",
EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
- STATIC_PARENT),
+ PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.default", 10.0f, 0.5f,
- STATIC_LEAF),
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test1", 4.0f, 0.2f,
- STATIC_LEAF),
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test2", 6.0f, 0.3f,
- STATIC_LEAF));
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
//Now create some auto created queues
createQueue("root.auto1");
@@ -255,42 +290,42 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
int sumOfWeights = 24;
ExpectedQueueWithProperties expectedRootQ =
new ExpectedQueueWithProperties("root",
- EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
- STATIC_PARENT);
+ EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
+ PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF);
validateSchedulerInfo(json, "weight",
expectedRootQ,
new ExpectedQueueWithProperties("root.auto1",
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
- AUTO_CREATED_LEAF),
+ LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.auto2",
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
- AUTO_CREATED_LEAF),
+ LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.auto3",
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
- AUTO_CREATED_LEAF),
+ LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.autoParent1",
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
- AUTO_CREATED_PARENT),
+ PARENT_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_FLEXIBLE),
new ExpectedQueueWithProperties("root.default", 10.0f,
10.0f / sumOfWeights,
- STATIC_LEAF),
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test1", 4.0f,
4.0f / sumOfWeights,
- STATIC_LEAF),
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test2", 6.0f,
6.0f / sumOfWeights,
- STATIC_LEAF));
+ LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
validateChildrenOfParent(json, "root.autoParent1", "weight",
expectedRootQ,
new ExpectedQueueWithProperties("root.autoParent1.auto4",
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
- AUTO_CREATED_LEAF));
+ LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF));
}
private void initAutoQueueHandler() throws Exception {
@@ -433,9 +468,18 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
Float.parseFloat(obj.getString("normalizedWeight")), DELTA);
//validate queue creation type
- Assert.assertEquals("Queue creation type does not match for queue " +
+ Assert.assertEquals("Queue type does not match for queue " +
queuePath,
expectedQueue.queueType, obj.getString("queueType"));
+
+ Assert.assertEquals("Queue creation type does not match for queue " +
+ queuePath,
+ expectedQueue.creationMethod, obj.getString("creationMethod"));
+
+ Assert.assertEquals("Queue auto creation eligibility does not " +
+ "match for queue " + queuePath,
+ expectedQueue.autoCreationEligibility,
+ obj.getString("autoCreationEligibility"));
}
//Validate queue paths and modes
@@ -470,6 +514,20 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
return createConfiguration(conf);
}
+ public static Configuration createPercentageConfigLegacyAutoCreation() {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, " +
+ "managedtest2");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
+ conf.put("yarn.scheduler.capacity.root.managedtest2.capacity", "50");
+ conf.put("yarn.scheduler.capacity.root.test1.maximum-capacity", "100");
+ conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING");
+ conf.put("yarn.scheduler.capacity.root.managedtest2.state", "RUNNING");
+ conf.put("yarn.scheduler.capacity.root.managedtest2." +
+ "auto-create-child-queue.enabled", "true");
+ return createConfiguration(conf);
+ }
+
public static Configuration createAbsoluteConfig() {
Map<String, String> conf = new HashMap<>();
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
index 7a28c91..8d9c453 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
@@ -574,7 +574,7 @@ public class TestRMWebServicesForCSWithPartitions extends
JerseyTestBase {
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo");
- assertEquals("incorrect number of elements", 16, info.length());
+ assertEquals("incorrect number of elements", 18, info.length());
JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
JSONArray partitionsCapsArray =
capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
