This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new eccc41c410b YARN-11682: Legacy auto created queue in absolute mode has
zero capacity after creation during app recovery (#8618) Contributed by Susheel
Gupta
eccc41c410b is described below
commit eccc41c410b09b242b200232048e67c09d50a344
Author: Susheel Gupta <[email protected]>
AuthorDate: Sun Jul 26 10:42:30 2026 +0530
YARN-11682: Legacy auto created queue in absolute mode has zero capacity
after creation during app recovery (#8618) Contributed by Susheel Gupta
During RM work-preserving recovery, an auto-created leaf queue can be
recreated before any NodeManager has registered. In absolute resource mode, the
zero cluster resource causes the queue template capacity to
be calculated as zero. If the queue is activated in this state, its
capacity remains zero even after NodeManagers register.
Defer activation of auto-created leaf queues while the cluster resource is
unavailable. Recompute the absolute-resource leaf queue template capacities
during periodic queue management so that the queue can be
activated with its configured guaranteed capacity after cluster resources
become available.
Add a regression test covering queue creation with zero cluster resources
and capacity restoration following NodeManager registration.
Signed-off-by: Shilun Fan <[email protected]>
---
.../scheduler/capacity/ManagedParentQueue.java | 20 ++++++++
.../GuaranteedOrZeroCapacityOverTimePolicy.java | 25 ++++++++-
.../TestAbsoluteResourceWithAutoQueue.java | 60 ++++++++++++++++++++++
3 files changed, 103 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
index fa749f07796..8170e4bc3fe 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
@@ -262,6 +262,26 @@ private void updateQueueCapacities(QueueCapacities
queueCapacities) {
}
}
+/**
+ * Recomputes leaf queue template capacities in ABSOLUTE_RESOURCE mode.
+ * This handles the case where a queue is recreated during RM recovery
+ * before any NodeManager has registered, causing the initial capacity
+ * to be computed as 0. The periodic queue management policy recalculates
+ * the template once cluster resources become available. No operation for other
+ * queue modes.
+ */
+ public void updateTemplateCapacitiesForAbsoluteResource() {
+ writeLock.lock();
+ try {
+ if (this.capacityConfigType.equals(
+ CapacityConfigType.ABSOLUTE_RESOURCE)) {
+ updateQueueCapacities(getLeafQueueTemplate().getQueueCapacities());
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
protected void validate(final CSQueue newlyParsedQueue) throws IOException {
// Sanity check
if (!(newlyParsedQueue instanceof ManagedParentQueue) || !newlyParsedQueue
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
index c4579b24daf..03ae36a21ef 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
@@ -51,6 +51,8 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
+ .capacity.AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
.capacity.CSQueueUtils.EPSILON;
@@ -310,6 +312,15 @@ private void
initializeLeafQueueTemplate(ManagedParentQueue parentQueue)
public List<QueueManagementChange> computeQueueManagementChanges()
throws SchedulerDynamicEditException {
+ // Recompute the leaf queue template capacities from the current cluster
+ // resource for ABSOLUTE_RESOURCE mode. The template fraction is otherwise
+ // computed only at reinitialize; if a leaf queue was auto-created
+ // while the cluster resource was zero during RM recovery before any
+ // NodeManager registered the fraction stays 0 and the queue never regains
+ // capacity. This refresh lets the template pick up the real capacity once
+ // NodeManagers register. No operation unless it is ABSOLUTE_RESOURCE.
+ managedParentQueue.updateTemplateCapacitiesForAbsoluteResource();
+
// Update template absolute capacities as the capacities could have changed
// in weight mode
updateTemplateAbsoluteCapacities(managedParentQueue.getQueueCapacities(),
@@ -716,8 +727,18 @@ public AutoCreatedLeafQueueConfig
getInitialLeafQueueConfiguration(
getAbsoluteCapacity(nodeLabel) - parentQueueState.
getAbsoluteActivatedChildQueueCapacity(nodeLabel) + EPSILON;
- if (availableCapacity >= leafQueueTemplateCapacities
- .getAbsoluteCapacity(nodeLabel)) {
+ // In ABSOLUTE_RESOURCE mode, don't activate a leaf queue while the
cluster
+ // resource is 0 during RM recovery before NodeManagers register.
+ // Otherwise, it gets initialized with zero capacity and never updates.
+ // Create it deactivated instead, and let the monitor activate it once
+ // cluster resources are available. Percentage and weight modes are
unaffected.
+ boolean deferActivationUntilClusterResource =
+ managedParentQueue.getCapacityConfigType() == ABSOLUTE_RESOURCE
+ && managedParentQueue.getQueueContext().getClusterResource()
+ .getMemorySize() <= 0;
+
+ if (!deferActivationUntilClusterResource && availableCapacity
+ >= leafQueueTemplateCapacities.getAbsoluteCapacity(nodeLabel)) {
updateCapacityFromTemplate(capacities, nodeLabel);
activate(leafQueue, nodeLabel);
} else{
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
index ca3e52fec15..28b59dbc544 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
@@ -21,6 +21,7 @@
import static
org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NO_LABEL;
import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -338,4 +339,63 @@ public void testApplicationRunningWithDRF() throws
Exception {
ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue(QUEUED);
assertEquals(parentQueue, autoCreatedLeafQueue.getParent());
}
+
+ @Test
+ @Timeout(value = 20)
+ public void testAutoCreatedLeafQueueRecoversCapacityAfterNodeRegisters()
+ throws Exception {
+ try {
+ CapacitySchedulerConfiguration csConf =
+ setupSimpleQueueConfiguration(false);
+ setupMinMaxResourceConfiguration(csConf);
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ csConf.setOverrideWithQueueMappings(true);
+ setupGroupQueueMappings(QUEUED, csConf, "%user");
+
+ mockRM = new MockRM(csConf);
+ cs = (CapacityScheduler) mockRM.getResourceScheduler();
+ mockRM.start();
+ cs.start();
+
+ submitApp(mockRM, cs.getQueue(QUEUED), TEST_GROUPUSER, TEST_GROUPUSER, 1,
+ 1);
+
+ AutoCreatedLeafQueue autoCreatedLeafQueue =
+ (AutoCreatedLeafQueue) cs.getQueue(TEST_GROUPUSER);
+ ManagedParentQueue parentQueue = (ManagedParentQueue)
cs.getQueue(QUEUED);
+ assertNotNull(autoCreatedLeafQueue, "Auto Creation of Queue failed");
+ assertEquals(parentQueue, autoCreatedLeafQueue.getParent());
+
+ GuaranteedOrZeroCapacityOverTimePolicy policy =
+ (GuaranteedOrZeroCapacityOverTimePolicy) parentQueue
+ .getAutoCreatedQueueManagementPolicy();
+
+ // With no cluster resource the queue must be created at zero capacity
and
+ // must NOT be activated - activating it here is exactly what strands it
at
+ // zero capacity for the lifetime of the queue.
+ assertEquals(0.0f, autoCreatedLeafQueue.getCapacity(), EPSILON);
+ assertEquals(0.0f, autoCreatedLeafQueue.getAbsoluteCapacity(), EPSILON);
+ assertFalse(policy.isActive(autoCreatedLeafQueue, NO_LABEL),
+ "Queue must not be activated while the cluster resource is
zero");
+
+ // A NodeManager registers: the cluster resource becomes available.
+ mockRM.registerNode("127.0.0.1:1234", 250 * GB, 40);
+ mockRM.drainEvents();
+
+ // The queue management monitor runs, which is what
+ // QueueManagementDynamicEditPolicy does on its monitoring interval.
+ List<QueueManagementChange> changes =
+ policy.computeQueueManagementChanges();
+ parentQueue.validateAndApplyQueueManagementChanges(changes);
+
+ validateCapacities(autoCreatedLeafQueue, 0.4f, 0.04f, 1f, 0.6f);
+ assertTrue(policy.isActive(autoCreatedLeafQueue, NO_LABEL),
+ "Queue must be activated once the cluster resource is
available");
+ assertEquals(0.04f,
+ policy.getAbsoluteActivatedChildQueueCapacity(NO_LABEL), EPSILON);
+ } finally {
+ cleanupQueue(TEST_GROUPUSER);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
