[
https://issues.apache.org/jira/browse/YARN-11000?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17739641#comment-17739641
]
ASF GitHub Bot commented on YARN-11000:
---------------------------------------
tomicooler commented on code in PR #5783:
URL: https://github.com/apache/hadoop/pull/5783#discussion_r1250921579
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java:
##########
@@ -1136,11 +1137,43 @@ public void refreshAfterResourceCalculation(Resource
clusterResource,
CSQueueUtils.updateConfiguredCapacityMetrics(resourceCalculator,
labelManager.getResourceByLabel(null, clusterResource),
RMNodeLabelsManager.NO_LABEL, this);
+
+ LOG.info("Refresh after resource calculation (PARENT) {}\n"
+ + "effectiveMinResource = {}\n"
+ + "effectiveMaxResource = {}\n"
+ + "capacity = {}\n"
+ + "maxCapacity = {}\n"
+ + "absoluteCapacity = {}\n"
+ + "absoluteMaxCapacity = {}",
Review Comment:
effectiveMin/Max resource by labels?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbsoluteResourceCapacityCalculator.java:
##########
@@ -44,8 +44,14 @@ public double calculateMinimumResource(
double remainingResourceRatio =
resourceCalculationDriver.getRemainingRatioOfResource(
label, resourceName);
- return normalizedRatio * remainingResourceRatio *
context.getCurrentMinimumCapacityEntry(
+ double minimumCapacity = context.getCurrentMinimumCapacityEntry(
Review Comment:
check: is this still needed?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java:
##########
@@ -85,7 +85,6 @@ public void
reinitializeFromTemplate(AutoCreatedLeafQueueConfig leafQueueTemplat
//reset capacities for the leaf queue
mergeCapacities(capacities, leafQueueTemplate.getResourceQuotas());
-
Review Comment:
remove
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java:
##########
@@ -194,9 +198,12 @@ public void testAMResourceLimit() throws Exception {
Resource clusterResource = Resource.newInstance(80 * GB, 40);
root.updateClusterResource(clusterResource, new ResourceLimits(
clusterResource));
+ queue = (LeafQueue) root.getChildQueues().stream().filter(
Review Comment:
no need for this
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java:
##########
@@ -1945,8 +1949,12 @@ private void updateCurrentResourceLimits(
public void refreshAfterResourceCalculation(Resource clusterResource,
ResourceLimits resourceLimits) {
lastClusterResource = clusterResource;
+
// Update maximum applications for the queue and for users
- updateMaximumApplications();
+
updateMaximumApplications(queueContext.getConfiguration().isLegacyQueueMode() ||
Review Comment:
NIT: refreshAfterResourceCalculation is only called from non Legacy
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java:
##########
@@ -53,6 +56,7 @@ public class TestReservationQueue {
new DefaultResourceCalculator();
private ReservationQueue autoCreatedLeafQueue;
private PlanQueue planQueue;
+ private Resource clusterResource = Resources.createResource(100 * 16 * GB,
100 * 32);
Review Comment:
final
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java:
##########
@@ -65,6 +69,12 @@ public void setup() throws IOException,
SchedulerDynamicEditException {
CapacitySchedulerQueueManager.class);
ConfiguredNodeLabels labels = new ConfiguredNodeLabels(csConf);
when(csQm.getConfiguredNodeLabelsForAllQueues()).thenReturn(labels);
+ NullRMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
+ mgr.init(csConf);
+ clusterResource = Resources.createResource(100 * 16 * GB, 100 * 32);
Review Comment:
it's already set
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbsoluteResourceCapacityCalculator.java:
##########
@@ -59,7 +65,20 @@ public double calculateMaximumResource(
public void updateCapacitiesAfterCalculation(
ResourceCalculationDriver resourceCalculationDriver, CSQueue queue,
String label) {
CapacitySchedulerQueueCapacityHandler.setQueueCapacities(
-
resourceCalculationDriver.getUpdateContext().getUpdatedClusterResource(label),
queue, label);
+ resourceCalculationDriver.getUpdateContext()
+ .getUpdatedClusterResource(label), queue, label);
+
+ if (queue.getParent() != null) {
Review Comment:
check: is this still needed?
> Replace queue resource calculation logic in updateClusterResource
> -----------------------------------------------------------------
>
> Key: YARN-11000
> URL: https://issues.apache.org/jira/browse/YARN-11000
> Project: Hadoop YARN
> Issue Type: Sub-task
> Components: capacity scheduler
> Reporter: Andras Gyori
> Assignee: Andras Gyori
> Priority: Major
> Labels: pull-request-available
> Time Spent: 2h 10m
> Remaining Estimate: 0h
>
> YARN-10965 introduces a brand new queue calculation system. In order to
> simplify the review process, this issue replaces the current logic with the
> newly introduced one.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
