This is an automated email from the ASF dual-hosted git repository.
bibinchundatt pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 71f5bfb YARN-9508. YarnConfiguration areNodeLabel enabled is costly
in allocation flow. Contributed by Bilwa S T.
71f5bfb is described below
commit 71f5bfb822ce8833a99c829185fb2efc4f88a173
Author: bibinchundatt <[email protected]>
AuthorDate: Wed May 15 13:30:09 2019 +0530
YARN-9508. YarnConfiguration areNodeLabel enabled is costly in allocation
flow. Contributed by Bilwa S T.
(cherry picked from commit 570fa2da20706490dc7823efd0ce0cef3ddc81f9)
---
.../resourcemanager/DefaultAMSProcessor.java | 5 ++++-
.../yarn/server/resourcemanager/RMAppManager.java | 5 ++++-
.../yarn/server/resourcemanager/RMServerUtils.java | 5 +++--
.../resourcemanager/scheduler/SchedulerUtils.java | 22 +++++++++++-----------
.../server/resourcemanager/TestAppManager.java | 1 +
.../scheduler/TestSchedulerUtils.java | 3 ++-
6 files changed, 25 insertions(+), 16 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index 9e05eda..a3bdb2f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -116,6 +116,7 @@ final class DefaultAMSProcessor implements
ApplicationMasterServiceProcessor {
private RMContext rmContext;
private ResourceProfilesManager resourceProfilesManager;
private boolean timelineServiceV2Enabled;
+ private boolean nodelabelsEnabled;
@Override
public void init(ApplicationMasterServiceContext amsContext,
@@ -124,6 +125,8 @@ final class DefaultAMSProcessor implements
ApplicationMasterServiceProcessor {
this.resourceProfilesManager = rmContext.getResourceProfilesManager();
this.timelineServiceV2Enabled = YarnConfiguration.
timelineServiceV2Enabled(rmContext.getYarnConfiguration());
+ this.nodelabelsEnabled = YarnConfiguration
+ .areNodeLabelsEnabled(rmContext.getYarnConfiguration());
}
@Override
@@ -241,7 +244,7 @@ final class DefaultAMSProcessor implements
ApplicationMasterServiceProcessor {
try {
RMServerUtils.normalizeAndValidateRequests(ask,
maximumCapacity, app.getQueue(),
- getScheduler(), getRmContext());
+ getScheduler(), getRmContext(), nodelabelsEnabled);
} catch (InvalidResourceRequestException e) {
RMAppAttempt rmAppAttempt = app.getRMAppAttempt(appAttemptId);
handleInvalidResourceException(e, rmAppAttempt);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 0679b0e..feaff00 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -96,6 +96,7 @@ public class RMAppManager implements
EventHandler<RMAppManagerEvent>,
private Configuration conf;
private YarnAuthorizationProvider authorizer;
private boolean timelineServiceV2Enabled;
+ private boolean nodeLabelsEnabled;
public RMAppManager(RMContext context,
YarnScheduler scheduler, ApplicationMasterService masterService,
@@ -118,6 +119,8 @@ public class RMAppManager implements
EventHandler<RMAppManagerEvent>,
this.authorizer = YarnAuthorizationProvider.getInstance(conf);
this.timelineServiceV2Enabled = YarnConfiguration.
timelineServiceV2Enabled(conf);
+ this.nodeLabelsEnabled = YarnConfiguration
+ .areNodeLabelsEnabled(rmContext.getYarnConfiguration());
}
/**
@@ -575,7 +578,7 @@ public class RMAppManager implements
EventHandler<RMAppManagerEvent>,
Resource maxAllocation = scheduler.getMaximumResourceCapability(queue);
for (ResourceRequest amReq : amReqs) {
SchedulerUtils.normalizeAndValidateRequest(amReq, maxAllocation,
- queue, scheduler, isRecovery, rmContext, null);
+ queue, isRecovery, rmContext, null, nodeLabelsEnabled);
amReq.setCapability(scheduler.getNormalizedResource(
amReq.getCapability(), maxAllocation));
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index bad890f..25eb979 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -244,7 +244,8 @@ public class RMServerUtils {
*/
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
Resource maximumAllocation, String queueName, YarnScheduler scheduler,
- RMContext rmContext) throws InvalidResourceRequestException {
+ RMContext rmContext, boolean nodeLabelsEnabled)
+ throws InvalidResourceRequestException {
// Get queue from scheduler
QueueInfo queueInfo = null;
try {
@@ -256,7 +257,7 @@ public class RMServerUtils {
for (ResourceRequest resReq : ask) {
SchedulerUtils.normalizeAndValidateRequest(resReq, maximumAllocation,
- queueName, scheduler, rmContext, queueInfo);
+ queueName, rmContext, queueInfo, nodeLabelsEnabled);
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index a048dac..36746c0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
@@ -260,12 +259,12 @@ public class SchedulerUtils {
}
public static void normalizeAndValidateRequest(ResourceRequest resReq,
- Resource maximumAllocation, String queueName, YarnScheduler scheduler,
- boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
- throws InvalidResourceRequestException {
+ Resource maximumAllocation, String queueName, boolean isRecovery,
+ RMContext rmContext, QueueInfo queueInfo, boolean nodeLabelsEnabled)
+ throws InvalidResourceRequestException {
Configuration conf = rmContext.getYarnConfiguration();
// If Node label is not enabled throw exception
- if (null != conf && !YarnConfiguration.areNodeLabelsEnabled(conf)) {
+ if (null != conf && !nodeLabelsEnabled) {
String labelExp = resReq.getNodeLabelExpression();
if (!(RMNodeLabelsManager.NO_LABEL.equals(labelExp)
|| null == labelExp)) {
@@ -281,7 +280,8 @@ public class SchedulerUtils {
}
if (null == queueInfo) {
try {
- queueInfo = scheduler.getQueueInfo(queueName, false, false);
+ queueInfo = rmContext.getScheduler().getQueueInfo(queueName, false,
+ false);
} catch (IOException e) {
//Queue may not exist since it could be auto-created in case of
// dynamic queues
@@ -295,11 +295,11 @@ public class SchedulerUtils {
}
public static void normalizeAndValidateRequest(ResourceRequest resReq,
- Resource maximumAllocation, String queueName, YarnScheduler scheduler,
- RMContext rmContext, QueueInfo queueInfo)
- throws InvalidResourceRequestException {
- normalizeAndValidateRequest(resReq, maximumAllocation, queueName,
scheduler,
- false, rmContext, queueInfo);
+ Resource maximumAllocation, String queueName, RMContext rmContext,
+ QueueInfo queueInfo, boolean nodeLabelsEnabled)
+ throws InvalidResourceRequestException {
+ normalizeAndValidateRequest(resReq, maximumAllocation, queueName, false,
+ rmContext, queueInfo, nodeLabelsEnabled);
}
/**
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 3c39f64..31c16a4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -203,6 +203,7 @@ public class TestAppManager extends AppManagerTestBase{
metricsPublisher = mock(SystemMetricsPublisher.class);
context.setSystemMetricsPublisher(metricsPublisher);
context.setRMApplicationHistoryWriter(writer);
+ ((RMContextImpl)context).setYarnConfiguration(new YarnConfiguration());
return context;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index 6969242..7302705 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -1050,7 +1050,8 @@ public class TestSchedulerUtils {
Resource maxAllocation)
throws InvalidResourceRequestException {
SchedulerUtils.normalizeAndValidateRequest(resReq, maxAllocation,
queueName,
- scheduler, rmContext, null);
+ rmContext, null, YarnConfiguration
+ .areNodeLabelsEnabled(rmContext.getYarnConfiguration()));
}
private static class InvalidResourceRequestExceptionMessageGenerator {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
