This is an automated email from the ASF dual-hosted git repository. jhung pushed a commit to branch YARN-8200 in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit df87983bb947eed840f3f08cdcba025eed1ecd8f Author: Jonathan Hung <jh...@linkedin.com> AuthorDate: Wed Mar 27 09:55:51 2019 -0700 YARN-9409. Port resource type changes from YARN-7237 to branch-2 --- .../scheduler/AbstractYarnScheduler.java | 15 +++----- .../hadoop/yarn/server/resourcemanager/MockNM.java | 41 ++++++++++++++-------- .../hadoop/yarn/server/resourcemanager/MockRM.java | 9 +++++ .../scheduler/capacity/TestUtils.java | 41 +++++++++++++++------- 4 files changed, 70 insertions(+), 36 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 425a0b1..fd50f20 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; @@ -797,7 +796,7 @@ public abstract class AbstractYarnScheduler writeLock.unlock(); } } - + /** * Process resource update on a node. */ @@ -900,12 +899,12 @@ public abstract class AbstractYarnScheduler LOG.info("Updated the cluste max priority to maxClusterLevelAppPriority = " + maxClusterLevelAppPriority); } - + /** * Sanity check increase/decrease request, and return * SchedulerContainerResourceChangeRequest according to given * UpdateContainerRequest. - * + * * <pre> * - Returns non-null value means validation succeeded * - Throw exception when any other error happens @@ -1334,9 +1333,7 @@ public abstract class AbstractYarnScheduler } /* - * Get a Resource object with for the minimum allocation possible. If resource - * profiles are enabled then the 'minimum' resource profile will be used. If - * they are not enabled, use the minimums specified in the config files. + * Get a Resource object with for the minimum allocation possible. * * @return a Resource object with the minimum allocation for the scheduler */ @@ -1347,9 +1344,7 @@ public abstract class AbstractYarnScheduler } /** - * Get a Resource object with for the maximum allocation possible. If resource - * profiles are enabled then the 'maximum' resource profile will be used. If - * they are not enabled, use the maximums specified in the config files. + * Get a Resource object with for the maximum allocation possible. * * @return a Resource object with the maximum allocation for the scheduler */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index db92d7c..a02ac84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -47,14 +47,14 @@ import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.YarnVersionInfo; +import org.apache.hadoop.yarn.util.resource.Resources; import org.mortbay.log.Log; public class MockNM { private int responseId; private NodeId nodeId; - private long memory; - private int vCores; + private Resource capatibility; private ResourceTrackerService resourceTracker; private int httpPort = 2; private MasterKey currentContainerTokenMasterKey; @@ -75,13 +75,25 @@ public class MockNM { public MockNM(String nodeIdStr, int memory, int vcores, ResourceTrackerService resourceTracker) { - this(nodeIdStr, memory, vcores, resourceTracker, YarnVersionInfo.getVersion()); + this(nodeIdStr, memory, vcores, resourceTracker, + YarnVersionInfo.getVersion()); } public MockNM(String nodeIdStr, int memory, int vcores, ResourceTrackerService resourceTracker, String version) { - this.memory = memory; - this.vCores = vcores; + this(nodeIdStr, Resource.newInstance(memory, vcores), resourceTracker, + version); + } + + public MockNM(String nodeIdStr, Resource capatibility, + ResourceTrackerService resourceTracker) { + this(nodeIdStr, capatibility, resourceTracker, + YarnVersionInfo.getVersion()); + } + + public MockNM(String nodeIdStr, Resource capatibility, + ResourceTrackerService resourceTracker, String version) { + this.capatibility = capatibility; this.resourceTracker = resourceTracker; this.version = version; String[] splits = nodeIdStr.split(":"); @@ -146,8 +158,7 @@ public class MockNM { RegisterNodeManagerRequest.class); req.setNodeId(nodeId); req.setHttpPort(httpPort); - Resource resource = BuilderUtils.newResource(memory, vCores); - req.setResource(resource); + req.setResource(capatibility); req.setContainerStatuses(containerReports); req.setNMVersion(version); req.setRunningApplications(runningApplications); @@ -158,8 +169,7 @@ public class MockNM { this.currentNMTokenMasterKey = registrationResponse.getNMTokenMasterKey(); Resource newResource = registrationResponse.getResource(); if (newResource != null) { - memory = (int) newResource.getMemorySize(); - vCores = newResource.getVirtualCores(); + capatibility = Resources.clone(newResource); } containerStats.clear(); if (containerReports != null) { @@ -185,7 +195,7 @@ public class MockNM { long containerId, ContainerState containerState) throws Exception { ContainerStatus containerStatus = BuilderUtils.newContainerStatus( BuilderUtils.newContainerId(attemptId, containerId), containerState, - "Success", 0, BuilderUtils.newResource(memory, vCores)); + "Success", 0, capatibility); ArrayList<ContainerStatus> containerStatusList = new ArrayList<ContainerStatus>(1); containerStatusList.add(containerStatus); @@ -266,19 +276,22 @@ public class MockNM { Resource newResource = heartbeatResponse.getResource(); if (newResource != null) { - memory = newResource.getMemorySize(); - vCores = newResource.getVirtualCores(); + capatibility = Resources.clone(newResource); } return heartbeatResponse; } public long getMemory() { - return memory; + return capatibility.getMemorySize(); } public int getvCores() { - return vCores; + return capatibility.getVirtualCores(); + } + + public Resource getCapatibility() { + return capatibility; } public String getVersion() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 0754584..fd327f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -848,6 +848,15 @@ public class MockRM extends ResourceManager { return nm; } + public MockNM registerNode(String nodeIdStr, Resource nodeCapatibility) + throws Exception { + MockNM nm = new MockNM(nodeIdStr, nodeCapatibility, + getResourceTrackerService()); + nm.registerNode(); + drainEventsImplicitly(); + return nm; + } + public void sendNodeStarted(MockNM nm) throws Exception { RMNodeImpl node = (RMNodeImpl) getRMContext().getRMNodes().get( nm.getNodeId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index e81ffbd..cb150e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -18,16 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.Map; -import java.util.Set; - +import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -38,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Event; @@ -53,20 +45,28 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import com.google.common.collect.Sets; +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; public class TestUtils { private static final Log LOG = LogFactory.getLog(TestUtils.class); @@ -456,4 +456,21 @@ public class TestUtils { cs.submitResourceCommitRequest(clusterResource, csAssignment); } + + /** + * An easy way to create resources other than memory and vcores for tests. + * @param memory memory + * @param vcores vcores + * @param nameToValues resource types other than memory and vcores. + * @return created resource + */ + public static Resource createResource(long memory, int vcores, + Map<String, Integer> nameToValues) { + Resource res = Resource.newInstance(memory, vcores); + for (Map.Entry<String, Integer> entry : nameToValues.entrySet()) { + res.setResourceInformation(entry.getKey(), ResourceInformation + .newInstance(entry.getKey(), "", entry.getValue())); + } + return res; + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org