Author: bikas
Date: Tue Apr 2 18:09:40 2013
New Revision: 1463656
URL: http://svn.apache.org/r1463656
Log:
Merge -c 1463653 from trunk to branch-2 for YARN-382. SchedulerUtils improve
way normalizeRequest sets the resource capabilities (Zhijie Shen via bikas)
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1463656&r1=1463655&r2=1463656&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Tue Apr 2
18:09:40 2013
@@ -117,6 +117,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-516. Fix failure in TestContainerLocalizer caused by HADOOP-9357.
(Andrew Wang via vinodkv)
+
+ YARN-382. SchedulerUtils improve way normalizeRequest sets the resource
+ capabilities. (Zhijie Shen via bikas)
Release 2.0.4-alpha - UNRELEASED
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1463656&r1=1463655&r2=1463656&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
Tue Apr 2 18:09:40 2013
@@ -771,6 +771,11 @@ public class RMAppAttemptImpl implements
// Set the masterContainer
appAttempt.setMasterContainer(amContainerAllocation.getContainers().get(
0));
+ // Updating CLC's resource is no longer necessary once YARN-486 is
+ // completed, because nothing from Container to CLC will be copied into
+ // CLC then.
+ appAttempt.getSubmissionContext().getAMContainerSpec().setResource(
+ appAttempt.getMasterContainer().getResource());
RMStateStore store = appAttempt.rmContext.getStateStore();
appAttempt.storeAttempt(store);
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java?rev=1463656&r1=1463655&r2=1463656&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
Tue Apr 2 18:09:40 2013
@@ -108,8 +108,7 @@ public class SchedulerUtils {
Resource normalized =
Resources.normalize(
resourceCalculator, ask.getCapability(), minimumResource);
- ask.getCapability().setMemory(normalized.getMemory());
- ask.getCapability().setVirtualCores(normalized.getVirtualCores());
+ ask.setCapability(normalized);
}
}
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1463656&r1=1463655&r2=1463656&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
Tue Apr 2 18:09:40 2013
@@ -202,9 +202,10 @@ public class TestRMAppAttemptTransitions
submissionContext = mock(ApplicationSubmissionContext.class);
when(submissionContext.getUser()).thenReturn(user);
when(submissionContext.getQueue()).thenReturn(queue);
- ContainerLaunchContext amContainerSpec =
mock(ContainerLaunchContext.class);
- Resource resource = mock(Resource.class);
- when(amContainerSpec.getResource()).thenReturn(resource);
+ Resource resource = BuilderUtils.newResource(1536, 1);
+ ContainerLaunchContext amContainerSpec =
+ BuilderUtils.newContainerLaunchContext(null, user, resource, null,
null,
+ null, null, null, null);
when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
unmanagedAM = false;
@@ -469,8 +470,10 @@ public class TestRMAppAttemptTransitions
// Mock the allocation of AM container
Container container = mock(Container.class);
+ Resource resource = BuilderUtils.newResource(2048, 1);
when(container.getId()).thenReturn(
BuilderUtils.newContainerId(applicationAttempt.getAppAttemptId(), 1));
+ when(container.getResource()).thenReturn(resource);
Allocation allocation = mock(Allocation.class);
when(allocation.getContainers()).
thenReturn(Collections.singletonList(container));
@@ -491,6 +494,9 @@ public class TestRMAppAttemptTransitions
applicationAttempt.handle(
new RMAppAttemptStoredEvent(
applicationAttempt.getAppAttemptId(), null));
+ assertEquals(resource,
+ applicationAttempt.getSubmissionContext()
+ .getAMContainerSpec().getResource());
testAppAttemptAllocatedState(container);
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java?rev=1463656&r1=1463655&r2=1463656&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
Tue Apr 2 18:09:40 2013
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -43,35 +42,23 @@ public class TestSchedulerUtils {
// case negative memory
ask.setCapability(Resources.createResource(-1024));
- Resource before = ask.getCapability();
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null,
minResource);
- Resource after = ask.getCapability();
assertEquals(minMemory, ask.getCapability().getMemory());
- assertTrue(before == after);
// case zero memory
ask.setCapability(Resources.createResource(0));
- before = ask.getCapability();
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null,
minResource);
- after = ask.getCapability();
assertEquals(minMemory, ask.getCapability().getMemory());
- assertTrue(before == after);
// case memory is a multiple of minMemory
ask.setCapability(Resources.createResource(2 * minMemory));
- before = ask.getCapability();
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null,
minResource);
- after = ask.getCapability();
assertEquals(2 * minMemory, ask.getCapability().getMemory());
- assertTrue(before == after);
// case memory is not a multiple of minMemory
ask.setCapability(Resources.createResource(minMemory + 10));
- before = ask.getCapability();
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null,
minResource);
- after = ask.getCapability();
assertEquals(2 * minMemory, ask.getCapability().getMemory());
- assertTrue(before == after);
}
@@ -86,33 +73,24 @@ public class TestSchedulerUtils {
// case negative memory/vcores
ask.setCapability(Resources.createResource(-1024, -1));
- Resource before = ask.getCapability();
SchedulerUtils.normalizeRequest(
ask, resourceCalculator, clusterResource, minResource);
- Resource after = ask.getCapability();
assertEquals(minResource, ask.getCapability());
- assertTrue(before == after);
// case zero memory/vcores
ask.setCapability(Resources.createResource(0, 0));
- before = ask.getCapability();
SchedulerUtils.normalizeRequest(
ask, resourceCalculator, clusterResource, minResource);
- after = ask.getCapability();
assertEquals(minResource, ask.getCapability());
assertEquals(1, ask.getCapability().getVirtualCores());
assertEquals(1024, ask.getCapability().getMemory());
- assertTrue(before == after);
// case non-zero memory & zero cores
ask.setCapability(Resources.createResource(1536, 0));
- before = ask.getCapability();
SchedulerUtils.normalizeRequest(
ask, resourceCalculator, clusterResource, minResource);
- after = ask.getCapability();
assertEquals(Resources.createResource(2048, 1), ask.getCapability());
assertEquals(1, ask.getCapability().getVirtualCores());
assertEquals(2048, ask.getCapability().getMemory());
- assertTrue(before == after);
}
}
