YARN-5270. Solve miscellaneous issues caused by YARN-4844. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/819224dc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/819224dc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/819224dc Branch: refs/heads/YARN-5355 Commit: 819224dcf9c683aa52f58633ac8e13663f1916d8 Parents: f292624 Author: Jian He <[email protected]> Authored: Mon Jul 11 22:36:20 2016 -0700 Committer: Jian He <[email protected]> Committed: Mon Jul 11 22:36:20 2016 -0700 ---------------------------------------------------------------------- .../v2/app/job/impl/TaskAttemptImpl.java | 3 +- .../v2/app/rm/RMContainerAllocator.java | 4 +- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 2 +- .../hadoop/mapreduce/TestTypeConverter.java | 4 +- .../org/apache/hadoop/mapred/YARNRunner.java | 2 +- .../org/apache/hadoop/yarn/sls/SLSRunner.java | 2 +- .../sls/scheduler/ResourceSchedulerWrapper.java | 12 ++--- .../sls/scheduler/SLSCapacityScheduler.java | 12 ++--- .../hadoop/yarn/api/records/Resource.java | 47 ++++++++++++------ .../src/main/proto/yarn_protos.proto | 2 +- .../TestApplicationClientProtocolOnHA.java | 2 +- .../api/records/impl/pb/ResourcePBImpl.java | 15 +++--- .../resource/DefaultResourceCalculator.java | 2 +- .../resource/DominantResourceCalculator.java | 8 +-- .../yarn/util/resource/ResourceCalculator.java | 17 ++++++- .../hadoop/yarn/util/resource/Resources.java | 51 ++++++++++++-------- .../yarn/util/resource/TestResources.java | 6 +-- .../hadoop/yarn/server/utils/BuilderUtils.java | 5 +- .../hadoop/yarn/TestYarnServerApiClasses.java | 2 +- .../amrmproxy/BaseAMRMProxyTest.java | 3 +- .../metrics/TestNodeManagerMetrics.java | 8 +-- .../scheduler/ClusterNodeTracker.java | 2 +- .../resourcemanager/scheduler/QueueMetrics.java | 27 ++++++----- .../scheduler/SchedulerApplicationAttempt.java | 2 +- .../capacity/CapacityHeadroomProvider.java | 2 +- .../scheduler/fair/FSAppAttempt.java | 4 +- .../scheduler/fair/FSLeafQueue.java | 6 +-- .../fair/policies/ComputeFairShares.java | 4 +- .../webapp/dao/ResourceInfo.java | 4 +- .../yarn/server/resourcemanager/MockAM.java | 3 +- .../yarn/server/resourcemanager/MockNodes.java | 6 +-- .../yarn/server/resourcemanager/MockRM.java | 10 ++-- .../TestResourceTrackerService.java | 6 +-- .../TestWorkPreservingRMRestart.java | 2 +- .../reservation/TestReservationSystemUtil.java | 2 +- .../scheduler/TestQueueMetrics.java | 6 +-- .../security/TestDelegationTokenRenewer.java | 4 +- 37 files changed, 172 insertions(+), 127 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 0da3afb..4305824 100755 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -135,7 +135,6 @@ import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.util.Clock; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.RackResolver; import com.google.common.annotations.VisibleForTesting; @@ -667,7 +666,7 @@ public abstract class TaskAttemptImpl implements //TODO:create the resource reqt for this Task attempt this.resourceCapability = recordFactory.newRecordInstance(Resource.class); - this.resourceCapability.setMemory( + this.resourceCapability.setMemorySize( getMemoryRequired(conf, taskId.getTaskType())); this.resourceCapability.setVirtualCores( getCpuRequired(conf, taskId.getTaskType())); http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index 217337e..ecd75db 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -389,7 +389,7 @@ public class RMContainerAllocator extends RMContainerRequestor } } // set the resources - reqEvent.getCapability().setMemory(mapResourceRequest.getMemorySize()); + reqEvent.getCapability().setMemorySize(mapResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( mapResourceRequest.getVirtualCores()); scheduledRequests.addMap(reqEvent);//maps are immediately scheduled @@ -417,7 +417,7 @@ public class RMContainerAllocator extends RMContainerRequestor } } // set the resources - reqEvent.getCapability().setMemory(reduceResourceRequest.getMemorySize()); + reqEvent.getCapability().setMemorySize(reduceResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( reduceResourceRequest.getVirtualCores()); if (reqEvent.getEarlierAttemptFailed()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index 6ba93e6..3472b1f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -254,7 +254,7 @@ public class MRApp extends MRAppMaster { // the job can reaches the final state when MRAppMaster shuts down. this.successfullyUnregistered.set(unregistered); this.assignedQueue = assignedQueue; - this.resource = Resource.newInstance(1234L, 2L); + this.resource = Resource.newInstance(1234L, 2); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java index 2027d01..629a246 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java @@ -89,7 +89,7 @@ public class TestTypeConverter { ApplicationResourceUsageReport appUsageRpt = Records .newRecord(ApplicationResourceUsageReport.class); Resource r = Records.newRecord(Resource.class); - r.setMemory(2048); + r.setMemorySize(2048); appUsageRpt.setNeededResources(r); appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); @@ -128,7 +128,7 @@ public class TestTypeConverter { ApplicationResourceUsageReport appUsageRpt = Records .newRecord(ApplicationResourceUsageReport.class); Resource r = Records.newRecord(Resource.class); - r.setMemory(2048); + r.setMemorySize(2048); appUsageRpt.setNeededResources(r); appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index b30641e..4c6f0f3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -337,7 +337,7 @@ public class YARNRunner implements ClientProtocol { // Setup resource requirements Resource capability = recordFactory.newRecordInstance(Resource.class); - capability.setMemory( + capability.setMemorySize( conf.getInt( MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB ) http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index 3be8ff1..c9c5c38 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -323,7 +323,7 @@ public class SLSRunner { if (jsonTask.containsKey("container.memory")) { int containerMemory = Integer.parseInt( jsonTask.get("container.memory").toString()); - res.setMemory(containerMemory); + res.setMemorySize(containerMemory); } if (jsonTask.containsKey("container.vcores")) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index 014aa89..393300c 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -542,11 +542,11 @@ final public class ResourceSchedulerWrapper } ); metrics.register("variable.cluster.allocated.vcores", - new Gauge<Long>() { + new Gauge<Integer>() { @Override - public Long getValue() { + public Integer getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0L; + return 0; } else { return scheduler.getRootQueueMetrics().getAllocatedVirtualCores(); } @@ -566,11 +566,11 @@ final public class ResourceSchedulerWrapper } ); metrics.register("variable.cluster.available.vcores", - new Gauge<Long>() { + new Gauge<Integer>() { @Override - public Long getValue() { + public Integer getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0L; + return 0; } else { return scheduler.getRootQueueMetrics().getAvailableVirtualCores(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java index 5cd53fe..1c3fa79 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java @@ -519,11 +519,11 @@ public class SLSCapacityScheduler extends CapacityScheduler implements } ); metrics.register("variable.cluster.allocated.vcores", - new Gauge<Long>() { + new Gauge<Integer>() { @Override - public Long getValue() { + public Integer getValue() { if(getRootQueueMetrics() == null) { - return 0L; + return 0; } else { return getRootQueueMetrics().getAllocatedVirtualCores(); } @@ -543,11 +543,11 @@ public class SLSCapacityScheduler extends CapacityScheduler implements } ); metrics.register("variable.cluster.available.vcores", - new Gauge<Long>() { + new Gauge<Integer>() { @Override - public Long getValue() { + public Integer getValue() { if(getRootQueueMetrics() == null) { - return 0l; + return 0; } else { return getRootQueueMetrics().getAvailableVirtualCores(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index e2d8472..0d31b6f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -18,14 +18,14 @@ package org.apache.hadoop.yarn.api.records; -import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.commons.lang.NotImplementedException; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.util.Records; + /** * <p><code>Resource</code> models a set of computer resources in the * cluster.</p> @@ -55,9 +55,18 @@ public abstract class Resource implements Comparable<Resource> { @Public @Stable - public static Resource newInstance(long memory, long vCores) { + public static Resource newInstance(int memory, int vCores) { + Resource resource = Records.newRecord(Resource.class); + resource.setMemorySize(memory); + resource.setVirtualCores(vCores); + return resource; + } + + @Public + @Stable + public static Resource newInstance(long memory, int vCores) { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(vCores); return resource; } @@ -77,17 +86,31 @@ public abstract class Resource implements Comparable<Resource> { * Get <em>memory</em> of the resource. * @return <em>memory</em> of the resource */ - @Private - @Unstable - public abstract long getMemorySize(); - + @Public + @Stable + public long getMemorySize() { + throw new NotImplementedException( + "This method is implemented by ResourcePBImpl"); + } + + /** + * Set <em>memory</em> of the resource. + * @param memory <em>memory</em> of the resource + */ + @Public + @Deprecated + public abstract void setMemory(int memory); + /** * Set <em>memory</em> of the resource. * @param memory <em>memory</em> of the resource */ @Public @Stable - public abstract void setMemory(long memory); + public void setMemorySize(long memory) { + throw new NotImplementedException( + "This method is implemented by ResourcePBImpl"); + } /** @@ -103,10 +126,6 @@ public abstract class Resource implements Comparable<Resource> { @Public @Evolving public abstract int getVirtualCores(); - - @Public - @Unstable - public abstract long getVirtualCoresSize(); /** * Set <em>number of virtual cpu cores</em> of the resource. @@ -120,7 +139,7 @@ public abstract class Resource implements Comparable<Resource> { */ @Public @Evolving - public abstract void setVirtualCores(long vCores); + public abstract void setVirtualCores(int vCores); @Override public int hashCode() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index c84f4e9..6c337cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -55,7 +55,7 @@ message ContainerIdProto { message ResourceProto { optional int64 memory = 1; - optional int64 virtual_cores = 2; + optional int32 virtual_cores = 2; } message ResourceUtilizationProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java index 8e00554..c9fa915 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java @@ -171,7 +171,7 @@ public class TestApplicationClientProtocolOnHA extends ProtocolHATestBase { Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(10); + capability.setMemorySize(10); capability.setVirtualCores(1); appContext.setResource(capability); ApplicationId appId = client.submitApplication(appContext); http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 937941a..6686696 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -67,24 +67,25 @@ public class ResourcePBImpl extends Resource { } @Override - public void setMemory(long memory) { - maybeInitBuilder(); - builder.setMemory(memory); + @SuppressWarnings("deprecation") + public void setMemory(int memory) { + setMemorySize(memory); } @Override - public int getVirtualCores() { - return (int) getVirtualCoresSize(); + public void setMemorySize(long memory) { + maybeInitBuilder(); + builder.setMemory(memory); } @Override - public long getVirtualCoresSize() { + public int getVirtualCores() { ResourceProtoOrBuilder p = viaProto ? proto : builder; return p.getVirtualCores(); } @Override - public void setVirtualCores(long vCores) { + public void setVirtualCores(int vCores) { maybeInitBuilder(); builder.setVirtualCores(vCores); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java index b98cf0c..fe78de2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java @@ -56,7 +56,7 @@ public class DefaultResourceCalculator extends ResourceCalculator { } @Override - public Resource divideAndCeil(Resource numerator, long denominator) { + public Resource divideAndCeil(Resource numerator, int denominator) { return Resources.createResource( divideAndCeil(numerator.getMemorySize(), denominator)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 8f97fab..2695811 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -142,7 +142,7 @@ public class DominantResourceCalculator extends ResourceCalculator { } @Override - public Resource divideAndCeil(Resource numerator, long denominator) { + public Resource divideAndCeil(Resource numerator, int denominator) { return Resources.createResource( divideAndCeil(numerator.getMemorySize(), denominator), divideAndCeil(numerator.getVirtualCores(), denominator) @@ -157,7 +157,7 @@ public class DominantResourceCalculator extends ResourceCalculator { Math.max(r.getMemorySize(), minimumResource.getMemorySize()), stepFactor.getMemorySize()), maximumResource.getMemorySize()); - long normalizedCores = Math.min( + int normalizedCores = Math.min( roundUp( Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()), stepFactor.getVirtualCores()), @@ -189,7 +189,7 @@ public class DominantResourceCalculator extends ResourceCalculator { roundUp( (int)Math.ceil(r.getMemorySize() * by), stepFactor.getMemorySize()), roundUp( - (int)Math.ceil(r.getVirtualCores() * by), + (int)Math.ceil(r.getVirtualCores() * by), stepFactor.getVirtualCores()) ); } @@ -203,7 +203,7 @@ public class DominantResourceCalculator extends ResourceCalculator { stepFactor.getMemorySize() ), roundDown( - (int)(r.getVirtualCores() * by), + (int)(r.getVirtualCores() * by), stepFactor.getVirtualCores() ) ); http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java index 3fe8ade..a7df9c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java @@ -30,6 +30,13 @@ public abstract class ResourceCalculator { public abstract int compare(Resource clusterResource, Resource lhs, Resource rhs); + + public static int divideAndCeil(int a, int b) { + if (b == 0) { + return 0; + } + return (a + (b - 1)) / b; + } public static long divideAndCeil(long a, long b) { if (b == 0) { @@ -38,6 +45,10 @@ public abstract class ResourceCalculator { return (a + (b - 1)) / b; } + public static int roundUp(int a, int b) { + return divideAndCeil(a, b) * b; + } + public static long roundUp(long a, long b) { return divideAndCeil(a, b) * b; } @@ -46,6 +57,10 @@ public abstract class ResourceCalculator { return (a / b) * b; } + public static int roundDown(int a, int b) { + return (a / b) * b; + } + /** * Compute the number of containers which can be allocated given * <code>available</code> and <code>required</code> resources. @@ -169,7 +184,7 @@ public abstract class ResourceCalculator { * @param denominator denominator * @return resultant resource */ - public abstract Resource divideAndCeil(Resource numerator, long denominator); + public abstract Resource divideAndCeil(Resource numerator, int denominator); /** * Check if a smaller resource can be contained by bigger resource. http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index e944e11..760b0ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -42,22 +42,23 @@ public class Resources { } @Override - public void setMemory(long memory) { + public void setMemorySize(long memory) { throw new RuntimeException("NONE cannot be modified!"); } @Override - public int getVirtualCores() { - return 0; + @SuppressWarnings("deprecation") + public void setMemory(int memory) { + throw new RuntimeException("NONE cannot be modified!"); } @Override - public long getVirtualCoresSize() { + public int getVirtualCores() { return 0; } @Override - public void setVirtualCores(long cores) { + public void setVirtualCores(int cores) { throw new RuntimeException("NONE cannot be modified!"); } @@ -86,22 +87,23 @@ public class Resources { } @Override - public void setMemory(long memory) { + @SuppressWarnings("deprecation") + public void setMemory(int memory) { throw new RuntimeException("UNBOUNDED cannot be modified!"); } @Override - public int getVirtualCores() { - return Integer.MAX_VALUE; + public void setMemorySize(long memory) { + throw new RuntimeException("UNBOUNDED cannot be modified!"); } @Override - public long getVirtualCoresSize() { - return Long.MAX_VALUE; + public int getVirtualCores() { + return Integer.MAX_VALUE; } @Override - public void setVirtualCores(long cores) { + public void setVirtualCores(int cores) { throw new RuntimeException("UNBOUNDED cannot be modified!"); } @@ -109,20 +111,31 @@ public class Resources { public int compareTo(Resource o) { long diff = Long.MAX_VALUE - o.getMemorySize(); if (diff == 0) { - diff = Long.MAX_VALUE - o.getVirtualCoresSize(); + diff = Integer.MAX_VALUE - o.getVirtualCores(); } return Long.signum(diff); } }; + public static Resource createResource(int memory) { + return createResource(memory, (memory > 0) ? 1 : 0); + } + + public static Resource createResource(int memory, int cores) { + Resource resource = Records.newRecord(Resource.class); + resource.setMemorySize(memory); + resource.setVirtualCores(cores); + return resource; + } + public static Resource createResource(long memory) { return createResource(memory, (memory > 0) ? 1 : 0); } - public static Resource createResource(long memory, long cores) { + public static Resource createResource(long memory, int cores) { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(cores); return resource; } @@ -140,7 +153,7 @@ public class Resources { } public static Resource addTo(Resource lhs, Resource rhs) { - lhs.setMemory(lhs.getMemorySize() + rhs.getMemorySize()); + lhs.setMemorySize(lhs.getMemorySize() + rhs.getMemorySize()); lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores()); return lhs; } @@ -150,7 +163,7 @@ public class Resources { } public static Resource subtractFrom(Resource lhs, Resource rhs) { - lhs.setMemory(lhs.getMemorySize() - rhs.getMemorySize()); + lhs.setMemorySize(lhs.getMemorySize() - rhs.getMemorySize()); lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores()); return lhs; } @@ -164,7 +177,7 @@ public class Resources { } public static Resource multiplyTo(Resource lhs, double by) { - lhs.setMemory((int)(lhs.getMemorySize() * by)); + lhs.setMemorySize((long)(lhs.getMemorySize() * by)); lhs.setVirtualCores((int)(lhs.getVirtualCores() * by)); return lhs; } @@ -179,7 +192,7 @@ public class Resources { */ public static Resource multiplyAndAddTo( Resource lhs, Resource rhs, double by) { - lhs.setMemory(lhs.getMemorySize() + (int)(rhs.getMemorySize() * by)); + lhs.setMemorySize(lhs.getMemorySize() + (long)(rhs.getMemorySize() * by)); lhs.setVirtualCores(lhs.getVirtualCores() + (int)(rhs.getVirtualCores() * by)); return lhs; @@ -197,7 +210,7 @@ public class Resources { public static Resource multiplyAndRoundDown(Resource lhs, double by) { Resource out = clone(lhs); - out.setMemory((int)(lhs.getMemorySize() * by)); + out.setMemorySize((long)(lhs.getMemorySize() * by)); out.setVirtualCores((int)(lhs.getVirtualCores() * by)); return out; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index 7396adc..057214b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -24,18 +24,18 @@ import static org.junit.Assert.assertTrue; public class TestResources { - public Resource createResource(long memory, long vCores) { + public Resource createResource(long memory, int vCores) { return Resource.newInstance(memory, vCores); } @Test(timeout=1000) public void testCompareToWithUnboundedResource() { assertTrue(Resources.unbounded().compareTo( - createResource(Long.MAX_VALUE, Long.MAX_VALUE)) == 0); + createResource(Long.MAX_VALUE, Integer.MAX_VALUE)) == 0); assertTrue(Resources.unbounded().compareTo( createResource(Long.MAX_VALUE, 0)) > 0); assertTrue(Resources.unbounded().compareTo( - createResource(0, Long.MAX_VALUE)) > 0); + createResource(0, Integer.MAX_VALUE)) > 0); } @Test(timeout=1000) http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 134f22d..a049fb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; -import org.apache.hadoop.yarn.util.ConverterUtils; import com.google.common.annotations.VisibleForTesting; @@ -442,9 +441,9 @@ public class BuilderUtils { return report; } - public static Resource newResource(long memory, long vCores) { + public static Resource newResource(long memory, int vCores) { Resource resource = recordFactory.newRecordInstance(Resource.class); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(vCores); return resource; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java index 475ce58..be350e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java @@ -201,7 +201,7 @@ public class TestYarnServerApiClasses { original.setHttpPort(8080); original.setNodeId(getNodeId()); Resource resource = recordFactory.newRecordInstance(Resource.class); - resource.setMemory(10000); + resource.setMemorySize(10000); resource.setVirtualCores(2); original.setResource(resource); RegisterNodeManagerRequestPBImpl copy = new RegisterNodeManagerRequestPBImpl( http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java index ae0cbc3..f716d44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java @@ -39,7 +39,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; @@ -465,7 +464,7 @@ public abstract class BaseAMRMProxyTest { pri.setPriority(priority); req.setPriority(pri); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(memory); + capability.setMemorySize(memory); capability.setVirtualCores(vCores); req.setCapability(capability); if (labelExpression != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java index c0210d5..a08ee82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java @@ -33,16 +33,16 @@ public class TestNodeManagerMetrics { DefaultMetricsSystem.initialize("NodeManager"); NodeManagerMetrics metrics = NodeManagerMetrics.create(); Resource total = Records.newRecord(Resource.class); - total.setMemory(8*GiB); + total.setMemorySize(8*GiB); total.setVirtualCores(16); Resource resource = Records.newRecord(Resource.class); - resource.setMemory(512); //512MiB + resource.setMemorySize(512); //512MiB resource.setVirtualCores(2); Resource largerResource = Records.newRecord(Resource.class); - largerResource.setMemory(1024); + largerResource.setMemorySize(1024); largerResource.setVirtualCores(2); Resource smallerResource = Records.newRecord(Resource.class); - smallerResource.setMemory(256); + smallerResource.setMemorySize(256); smallerResource.setVirtualCores(1); metrics.addResource(total); http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java index feb071f..413bbdf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java @@ -57,7 +57,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> { // Max allocation private long maxNodeMemory = -1; - private long maxNodeVCores = -1; + private int maxNodeVCores = -1; private Resource configuredMaxAllocation; private boolean forceConfiguredMaxAllocation = true; private long configuredMaxAllocationWaitTime; http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 69c3e8b..4e364f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -61,7 +61,7 @@ public class QueueMetrics implements MetricsSource { @Metric("# of apps failed") MutableCounterInt appsFailed; @Metric("Allocated memory in MB") MutableGaugeLong allocatedMB; - @Metric("Allocated CPU in virtual cores") MutableGaugeLong allocatedVCores; + @Metric("Allocated CPU in virtual cores") MutableGaugeInt allocatedVCores; @Metric("# of allocated containers") MutableGaugeInt allocatedContainers; @Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated; @Metric("Aggregate # of allocated node-local containers") @@ -72,12 +72,12 @@ public class QueueMetrics implements MetricsSource { MutableCounterLong aggregateOffSwitchContainersAllocated; @Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased; @Metric("Available memory in MB") MutableGaugeLong availableMB; - @Metric("Available CPU in virtual cores") MutableGaugeLong availableVCores; + @Metric("Available CPU in virtual cores") MutableGaugeInt availableVCores; @Metric("Pending memory allocation in MB") MutableGaugeLong pendingMB; - @Metric("Pending CPU allocation in virtual cores") MutableGaugeLong pendingVCores; + @Metric("Pending CPU allocation in virtual cores") MutableGaugeInt pendingVCores; @Metric("# of pending containers") MutableGaugeInt pendingContainers; @Metric("# of reserved memory in MB") MutableGaugeLong reservedMB; - @Metric("Reserved CPU in virtual cores") MutableGaugeLong reservedVCores; + @Metric("Reserved CPU in virtual cores") MutableGaugeInt reservedVCores; @Metric("# of reserved containers") MutableGaugeInt reservedContainers; @Metric("# of active users") MutableGaugeInt activeUsers; @Metric("# of active applications") MutableGaugeInt activeApplications; @@ -364,7 +364,7 @@ public class QueueMetrics implements MetricsSource { private void _incrPendingResources(int containers, Resource res) { pendingContainers.incr(containers); pendingMB.incr(res.getMemorySize() * containers); - pendingVCores.incr(res.getVirtualCoresSize() * containers); + pendingVCores.incr(res.getVirtualCores() * containers); } public void decrPendingResources(String user, int containers, Resource res) { @@ -381,7 +381,7 @@ public class QueueMetrics implements MetricsSource { private void _decrPendingResources(int containers, Resource res) { pendingContainers.decr(containers); pendingMB.decr(res.getMemorySize() * containers); - pendingVCores.decr(res.getVirtualCoresSize() * containers); + pendingVCores.decr(res.getVirtualCores() * containers); } public void incrNodeTypeAggregations(String user, NodeType type) { @@ -409,7 +409,7 @@ public class QueueMetrics implements MetricsSource { aggregateContainersAllocated.incr(containers); allocatedMB.incr(res.getMemorySize() * containers); - allocatedVCores.incr(res.getVirtualCoresSize() * containers); + allocatedVCores.incr(res.getVirtualCores() * containers); if (decrPending) { _decrPendingResources(containers, res); } @@ -448,7 +448,7 @@ public class QueueMetrics implements MetricsSource { allocatedContainers.decr(containers); aggregateContainersReleased.incr(containers); allocatedMB.decr(res.getMemorySize() * containers); - allocatedVCores.decr(res.getVirtualCoresSize() * containers); + allocatedVCores.decr(res.getVirtualCores() * containers); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.releaseResources(user, containers, res); @@ -561,14 +561,15 @@ public class QueueMetrics implements MetricsSource { } public Resource getAllocatedResources() { - return BuilderUtils.newResource(allocatedMB.value(), allocatedVCores.value()); + return BuilderUtils.newResource(allocatedMB.value(), + (int) allocatedVCores.value()); } public long getAllocatedMB() { return allocatedMB.value(); } - public long getAllocatedVirtualCores() { + public int getAllocatedVirtualCores() { return allocatedVCores.value(); } @@ -580,7 +581,7 @@ public class QueueMetrics implements MetricsSource { return availableMB.value(); } - public long getAvailableVirtualCores() { + public int getAvailableVirtualCores() { return availableVCores.value(); } @@ -588,7 +589,7 @@ public class QueueMetrics implements MetricsSource { return pendingMB.value(); } - public long getPendingVirtualCores() { + public int getPendingVirtualCores() { return pendingVCores.value(); } @@ -600,7 +601,7 @@ public class QueueMetrics implements MetricsSource { return reservedMB.value(); } - public long getReservedVirtualCores() { + public int getReservedVirtualCores() { return reservedVCores.value(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index aae0770..b4a2639 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -465,7 +465,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { public synchronized Resource getHeadroom() { // Corner case to deal with applications being slightly over-limit if (resourceLimit.getMemorySize() < 0) { - resourceLimit.setMemory(0); + resourceLimit.setMemorySize(0); } return resourceLimit; http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java index 49892f7..5605f18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java @@ -66,7 +66,7 @@ public class CapacityHeadroomProvider { } // Corner case to deal with applications being slightly over-limit if (headroom.getMemorySize() < 0) { - headroom.setMemory(0); + headroom.setMemorySize(0); } return headroom; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 634f667..1eead9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -194,7 +194,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt node.getUnallocatedResource()); } if (availableResources.getMemorySize() < 0) { - availableResources.setMemory(0); + availableResources.setMemorySize(0); } if (availableResources.getVirtualCores() < 0) { availableResources.setVirtualCores(0); @@ -460,7 +460,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt } public void clearPreemptedResources() { - preemptedResources.setMemory(0); + preemptedResources.setMemorySize(0); preemptedResources.setVirtualCores(0); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 6723fd9..7c9b11e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -496,15 +496,15 @@ public class FSLeafQueue extends FSQueue { // maxAMResource Resource maxResource = Resources.clone(getFairShare()); if (maxResource.getMemorySize() == 0) { - maxResource.setMemory( + maxResource.setMemorySize( Math.min(scheduler.getRootQueueMetrics().getAvailableMB(), getMaxShare().getMemorySize())); } - if (maxResource.getVirtualCoresSize() == 0) { + if (maxResource.getVirtualCores() == 0) { maxResource.setVirtualCores(Math.min( scheduler.getRootQueueMetrics().getAvailableVirtualCores(), - getMaxShare().getVirtualCoresSize())); + getMaxShare().getVirtualCores())); } Resource maxAMResource = Resources.multiply(maxResource, maxAMShare); http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java index 5f500cd..440c73c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java @@ -266,10 +266,10 @@ public class ComputeFairShares { private static void setResourceValue(long val, Resource resource, ResourceType type) { switch (type) { case MEMORY: - resource.setMemory(val); + resource.setMemorySize(val); break; case CPU: - resource.setVirtualCores(val); + resource.setVirtualCores((int)val); break; default: throw new IllegalArgumentException("Invalid resource"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index 76d43e0..d198bb9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.api.records.Resource; @XmlAccessorType(XmlAccessType.FIELD) public class ResourceInfo { long memory; - long vCores; + int vCores; public ResourceInfo() { } @@ -42,7 +42,7 @@ public class ResourceInfo { return memory; } - public long getvCores() { + public int getvCores() { return vCores; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index f2b4ea7..dc17f5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.util.Records; import org.apache.log4j.Logger; -import org.junit.Assert; public class MockAM { @@ -207,7 +206,7 @@ public class MockAM { pri.setPriority(priority); req.setPriority(pri); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(memory); + capability.setMemorySize(memory); req.setCapability(capability); if (labelExpression != null) { req.setNodeLabelExpression(labelExpression); http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 630a8db..22aa0ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -82,19 +82,19 @@ public class MockNodes { public static Resource newResource(int mem) { Resource rs = recordFactory.newRecordInstance(Resource.class); - rs.setMemory(mem); + rs.setMemorySize(mem); return rs; } public static Resource newUsedResource(Resource total) { Resource rs = recordFactory.newRecordInstance(Resource.class); - rs.setMemory((int)(Math.random() * total.getMemorySize())); + rs.setMemorySize((int)(Math.random() * total.getMemorySize())); return rs; } public static Resource newAvailResource(Resource total, Resource used) { Resource rs = recordFactory.newRecordInstance(Resource.class); - rs.setMemory(total.getMemorySize() - used.getMemorySize()); + rs.setMemorySize(total.getMemorySize() - used.getMemorySize()); return rs; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index f1f3d51..84fe7c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -455,7 +455,7 @@ public class MockRM extends ResourceManager { Map<ApplicationAccessType, String> acls, String queue, String amLabel) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, name, user, acls, false, queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, @@ -508,7 +508,7 @@ public class MockRM extends ResourceManager { int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); return submitApp(resource, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, false, null, 0, null, true, Priority.newInstance(0)); @@ -517,7 +517,7 @@ public class MockRM extends ResourceManager { public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, @@ -532,7 +532,7 @@ public class MockRM extends ResourceManager { boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, ApplicationId applicationId) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, @@ -542,7 +542,7 @@ public class MockRM extends ResourceManager { public RMApp submitApp(int masterMemory, LogAggregationContext logAggregationContext) throws Exception { Resource resource = Records.newRecord(Resource.class); - resource.setMemory(masterMemory); + resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 498112e..098ba54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -826,21 +826,21 @@ public class TestResourceTrackerService extends NodeLabelTestBase { resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction()); - capability.setMemory(2048); + capability.setMemorySize(2048); capability.setVirtualCores(1); req.setResource(capability); RegisterNodeManagerResponse response2 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction()); - capability.setMemory(1024); + capability.setMemorySize(1024); capability.setVirtualCores(4); req.setResource(capability); RegisterNodeManagerResponse response3 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction()); - capability.setMemory(2048); + capability.setMemorySize(2048); capability.setVirtualCores(4); req.setResource(capability); RegisterNodeManagerResponse response4 = http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 3a60e02..a9f1f63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -1345,7 +1345,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase // submit app with keepContainersAcrossApplicationAttempts true Resource resource = Records.newRecord(Resource.class); - resource.setMemory(200); + resource.setMemorySize(200); RMApp app0 = rm1.submitApp(resource, "", UserGroupInformation .getCurrentUser().getShortUserName(), null, false, null, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true, http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java index 4cb235f..d383bac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationSystemUtil.java @@ -127,7 +127,7 @@ public class TestReservationSystemUtil { public Resource createResource(int memory, int vCores) { Resource resource = new ResourcePBImpl(); - resource.setMemory(memory); + resource.setMemorySize(memory); resource.setVirtualCores(vCores); return resource; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index 62b1082..cb1104b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -402,9 +402,9 @@ public class TestQueueMetrics { } public static void checkResources(MetricsSource source, long allocatedMB, - long allocatedCores, int allocCtnrs, long aggreAllocCtnrs, - long aggreReleasedCtnrs, long availableMB, long availableCores, long pendingMB, - long pendingCores, int pendingCtnrs, long reservedMB, long reservedCores, + int allocatedCores, int allocCtnrs, long aggreAllocCtnrs, + long aggreReleasedCtnrs, long availableMB, int availableCores, long pendingMB, + int pendingCores, int pendingCtnrs, long reservedMB, int reservedCores, int reservedCtnrs) { MetricsRecordBuilder rb = getMetrics(source); assertGauge("AllocatedMB", allocatedMB, rb); http://git-wip-us.apache.org/repos/asf/hadoop/blob/819224dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 74fe534..4aee3c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -1143,7 +1143,7 @@ public class TestDelegationTokenRenewer { // submit app1 with a token, set cancelTokenWhenComplete to false; Resource resource = Records.newRecord(Resource.class); - resource.setMemory(200); + resource.setMemorySize(200); RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, false, null); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); @@ -1205,7 +1205,7 @@ public class TestDelegationTokenRenewer { Assert.assertFalse(Renewer.cancelled); Resource resource = Records.newRecord(Resource.class); - resource.setMemory(200); + resource.setMemorySize(200); RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2, credentials, null, true, false, false, null, 0, null, true, null); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
