YARN-4844. Add getMemorySize/getVirtualCoresSize to o.a.h.y.api.records.Resource. (wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19e57887 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19e57887 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19e57887 Branch: refs/heads/branch-2.8 Commit: 19e578870d3245354dbd812186d39a86717a407f Parents: 333490b Author: Wangda Tan <wan...@apache.org> Authored: Tue Jun 7 12:41:50 2016 -0700 Committer: Wangda Tan <wan...@apache.org> Committed: Tue Jun 7 12:41:50 2016 -0700 ---------------------------------------------------------------------- .../jobhistory/JobHistoryEventHandler.java | 4 +- .../v2/app/job/impl/TaskAttemptImpl.java | 1 + .../v2/app/rm/RMContainerAllocator.java | 16 +- .../v2/app/rm/ResourceCalculatorUtils.java | 17 +- .../hadoop/mapreduce/v2/app/TestRecovery.java | 2 +- .../v2/app/job/impl/TestTaskAttempt.java | 19 +- .../v2/app/rm/TestRMContainerAllocator.java | 2 +- .../v2/app/rm/TestResourceCalculatorUtils.java | 4 +- .../apache/hadoop/mapreduce/TypeConverter.java | 6 +- .../org/apache/hadoop/mapreduce/JobStatus.java | 18 +- .../jobhistory/NormalizedResourceEvent.java | 6 +- .../org/apache/hadoop/mapreduce/tools/CLI.java | 7 +- .../apache/hadoop/mapred/JobClientUnitTest.java | 6 +- .../sls/scheduler/FairSchedulerMetrics.java | 60 +-- .../sls/scheduler/ResourceSchedulerWrapper.java | 36 +- .../hadoop/yarn/api/records/Resource.java | 34 +- .../src/main/proto/yarn_protos.proto | 4 +- .../distributedshell/ApplicationMaster.java | 6 +- .../applications/distributedshell/Client.java | 4 +- .../yarn/client/api/impl/AMRMClientImpl.java | 16 +- .../apache/hadoop/yarn/client/cli/NodeCLI.java | 4 +- .../apache/hadoop/yarn/client/cli/TopCLI.java | 4 +- .../yarn/client/api/impl/TestYarnClient.java | 2 +- .../api/records/impl/pb/ResourcePBImpl.java | 32 +- .../resource/DefaultResourceCalculator.java | 36 +- .../resource/DominantResourceCalculator.java | 46 +- .../yarn/util/resource/ResourceCalculator.java | 10 +- .../hadoop/yarn/util/resource/Resources.java | 64 ++- .../yarn/util/resource/TestResources.java | 8 +- .../hadoop/yarn/server/utils/BuilderUtils.java | 2 +- .../hadoop/yarn/server/webapp/dao/AppInfo.java | 10 +- .../yarn/server/webapp/dao/ContainerInfo.java | 10 +- .../hadoop/yarn/TestYarnServerApiClasses.java | 6 +- .../server/nodemanager/ContainerExecutor.java | 2 +- .../container/ContainerImpl.java | 4 +- .../monitor/ContainersMonitorImpl.java | 4 +- .../nodemanager/metrics/NodeManagerMetrics.java | 12 +- .../nodemanager/webapp/dao/ContainerInfo.java | 2 +- .../nodemanager/TestNodeStatusUpdater.java | 4 +- .../monitor/TestContainersMonitor.java | 2 +- .../yarn/server/resourcemanager/RMNMInfo.java | 4 +- .../server/resourcemanager/RMServerUtils.java | 14 +- .../resourcemanager/ResourceTrackerService.java | 2 +- .../metrics/SystemMetricsPublisher.java | 2 +- .../ProportionalCapacityPreemptionPolicy.java | 12 +- .../AbstractSchedulerPlanFollower.java | 2 +- .../reservation/CapacityOverTimePolicy.java | 6 +- .../reservation/ReservationSystemUtil.java | 2 +- .../planning/StageEarliestStartByDemand.java | 2 +- .../rmcontainer/RMContainerImpl.java | 2 +- .../scheduler/AbstractYarnScheduler.java | 14 +- .../resourcemanager/scheduler/QueueMetrics.java | 61 +-- .../scheduler/SchedulerApplicationAttempt.java | 6 +- .../scheduler/SchedulerUtils.java | 8 +- .../scheduler/capacity/CSQueueMetrics.java | 24 +- .../capacity/CapacityHeadroomProvider.java | 2 +- .../CapacitySchedulerConfiguration.java | 6 +- .../scheduler/capacity/LeafQueue.java | 2 +- .../allocator/RegularContainerAllocator.java | 2 +- .../scheduler/fair/FSParentQueue.java | 2 +- .../resourcemanager/scheduler/fair/FSQueue.java | 12 +- .../scheduler/fair/FSQueueMetrics.java | 43 +- .../scheduler/fair/FairScheduler.java | 6 +- .../fair/policies/ComputeFairShares.java | 18 +- .../DominantResourceFairnessPolicy.java | 10 +- .../fair/policies/FairSharePolicy.java | 20 +- .../scheduler/fair/policies/FifoPolicy.java | 8 +- .../scheduler/fifo/FifoScheduler.java | 21 +- .../scheduler/policy/FairOrderingPolicy.java | 5 +- .../webapp/DefaultSchedulerPage.java | 4 +- .../webapp/FairSchedulerAppsBlock.java | 2 +- .../resourcemanager/webapp/RMWebServices.java | 8 +- .../resourcemanager/webapp/dao/AppInfo.java | 20 +- .../webapp/dao/FairSchedulerInfo.java | 4 +- .../webapp/dao/FairSchedulerQueueInfo.java | 14 +- .../webapp/dao/FifoSchedulerInfo.java | 18 +- .../resourcemanager/webapp/dao/NodeInfo.java | 4 +- .../webapp/dao/ResourceInfo.java | 10 +- .../server/resourcemanager/Application.java | 2 +- .../yarn/server/resourcemanager/MockNM.java | 12 +- .../yarn/server/resourcemanager/MockNodes.java | 4 +- .../server/resourcemanager/NodeManager.java | 8 +- .../ReservationACLsTestBase.java | 2 +- .../resourcemanager/TestApplicationACLs.java | 6 +- .../resourcemanager/TestApplicationCleanup.java | 2 +- .../TestContainerResourceUsage.java | 2 +- .../yarn/server/resourcemanager/TestRM.java | 8 +- .../yarn/server/resourcemanager/TestRMHA.java | 4 +- .../resourcemanager/TestRMNodeTransitions.java | 22 +- .../TestReservationSystemWithRMHA.java | 2 +- .../TestResourceTrackerService.java | 2 +- .../TestWorkPreservingRMRestart.java | 29 +- .../metrics/TestSystemMetricsPublisher.java | 10 +- ...estProportionalCapacityPreemptionPolicy.java | 24 +- ...pacityPreemptionPolicyForNodePartitions.java | 2 +- .../nodelabels/TestRMNodeLabelsManager.java | 2 +- .../reservation/ReservationSystemTestUtil.java | 2 +- .../TestRLESparseResourceAllocation.java | 18 +- .../planning/TestSimpleCapacityReplanner.java | 4 +- .../scheduler/TestAbstractYarnScheduler.java | 16 +- .../scheduler/TestQueueMetrics.java | 8 +- .../scheduler/TestResourceUsage.java | 2 +- .../scheduler/TestSchedulerUtils.java | 20 +- .../capacity/TestApplicationLimits.java | 6 +- .../capacity/TestApplicationPriority.java | 52 +- .../capacity/TestCapacityScheduler.java | 124 ++--- .../TestCapacitySchedulerNodeLabelUpdate.java | 52 +- .../scheduler/capacity/TestChildQueueOrder.java | 4 +- .../capacity/TestContainerAllocation.java | 10 +- .../capacity/TestContainerResizing.java | 152 +++--- .../capacity/TestIncreaseAllocationExpirer.java | 36 +- .../scheduler/capacity/TestLeafQueue.java | 493 ++++++++++--------- .../TestNodeLabelContainerAllocation.java | 4 +- .../scheduler/capacity/TestParentQueue.java | 4 +- .../scheduler/capacity/TestQueueCapacities.java | 2 +- .../scheduler/capacity/TestReservations.java | 410 +++++++-------- ...TestWorkPreservingRMRestartForNodeLabel.java | 4 +- .../scheduler/fair/TestComputeFairShares.java | 2 +- .../fair/TestContinuousScheduling.java | 4 +- .../scheduler/fair/TestFSAppAttempt.java | 38 +- .../scheduler/fair/TestFSLeafQueue.java | 15 +- .../scheduler/fair/TestFairScheduler.java | 341 ++++++------- .../fair/TestFairSchedulerFairShare.java | 32 +- .../fair/TestFairSchedulerPreemption.java | 4 +- .../scheduler/fifo/TestFifoScheduler.java | 63 +-- .../webapp/TestRMWebServicesApps.java | 2 +- .../webapp/TestRMWebServicesNodes.java | 4 +- 127 files changed, 1533 insertions(+), 1472 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index 0af8f14..d9025b8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -689,9 +689,9 @@ public class JobHistoryEventHandler extends AbstractService NormalizedResourceEvent normalizedResourceEvent = (NormalizedResourceEvent) event; if (normalizedResourceEvent.getTaskType() == TaskType.MAP) { - summary.setResourcesPerMap(normalizedResourceEvent.getMemory()); + summary.setResourcesPerMap((int) normalizedResourceEvent.getMemory()); } else if (normalizedResourceEvent.getTaskType() == TaskType.REDUCE) { - summary.setResourcesPerReduce(normalizedResourceEvent.getMemory()); + summary.setResourcesPerReduce((int) normalizedResourceEvent.getMemory()); } break; case JOB_INITED: http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 529b772..f87b3974 100755 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -1436,6 +1436,7 @@ public abstract class TaskAttemptImpl implements TaskAttemptImpl taskAttempt) { TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime()); + int mbRequired = taskAttempt.getMemoryRequired(taskAttempt.conf, taskType); int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index 1d63fb6..9b244fd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -357,10 +357,10 @@ public class RMContainerAllocator extends RMContainerRequestor eventHandler.handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent( org.apache.hadoop.mapreduce.TaskType.MAP, mapResourceRequest - .getMemory()))); + .getMemorySize()))); LOG.info("mapResourceRequest:" + mapResourceRequest); - if (mapResourceRequest.getMemory() > supportedMaxContainerCapability - .getMemory() + if (mapResourceRequest.getMemorySize() > supportedMaxContainerCapability + .getMemorySize() || mapResourceRequest.getVirtualCores() > supportedMaxContainerCapability .getVirtualCores()) { String diagMsg = @@ -374,7 +374,7 @@ public class RMContainerAllocator extends RMContainerRequestor } } // set the resources - reqEvent.getCapability().setMemory(mapResourceRequest.getMemory()); + reqEvent.getCapability().setMemory(mapResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( mapResourceRequest.getVirtualCores()); scheduledRequests.addMap(reqEvent);//maps are immediately scheduled @@ -384,10 +384,10 @@ public class RMContainerAllocator extends RMContainerRequestor eventHandler.handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent( org.apache.hadoop.mapreduce.TaskType.REDUCE, - reduceResourceRequest.getMemory()))); + reduceResourceRequest.getMemorySize()))); LOG.info("reduceResourceRequest:" + reduceResourceRequest); - if (reduceResourceRequest.getMemory() > supportedMaxContainerCapability - .getMemory() + if (reduceResourceRequest.getMemorySize() > supportedMaxContainerCapability + .getMemorySize() || reduceResourceRequest.getVirtualCores() > supportedMaxContainerCapability .getVirtualCores()) { String diagMsg = @@ -402,7 +402,7 @@ public class RMContainerAllocator extends RMContainerRequestor } } // set the resources - reqEvent.getCapability().setMemory(reduceResourceRequest.getMemory()); + reqEvent.getCapability().setMemory(reduceResourceRequest.getMemorySize()); reqEvent.getCapability().setVirtualCores( reduceResourceRequest.getVirtualCores()); if (reqEvent.getEarlierAttemptFailed()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java index 39cb22e..4748dfd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java @@ -20,43 +20,42 @@ package org.apache.hadoop.mapreduce.v2.app.rm; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; -import org.apache.hadoop.yarn.util.Records; import java.util.EnumSet; public class ResourceCalculatorUtils { - public static int divideAndCeil(int a, int b) { + public static int divideAndCeil(long a, long b) { if (b == 0) { return 0; } - return (a + (b - 1)) / b; + return (int) ((a + (b - 1)) / b); } public static int computeAvailableContainers(Resource available, Resource required, EnumSet<SchedulerResourceTypes> resourceTypes) { if (resourceTypes.contains(SchedulerResourceTypes.CPU)) { return Math.min( - calculateRatioOrMaxValue(available.getMemory(), required.getMemory()), + calculateRatioOrMaxValue(available.getMemorySize(), required.getMemorySize()), calculateRatioOrMaxValue(available.getVirtualCores(), required .getVirtualCores())); } return calculateRatioOrMaxValue( - available.getMemory(), required.getMemory()); + available.getMemorySize(), required.getMemorySize()); } public static int divideAndCeilContainers(Resource required, Resource factor, EnumSet<SchedulerResourceTypes> resourceTypes) { if (resourceTypes.contains(SchedulerResourceTypes.CPU)) { - return Math.max(divideAndCeil(required.getMemory(), factor.getMemory()), + return Math.max(divideAndCeil(required.getMemorySize(), factor.getMemorySize()), divideAndCeil(required.getVirtualCores(), factor.getVirtualCores())); } - return divideAndCeil(required.getMemory(), factor.getMemory()); + return divideAndCeil(required.getMemorySize(), factor.getMemorySize()); } - private static int calculateRatioOrMaxValue(int numerator, int denominator) { + private static int calculateRatioOrMaxValue(long numerator, long denominator) { if (denominator == 0) { return Integer.MAX_VALUE; } - return numerator / denominator; + return (int) (numerator / denominator); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java index 0e07bcf..7c60992 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java @@ -1789,7 +1789,7 @@ public class TestRecovery { int appAttemptId = 3; MRAppMetrics metrics = mock(MRAppMetrics.class); Resource minContainerRequirements = mock(Resource.class); - when(minContainerRequirements.getMemory()).thenReturn(1000); + when(minContainerRequirements.getMemorySize()).thenReturn(1000L); ClusterInfo clusterInfo = mock(ClusterInfo.class); AppContext appContext = mock(AppContext.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java index df59991..ac7c49c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java @@ -305,6 +305,7 @@ public class TestTaskAttempt{ Assert.assertEquals(rta.getLaunchTime(), 10); Counters counters = job.getAllCounters(); Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize), + counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue()); Assert.assertEquals((int) Math.ceil((float) reduceMemMb / minContainerSize), counters.findCounter(JobCounter.SLOTS_MILLIS_REDUCES).getValue()); @@ -566,7 +567,7 @@ public class TestTaskAttempt{ ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(resource.getMemory()).thenReturn(1024); + when(resource.getMemorySize()).thenReturn(1024L); setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx); TaskAttemptImpl taImpl = @@ -624,7 +625,7 @@ public class TestTaskAttempt{ ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(resource.getMemory()).thenReturn(1024); + when(resource.getMemorySize()).thenReturn(1024L); setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx); TaskAttemptImpl taImpl = @@ -688,7 +689,7 @@ public class TestTaskAttempt{ ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(resource.getMemory()).thenReturn(1024); + when(resource.getMemorySize()).thenReturn(1024L); setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx); TaskAttemptImpl taImpl = @@ -758,7 +759,7 @@ public class TestTaskAttempt{ ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(resource.getMemory()).thenReturn(1024); + when(resource.getMemorySize()).thenReturn(1024L); setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx); TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, @@ -815,7 +816,7 @@ public class TestTaskAttempt{ ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(resource.getMemory()).thenReturn(1024); + when(resource.getMemorySize()).thenReturn(1024L); setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx); TaskAttemptImpl taImpl = @@ -883,7 +884,7 @@ public class TestTaskAttempt{ ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(resource.getMemory()).thenReturn(1024); + when(resource.getMemorySize()).thenReturn(1024L); setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx); TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, @@ -1043,7 +1044,7 @@ public class TestTaskAttempt{ ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(resource.getMemory()).thenReturn(1024); + when(resource.getMemorySize()).thenReturn(1024L); TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, splits, jobConf, taListener, new Token(), @@ -1097,7 +1098,7 @@ public class TestTaskAttempt{ ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(resource.getMemory()).thenReturn(1024); + when(resource.getMemorySize()).thenReturn(1024L); TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, splits, jobConf, taListener, new Token(), @@ -1154,7 +1155,7 @@ public class TestTaskAttempt{ ClusterInfo clusterInfo = mock(ClusterInfo.class); Resource resource = mock(Resource.class); when(appCtx.getClusterInfo()).thenReturn(clusterInfo); - when(resource.getMemory()).thenReturn(1024); + when(resource.getMemorySize()).thenReturn(1024L); TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, splits, jobConf, taListener, new Token(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java index 5c05189..755f8ee 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java @@ -1771,7 +1771,7 @@ public class TestRMContainerAllocator { when(excessC.getId()).thenReturn(containerId); when(excessC.getPriority()).thenReturn(RMContainerAllocator.PRIORITY_REDUCE); Resource mockR = mock(Resource.class); - when(mockR.getMemory()).thenReturn(2048); + when(mockR.getMemorySize()).thenReturn(2048L); when(excessC.getResource()).thenReturn(mockR); NodeId nId = mock(NodeId.class); when(nId.getHost()).thenReturn("local"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java index d87f6db..cab8f54 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java @@ -47,8 +47,8 @@ public class TestResourceCalculatorUtils { Integer.MAX_VALUE, expectedNumberOfContainersForCPU); - Resource zeroCpuResource = Resource.newInstance(nonZeroResource.getMemory(), - 0); + Resource zeroCpuResource = Resource.newInstance( + nonZeroResource.getMemorySize(), 0); verifyDifferentResourceTypes(clusterAvailableResources, zeroCpuResource, expectedNumberOfContainersForMemory, http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java index 4af5b89..5979eda 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java @@ -520,13 +520,13 @@ public class TypeConverter { application.getApplicationResourceUsageReport(); if (resourceUsageReport != null) { jobStatus.setNeededMem( - resourceUsageReport.getNeededResources().getMemory()); + resourceUsageReport.getNeededResources().getMemorySize()); jobStatus.setNumReservedSlots( resourceUsageReport.getNumReservedContainers()); jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers()); jobStatus.setReservedMem( - resourceUsageReport.getReservedResources().getMemory()); - jobStatus.setUsedMem(resourceUsageReport.getUsedResources().getMemory()); + resourceUsageReport.getReservedResources().getMemorySize()); + jobStatus.setUsedMem(resourceUsageReport.getUsedResources().getMemorySize()); } return jobStatus; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java index df7f747..d0c5f10 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java @@ -95,9 +95,9 @@ public class JobStatus implements Writable, Cloneable { private String trackingUrl =""; private int numUsedSlots; private int numReservedSlots; - private int usedMem; - private int reservedMem; - private int neededMem; + private long usedMem; + private long reservedMem; + private long neededMem; private boolean isUber; /** @@ -552,42 +552,42 @@ public class JobStatus implements Writable, Cloneable { /** * @return the used memory */ - public int getUsedMem() { + public long getUsedMem() { return usedMem; } /** * @param m the used memory */ - public void setUsedMem(int m) { + public void setUsedMem(long m) { this.usedMem = m; } /** * @return the reserved memory */ - public int getReservedMem() { + public long getReservedMem() { return reservedMem; } /** * @param r the reserved memory */ - public void setReservedMem(int r) { + public void setReservedMem(long r) { this.reservedMem = r; } /** * @return the needed memory */ - public int getNeededMem() { + public long getNeededMem() { return neededMem; } /** * @param n the needed memory */ - public void setNeededMem(int n) { + public void setNeededMem(long n) { this.neededMem = n; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java index b8f049c..eead9cf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java @@ -28,7 +28,7 @@ import org.apache.hadoop.mapreduce.TaskType; @InterfaceAudience.Private @InterfaceStability.Unstable public class NormalizedResourceEvent implements HistoryEvent { - private int memory; + private long memory; private TaskType taskType; /** @@ -36,7 +36,7 @@ public class NormalizedResourceEvent implements HistoryEvent { * @param taskType the tasktype of the request. * @param memory the normalized memory requirements. */ - public NormalizedResourceEvent(TaskType taskType, int memory) { + public NormalizedResourceEvent(TaskType taskType, long memory) { this.memory = memory; this.taskType = taskType; } @@ -53,7 +53,7 @@ public class NormalizedResourceEvent implements HistoryEvent { * the normalized memory * @return the normalized memory */ - public int getMemory() { + public long getMemory() { return this.memory; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java index 79fea2c..5af3c79 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java @@ -666,9 +666,10 @@ public class CLI extends Configured implements Tool { for (JobStatus job : jobs) { int numUsedSlots = job.getNumUsedSlots(); int numReservedSlots = job.getNumReservedSlots(); - int usedMem = job.getUsedMem(); - int rsvdMem = job.getReservedMem(); - int neededMem = job.getNeededMem(); + + long usedMem = job.getUsedMem(); + long rsvdMem = job.getReservedMem(); + long neededMem = job.getNeededMem(); writer.printf(dataPattern, job.getJobID().toString(), job.getState(), job.getStartTime(), job.getUsername(), job.getQueue(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java index 84b76bf..4895a5b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java @@ -170,9 +170,9 @@ public class JobClientUnitTest { when(mockJobStatus.getPriority()).thenReturn(JobPriority.NORMAL); when(mockJobStatus.getNumUsedSlots()).thenReturn(1); when(mockJobStatus.getNumReservedSlots()).thenReturn(1); - when(mockJobStatus.getUsedMem()).thenReturn(1024); - when(mockJobStatus.getReservedMem()).thenReturn(512); - when(mockJobStatus.getNeededMem()).thenReturn(2048); + when(mockJobStatus.getUsedMem()).thenReturn(1024L); + when(mockJobStatus.getReservedMem()).thenReturn(512L); + when(mockJobStatus.getNeededMem()).thenReturn(2048L); when(mockJobStatus.getSchedulingInfo()).thenReturn("NA"); Job mockJob = mock(Job.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java index d100e1d..3b539fa 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java @@ -67,10 +67,10 @@ public class FairSchedulerMetrics extends SchedulerMetrics { FairScheduler fair = (FairScheduler) scheduler; final FSAppAttempt app = fair.getSchedulerApp(appAttemptId); metrics.register("variable.app." + oldAppId + ".demand.memory", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { - return app.getDemand().getMemory(); + public Long getValue() { + return app.getDemand().getMemorySize(); } } ); @@ -83,10 +83,10 @@ public class FairSchedulerMetrics extends SchedulerMetrics { } ); metrics.register("variable.app." + oldAppId + ".usage.memory", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { - return app.getResourceUsage().getMemory(); + public Long getValue() { + return app.getResourceUsage().getMemorySize(); } } ); @@ -99,26 +99,26 @@ public class FairSchedulerMetrics extends SchedulerMetrics { } ); metrics.register("variable.app." + oldAppId + ".minshare.memory", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { - return app.getMinShare().getMemory(); + public Long getValue() { + return app.getMinShare().getMemorySize(); } } ); metrics.register("variable.app." + oldAppId + ".minshare.vcores", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { - return app.getMinShare().getMemory(); + public Long getValue() { + return app.getMinShare().getMemorySize(); } } ); metrics.register("variable.app." + oldAppId + ".maxshare.memory", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { - return Math.min(app.getMaxShare().getMemory(), totalMemoryMB); + public Long getValue() { + return Math.min(app.getMaxShare().getMemorySize(), totalMemoryMB); } } ); @@ -154,10 +154,10 @@ public class FairSchedulerMetrics extends SchedulerMetrics { FairScheduler fair = (FairScheduler) scheduler; final FSQueue queue = fair.getQueueManager().getQueue(queueName); metrics.register("variable.queue." + queueName + ".demand.memory", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { - return queue.getDemand().getMemory(); + public Long getValue() { + return queue.getDemand().getMemorySize(); } } ); @@ -170,10 +170,10 @@ public class FairSchedulerMetrics extends SchedulerMetrics { } ); metrics.register("variable.queue." + queueName + ".usage.memory", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { - return queue.getResourceUsage().getMemory(); + public Long getValue() { + return queue.getResourceUsage().getMemorySize(); } } ); @@ -186,10 +186,10 @@ public class FairSchedulerMetrics extends SchedulerMetrics { } ); metrics.register("variable.queue." + queueName + ".minshare.memory", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { - return queue.getMinShare().getMemory(); + public Long getValue() { + return queue.getMinShare().getMemorySize(); } } ); @@ -202,9 +202,9 @@ public class FairSchedulerMetrics extends SchedulerMetrics { } ); metrics.register("variable.queue." + queueName + ".maxshare.memory", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { + public Long getValue() { if (! maxReset && SLSRunner.simulateInfoMap.containsKey("Number of nodes") && SLSRunner.simulateInfoMap.containsKey("Node memory (MB)") && @@ -221,7 +221,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics { maxReset = false; } - return Math.min(queue.getMaxShare().getMemory(), totalMemoryMB); + return Math.min(queue.getMaxShare().getMemorySize(), totalMemoryMB); } } ); @@ -234,10 +234,10 @@ public class FairSchedulerMetrics extends SchedulerMetrics { } ); metrics.register("variable.queue." + queueName + ".fairshare.memory", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { - return queue.getFairShare().getMemory(); + public Long getValue() { + return queue.getFairShare().getMemorySize(); } } ); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index 8835deb..1a936ab 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -267,7 +267,7 @@ public class ResourceSchedulerWrapper // should have one container which is AM container RMContainer rmc = app.getLiveContainers().iterator().next(); updateQueueMetrics(queue, - rmc.getContainer().getResource().getMemory(), + rmc.getContainer().getResource().getMemorySize(), rmc.getContainer().getResource().getVirtualCores()); } } @@ -323,7 +323,7 @@ public class ResourceSchedulerWrapper if (status.getExitStatus() == ContainerExitStatus.SUCCESS) { for (RMContainer rmc : app.getLiveContainers()) { if (rmc.getContainerId() == containerId) { - releasedMemory += rmc.getContainer().getResource().getMemory(); + releasedMemory += rmc.getContainer().getResource().getMemorySize(); releasedVCores += rmc.getContainer() .getResource().getVirtualCores(); break; @@ -332,7 +332,7 @@ public class ResourceSchedulerWrapper } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) { if (preemptionContainerMap.containsKey(containerId)) { Resource preResource = preemptionContainerMap.get(containerId); - releasedMemory += preResource.getMemory(); + releasedMemory += preResource.getMemorySize(); releasedVCores += preResource.getVirtualCores(); preemptionContainerMap.remove(containerId); } @@ -423,9 +423,9 @@ public class ResourceSchedulerWrapper "counter.queue." + queueName + ".pending.cores", "counter.queue." + queueName + ".allocated.memory", "counter.queue." + queueName + ".allocated.cores"}; - int values[] = new int[]{pendingResource.getMemory(), + long values[] = new long[]{pendingResource.getMemorySize(), pendingResource.getVirtualCores(), - allocatedResource.getMemory(), allocatedResource.getVirtualCores()}; + allocatedResource.getMemorySize(), allocatedResource.getVirtualCores()}; for (int i = names.length - 1; i >= 0; i --) { if (! counterMap.containsKey(names[i])) { metrics.counter(names[i]); @@ -531,11 +531,11 @@ public class ResourceSchedulerWrapper private void registerClusterResourceMetrics() { metrics.register("variable.cluster.allocated.memory", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { + public Long getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0; + return 0L; } else { return scheduler.getRootQueueMetrics().getAllocatedMB(); } @@ -543,11 +543,11 @@ public class ResourceSchedulerWrapper } ); metrics.register("variable.cluster.allocated.vcores", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { + public Long getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0; + return 0L; } else { return scheduler.getRootQueueMetrics().getAllocatedVirtualCores(); } @@ -555,11 +555,11 @@ public class ResourceSchedulerWrapper } ); metrics.register("variable.cluster.available.memory", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { + public Long getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0; + return 0L; } else { return scheduler.getRootQueueMetrics().getAvailableMB(); } @@ -567,11 +567,11 @@ public class ResourceSchedulerWrapper } ); metrics.register("variable.cluster.available.vcores", - new Gauge<Integer>() { + new Gauge<Long>() { @Override - public Integer getValue() { + public Long getValue() { if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0; + return 0L; } else { return scheduler.getRootQueueMetrics().getAvailableVirtualCores(); } @@ -749,7 +749,7 @@ public class ResourceSchedulerWrapper } private void updateQueueMetrics(String queue, - int releasedMemory, int releasedVCores) { + long releasedMemory, int releasedVCores) { // update queue counters SortedMap<String, Counter> counterMap = metrics.getCounters(); if (releasedMemory != 0) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 88b57f1..e2d8472 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -18,7 +18,9 @@ package org.apache.hadoop.yarn.api.records; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; @@ -53,7 +55,7 @@ public abstract class Resource implements Comparable<Resource> { @Public @Stable - public static Resource newInstance(int memory, int vCores) { + public static Resource newInstance(long memory, long vCores) { Resource resource = Records.newRecord(Resource.class); resource.setMemory(memory); resource.setVirtualCores(vCores); @@ -61,12 +63,23 @@ public abstract class Resource implements Comparable<Resource> { } /** + * This method is DEPRECATED: + * Use {@link Resource#getMemorySize()} instead + * * Get <em>memory</em> of the resource. * @return <em>memory</em> of the resource */ @Public - @Stable + @Deprecated public abstract int getMemory(); + + /** + * Get <em>memory</em> of the resource. + * @return <em>memory</em> of the resource + */ + @Private + @Unstable + public abstract long getMemorySize(); /** * Set <em>memory</em> of the resource. @@ -74,7 +87,7 @@ public abstract class Resource implements Comparable<Resource> { */ @Public @Stable - public abstract void setMemory(int memory); + public abstract void setMemory(long memory); /** @@ -90,6 +103,10 @@ public abstract class Resource implements Comparable<Resource> { @Public @Evolving public abstract int getVirtualCores(); + + @Public + @Unstable + public abstract long getVirtualCoresSize(); /** * Set <em>number of virtual cpu cores</em> of the resource. @@ -103,13 +120,14 @@ public abstract class Resource implements Comparable<Resource> { */ @Public @Evolving - public abstract void setVirtualCores(int vCores); + public abstract void setVirtualCores(long vCores); @Override public int hashCode() { final int prime = 263167; - int result = 3571; - result = 939769357 + getMemory(); // prime * result = 939769357 initially + + int result = (int) (939769357 + + getMemorySize()); // prime * result = 939769357 initially result = prime * result + getVirtualCores(); return result; } @@ -123,7 +141,7 @@ public abstract class Resource implements Comparable<Resource> { if (!(obj instanceof Resource)) return false; Resource other = (Resource) obj; - if (getMemory() != other.getMemory() || + if (getMemorySize() != other.getMemorySize() || getVirtualCores() != other.getVirtualCores()) { return false; } @@ -132,6 +150,6 @@ public abstract class Resource implements Comparable<Resource> { @Override public String toString() { - return "<memory:" + getMemory() + ", vCores:" + getVirtualCores() + ">"; + return "<memory:" + getMemorySize() + ", vCores:" + getVirtualCores() + ">"; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 02fb28b..a3741f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -54,8 +54,8 @@ message ContainerIdProto { } message ResourceProto { - optional int32 memory = 1; - optional int32 virtual_cores = 2; + optional int64 memory = 1; + optional int64 virtual_cores = 2; } message ResourceUtilizationProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 23920d1..0c670df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -221,7 +221,7 @@ public class ApplicationMaster { @VisibleForTesting protected int numTotalContainers = 1; // Memory to request for the container on which the shell command will run - private int containerMemory = 10; + private long containerMemory = 10; // VirtualCores to request for the container on which the shell command will run private int containerVirtualCores = 1; // Priority of the request @@ -593,7 +593,7 @@ public class ApplicationMaster { appMasterTrackingUrl); // Dump out information about cluster capability as seen by the // resource manager - int maxMem = response.getMaximumResourceCapability().getMemory(); + long maxMem = response.getMaximumResourceCapability().getMemorySize(); LOG.info("Max mem capability of resources in this cluster " + maxMem); int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); @@ -823,7 +823,7 @@ public class ApplicationMaster { + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory" - + allocatedContainer.getResource().getMemory() + + allocatedContainer.getResource().getMemorySize() + ", containerResourceVirtualCores" + allocatedContainer.getResource().getVirtualCores()); // + ", containerToken" http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 68d2bde..c632794 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -126,7 +126,7 @@ public class Client { // Queue for App master private String amQueue = ""; // Amt. of memory resource to request for to run the App Master - private int amMemory = 10; + private long amMemory = 10; // Amt. of virtual core resource to request for to run the App Master private int amVCores = 1; @@ -487,7 +487,7 @@ public class Client { // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager - int maxMem = appResponse.getMaximumResourceCapability().getMemory(); + long maxMem = appResponse.getMaximumResourceCapability().getMemorySize(); LOG.info("Max mem capability of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java index 46ddc4d..4366c25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java @@ -122,10 +122,10 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> { class ResourceReverseMemoryThenCpuComparator implements Comparator<Resource> { @Override public int compare(Resource arg0, Resource arg1) { - int mem0 = arg0.getMemory(); - int mem1 = arg1.getMemory(); - int cpu0 = arg0.getVirtualCores(); - int cpu1 = arg1.getVirtualCores(); + long mem0 = arg0.getMemorySize(); + long mem1 = arg1.getMemorySize(); + long cpu0 = arg0.getVirtualCores(); + long cpu1 = arg1.getVirtualCores(); if(mem0 == mem1) { if(cpu0 == cpu1) { return 0; @@ -143,10 +143,10 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> { } static boolean canFit(Resource arg0, Resource arg1) { - int mem0 = arg0.getMemory(); - int mem1 = arg1.getMemory(); - int cpu0 = arg0.getVirtualCores(); - int cpu1 = arg1.getVirtualCores(); + long mem0 = arg0.getMemorySize(); + long mem1 = arg1.getMemorySize(); + long cpu0 = arg0.getVirtualCores(); + long cpu1 = arg1.getVirtualCores(); return (mem0 <= mem1 && cpu0 <= cpu1); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java index c112857..a89551f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java @@ -275,9 +275,9 @@ public class NodeCLI extends YarnCLI { nodeReportStr.println(nodeReport.getNumContainers()); nodeReportStr.print("\tMemory-Used : "); nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB" - : (nodeReport.getUsed().getMemory() + "MB")); + : (nodeReport.getUsed().getMemorySize() + "MB")); nodeReportStr.print("\tMemory-Capacity : "); - nodeReportStr.println(nodeReport.getCapability().getMemory() + "MB"); + nodeReportStr.println(nodeReport.getCapability().getMemorySize() + "MB"); nodeReportStr.print("\tCPU-Used : "); nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores" : (nodeReport.getUsed().getVirtualCores() + " vcores")); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java index 79d451a..157cf51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java @@ -155,7 +155,7 @@ public class TopCLI extends YarnCLI { displayStringsMap.put(Columns.VCORES, String.valueOf(usedVirtualCores)); usedMemory = appReport.getApplicationResourceUsageReport().getUsedResources() - .getMemory() / 1024; + .getMemorySize() / 1024; displayStringsMap.put(Columns.MEM, String.valueOf(usedMemory) + "G"); reservedVirtualCores = appReport.getApplicationResourceUsageReport().getReservedResources() @@ -164,7 +164,7 @@ public class TopCLI extends YarnCLI { String.valueOf(reservedVirtualCores)); reservedMemory = appReport.getApplicationResourceUsageReport().getReservedResources() - .getMemory() / 1024; + .getMemorySize() / 1024; displayStringsMap.put(Columns.RMEM, String.valueOf(reservedMemory) + "G"); attempts = appReport.getCurrentApplicationAttemptId().getAttemptId(); nodes = 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 9b0211f..dcbad49 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -1206,7 +1206,7 @@ public class TestYarnClient { for (attempts = 10; attempts > 0; attempts--) { if (cluster.getResourceManager().getRMContext().getReservationSystem() .getPlan(ReservationSystemTestUtil.reservationQ).getTotalCapacity() - .getMemory() > 6000) { + .getMemorySize() > 6000) { break; } try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index a28c6ed..928dc82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.api.records.impl.pb; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProtoOrBuilder; @@ -53,39 +53,49 @@ public class ResourcePBImpl extends Resource { } viaProto = false; } - - + @Override + @SuppressWarnings("deprecation") public int getMemory() { + return (int) getMemorySize(); + } + + @Override + public long getMemorySize() { ResourceProtoOrBuilder p = viaProto ? proto : builder; - return (p.getMemory()); + return p.getMemory(); } @Override - public void setMemory(int memory) { + public void setMemory(long memory) { maybeInitBuilder(); - builder.setMemory((memory)); + builder.setMemory(memory); } @Override public int getVirtualCores() { + return (int) getVirtualCoresSize(); + } + + @Override + public long getVirtualCoresSize() { ResourceProtoOrBuilder p = viaProto ? proto : builder; - return (p.getVirtualCores()); + return p.getVirtualCores(); } @Override - public void setVirtualCores(int vCores) { + public void setVirtualCores(long vCores) { maybeInitBuilder(); - builder.setVirtualCores((vCores)); + builder.setVirtualCores(vCores); } @Override public int compareTo(Resource other) { - int diff = this.getMemory() - other.getMemory(); + long diff = this.getMemorySize() - other.getMemorySize(); if (diff == 0) { diff = this.getVirtualCores() - other.getVirtualCores(); } - return diff; + return diff == 0 ? 0 : (diff > 0 ? 1 : -1); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java index 2fdf214..b98cf0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java @@ -28,13 +28,13 @@ public class DefaultResourceCalculator extends ResourceCalculator { @Override public int compare(Resource unused, Resource lhs, Resource rhs) { // Only consider memory - return lhs.getMemory() - rhs.getMemory(); + return Long.compare(lhs.getMemorySize(), rhs.getMemorySize()); } @Override - public int computeAvailableContainers(Resource available, Resource required) { + public long computeAvailableContainers(Resource available, Resource required) { // Only consider memory - return available.getMemory() / required.getMemory(); + return available.getMemorySize() / required.getMemorySize(); } @Override @@ -44,7 +44,7 @@ public class DefaultResourceCalculator extends ResourceCalculator { } public boolean isInvalidDivisor(Resource r) { - if (r.getMemory() == 0.0f) { + if (r.getMemorySize() == 0.0f) { return true; } return false; @@ -52,23 +52,23 @@ public class DefaultResourceCalculator extends ResourceCalculator { @Override public float ratio(Resource a, Resource b) { - return (float)a.getMemory() / b.getMemory(); + return (float)a.getMemorySize() / b.getMemorySize(); } @Override - public Resource divideAndCeil(Resource numerator, int denominator) { + public Resource divideAndCeil(Resource numerator, long denominator) { return Resources.createResource( - divideAndCeil(numerator.getMemory(), denominator)); + divideAndCeil(numerator.getMemorySize(), denominator)); } @Override public Resource normalize(Resource r, Resource minimumResource, Resource maximumResource, Resource stepFactor) { - int normalizedMemory = Math.min( + long normalizedMemory = Math.min( roundUp( - Math.max(r.getMemory(), minimumResource.getMemory()), - stepFactor.getMemory()), - maximumResource.getMemory()); + Math.max(r.getMemorySize(), minimumResource.getMemorySize()), + stepFactor.getMemorySize()), + maximumResource.getMemorySize()); return Resources.createResource(normalizedMemory); } @@ -81,22 +81,22 @@ public class DefaultResourceCalculator extends ResourceCalculator { @Override public Resource roundUp(Resource r, Resource stepFactor) { return Resources.createResource( - roundUp(r.getMemory(), stepFactor.getMemory()) + roundUp(r.getMemorySize(), stepFactor.getMemorySize()) ); } @Override public Resource roundDown(Resource r, Resource stepFactor) { return Resources.createResource( - roundDown(r.getMemory(), stepFactor.getMemory())); + roundDown(r.getMemorySize(), stepFactor.getMemorySize())); } @Override public Resource multiplyAndNormalizeUp(Resource r, double by, Resource stepFactor) { return Resources.createResource( - roundUp((int)(r.getMemory() * by + 0.5), stepFactor.getMemory()) - ); + roundUp((long) (r.getMemorySize() * by + 0.5), + stepFactor.getMemorySize())); } @Override @@ -104,8 +104,8 @@ public class DefaultResourceCalculator extends ResourceCalculator { Resource stepFactor) { return Resources.createResource( roundDown( - (int)(r.getMemory() * by), - stepFactor.getMemory() + (long)(r.getMemorySize() * by), + stepFactor.getMemorySize() ) ); } @@ -113,6 +113,6 @@ public class DefaultResourceCalculator extends ResourceCalculator { @Override public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) { - return smaller.getMemory() <= bigger.getMemory(); + return smaller.getMemorySize() <= bigger.getMemorySize(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index b5c9967..8f97fab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -54,15 +54,15 @@ public class DominantResourceCalculator extends ResourceCalculator { } if (isInvalidDivisor(clusterResource)) { - if ((lhs.getMemory() < rhs.getMemory() && lhs.getVirtualCores() > rhs + if ((lhs.getMemorySize() < rhs.getMemorySize() && lhs.getVirtualCores() > rhs .getVirtualCores()) - || (lhs.getMemory() > rhs.getMemory() && lhs.getVirtualCores() < rhs + || (lhs.getMemorySize() > rhs.getMemorySize() && lhs.getVirtualCores() < rhs .getVirtualCores())) { return 0; - } else if (lhs.getMemory() > rhs.getMemory() + } else if (lhs.getMemorySize() > rhs.getMemorySize() || lhs.getVirtualCores() > rhs.getVirtualCores()) { return 1; - } else if (lhs.getMemory() < rhs.getMemory() + } else if (lhs.getMemorySize() < rhs.getMemorySize() || lhs.getVirtualCores() < rhs.getVirtualCores()) { return -1; } @@ -100,20 +100,20 @@ public class DominantResourceCalculator extends ResourceCalculator { // Just use 'dominant' resource return (dominant) ? Math.max( - (float)resource.getMemory() / clusterResource.getMemory(), + (float)resource.getMemorySize() / clusterResource.getMemorySize(), (float)resource.getVirtualCores() / clusterResource.getVirtualCores() ) : Math.min( - (float)resource.getMemory() / clusterResource.getMemory(), + (float)resource.getMemorySize() / clusterResource.getMemorySize(), (float)resource.getVirtualCores() / clusterResource.getVirtualCores() ); } @Override - public int computeAvailableContainers(Resource available, Resource required) { + public long computeAvailableContainers(Resource available, Resource required) { return Math.min( - available.getMemory() / required.getMemory(), + available.getMemorySize() / required.getMemorySize(), available.getVirtualCores() / required.getVirtualCores()); } @@ -127,7 +127,7 @@ public class DominantResourceCalculator extends ResourceCalculator { @Override public boolean isInvalidDivisor(Resource r) { - if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f) { + if (r.getMemorySize() == 0.0f || r.getVirtualCores() == 0.0f) { return true; } return false; @@ -136,15 +136,15 @@ public class DominantResourceCalculator extends ResourceCalculator { @Override public float ratio(Resource a, Resource b) { return Math.max( - (float)a.getMemory()/b.getMemory(), + (float)a.getMemorySize()/b.getMemorySize(), (float)a.getVirtualCores()/b.getVirtualCores() ); } @Override - public Resource divideAndCeil(Resource numerator, int denominator) { + public Resource divideAndCeil(Resource numerator, long denominator) { return Resources.createResource( - divideAndCeil(numerator.getMemory(), denominator), + divideAndCeil(numerator.getMemorySize(), denominator), divideAndCeil(numerator.getVirtualCores(), denominator) ); } @@ -152,12 +152,12 @@ public class DominantResourceCalculator extends ResourceCalculator { @Override public Resource normalize(Resource r, Resource minimumResource, Resource maximumResource, Resource stepFactor) { - int normalizedMemory = Math.min( + long normalizedMemory = Math.min( roundUp( - Math.max(r.getMemory(), minimumResource.getMemory()), - stepFactor.getMemory()), - maximumResource.getMemory()); - int normalizedCores = Math.min( + Math.max(r.getMemorySize(), minimumResource.getMemorySize()), + stepFactor.getMemorySize()), + maximumResource.getMemorySize()); + long normalizedCores = Math.min( roundUp( Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()), stepFactor.getVirtualCores()), @@ -169,7 +169,7 @@ public class DominantResourceCalculator extends ResourceCalculator { @Override public Resource roundUp(Resource r, Resource stepFactor) { return Resources.createResource( - roundUp(r.getMemory(), stepFactor.getMemory()), + roundUp(r.getMemorySize(), stepFactor.getMemorySize()), roundUp(r.getVirtualCores(), stepFactor.getVirtualCores()) ); } @@ -177,7 +177,7 @@ public class DominantResourceCalculator extends ResourceCalculator { @Override public Resource roundDown(Resource r, Resource stepFactor) { return Resources.createResource( - roundDown(r.getMemory(), stepFactor.getMemory()), + roundDown(r.getMemorySize(), stepFactor.getMemorySize()), roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()) ); } @@ -187,7 +187,7 @@ public class DominantResourceCalculator extends ResourceCalculator { Resource stepFactor) { return Resources.createResource( roundUp( - (int)Math.ceil(r.getMemory() * by), stepFactor.getMemory()), + (int)Math.ceil(r.getMemorySize() * by), stepFactor.getMemorySize()), roundUp( (int)Math.ceil(r.getVirtualCores() * by), stepFactor.getVirtualCores()) @@ -199,8 +199,8 @@ public class DominantResourceCalculator extends ResourceCalculator { Resource stepFactor) { return Resources.createResource( roundDown( - (int)(r.getMemory() * by), - stepFactor.getMemory() + (int)(r.getMemorySize() * by), + stepFactor.getMemorySize() ), roundDown( (int)(r.getVirtualCores() * by), @@ -212,7 +212,7 @@ public class DominantResourceCalculator extends ResourceCalculator { @Override public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) { - return smaller.getMemory() <= bigger.getMemory() + return smaller.getMemorySize() <= bigger.getMemorySize() && smaller.getVirtualCores() <= bigger.getVirtualCores(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java index 3a31225..3fe8ade 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java @@ -31,18 +31,18 @@ public abstract class ResourceCalculator { public abstract int compare(Resource clusterResource, Resource lhs, Resource rhs); - public static int divideAndCeil(int a, int b) { + public static long divideAndCeil(long a, long b) { if (b == 0) { return 0; } return (a + (b - 1)) / b; } - public static int roundUp(int a, int b) { + public static long roundUp(long a, long b) { return divideAndCeil(a, b) * b; } - public static int roundDown(int a, int b) { + public static long roundDown(long a, long b) { return (a / b) * b; } @@ -54,7 +54,7 @@ public abstract class ResourceCalculator { * @param required required resources * @return number of containers which can be allocated */ - public abstract int computeAvailableContainers( + public abstract long computeAvailableContainers( Resource available, Resource required); /** @@ -169,7 +169,7 @@ public abstract class ResourceCalculator { * @param denominator denominator * @return resultant resource */ - public abstract Resource divideAndCeil(Resource numerator, int denominator); + public abstract Resource divideAndCeil(Resource numerator, long denominator); /** * Check if a smaller resource can be contained by bigger resource. http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index b05d021..9393298 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.util.resource; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.util.Records; @InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"}) @@ -31,12 +31,18 @@ public class Resources { private static final Resource NONE = new Resource() { @Override + @SuppressWarnings("deprecation") public int getMemory() { return 0; } @Override - public void setMemory(int memory) { + public long getMemorySize() { + return 0; + } + + @Override + public void setMemory(long memory) { throw new RuntimeException("NONE cannot be modified!"); } @@ -46,17 +52,22 @@ public class Resources { } @Override - public void setVirtualCores(int cores) { + public long getVirtualCoresSize() { + return 0; + } + + @Override + public void setVirtualCores(long cores) { throw new RuntimeException("NONE cannot be modified!"); } @Override public int compareTo(Resource o) { - int diff = 0 - o.getMemory(); + long diff = 0 - o.getMemorySize(); if (diff == 0) { diff = 0 - o.getVirtualCores(); } - return diff; + return Long.signum(diff); } }; @@ -64,12 +75,18 @@ public class Resources { private static final Resource UNBOUNDED = new Resource() { @Override + @SuppressWarnings("deprecation") public int getMemory() { return Integer.MAX_VALUE; } @Override - public void setMemory(int memory) { + public long getMemorySize() { + return Long.MAX_VALUE; + } + + @Override + public void setMemory(long memory) { throw new RuntimeException("UNBOUNDED cannot be modified!"); } @@ -79,26 +96,31 @@ public class Resources { } @Override - public void setVirtualCores(int cores) { + public long getVirtualCoresSize() { + return Long.MAX_VALUE; + } + + @Override + public void setVirtualCores(long cores) { throw new RuntimeException("UNBOUNDED cannot be modified!"); } @Override public int compareTo(Resource o) { - int diff = Integer.MAX_VALUE - o.getMemory(); + long diff = Long.MAX_VALUE - o.getMemorySize(); if (diff == 0) { - diff = Integer.MAX_VALUE - o.getVirtualCores(); + diff = Long.MAX_VALUE - o.getVirtualCoresSize(); } - return diff; + return Long.signum(diff); } }; - public static Resource createResource(int memory) { + public static Resource createResource(long memory) { return createResource(memory, (memory > 0) ? 1 : 0); } - public static Resource createResource(int memory, int cores) { + public static Resource createResource(long memory, long cores) { Resource resource = Records.newRecord(Resource.class); resource.setMemory(memory); resource.setVirtualCores(cores); @@ -114,11 +136,11 @@ public class Resources { } public static Resource clone(Resource res) { - return createResource(res.getMemory(), res.getVirtualCores()); + return createResource(res.getMemorySize(), res.getVirtualCores()); } public static Resource addTo(Resource lhs, Resource rhs) { - lhs.setMemory(lhs.getMemory() + rhs.getMemory()); + lhs.setMemory(lhs.getMemorySize() + rhs.getMemorySize()); lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores()); return lhs; } @@ -128,7 +150,7 @@ public class Resources { } public static Resource subtractFrom(Resource lhs, Resource rhs) { - lhs.setMemory(lhs.getMemory() - rhs.getMemory()); + lhs.setMemory(lhs.getMemorySize() - rhs.getMemorySize()); lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores()); return lhs; } @@ -142,7 +164,7 @@ public class Resources { } public static Resource multiplyTo(Resource lhs, double by) { - lhs.setMemory((int)(lhs.getMemory() * by)); + lhs.setMemory((int)(lhs.getMemorySize() * by)); lhs.setVirtualCores((int)(lhs.getVirtualCores() * by)); return lhs; } @@ -157,7 +179,7 @@ public class Resources { */ public static Resource multiplyAndAddTo( Resource lhs, Resource rhs, double by) { - lhs.setMemory(lhs.getMemory() + (int)(rhs.getMemory() * by)); + lhs.setMemory(lhs.getMemorySize() + (int)(rhs.getMemorySize() * by)); lhs.setVirtualCores(lhs.getVirtualCores() + (int)(rhs.getVirtualCores() * by)); return lhs; @@ -175,7 +197,7 @@ public class Resources { public static Resource multiplyAndRoundDown(Resource lhs, double by) { Resource out = clone(lhs); - out.setMemory((int)(lhs.getMemory() * by)); + out.setMemory((int)(lhs.getMemorySize() * by)); out.setVirtualCores((int)(lhs.getVirtualCores() * by)); return out; } @@ -264,7 +286,7 @@ public class Resources { } public static boolean fitsIn(Resource smaller, Resource bigger) { - return smaller.getMemory() <= bigger.getMemory() && + return smaller.getMemorySize() <= bigger.getMemorySize() && smaller.getVirtualCores() <= bigger.getVirtualCores(); } @@ -274,12 +296,12 @@ public class Resources { } public static Resource componentwiseMin(Resource lhs, Resource rhs) { - return createResource(Math.min(lhs.getMemory(), rhs.getMemory()), + return createResource(Math.min(lhs.getMemorySize(), rhs.getMemorySize()), Math.min(lhs.getVirtualCores(), rhs.getVirtualCores())); } public static Resource componentwiseMax(Resource lhs, Resource rhs) { - return createResource(Math.max(lhs.getMemory(), rhs.getMemory()), + return createResource(Math.max(lhs.getMemorySize(), rhs.getMemorySize()), Math.max(lhs.getVirtualCores(), rhs.getVirtualCores())); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index d38ffdc..7396adc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -24,18 +24,18 @@ import static org.junit.Assert.assertTrue; public class TestResources { - public Resource createResource(int memory, int vCores) { + public Resource createResource(long memory, long vCores) { return Resource.newInstance(memory, vCores); } @Test(timeout=1000) public void testCompareToWithUnboundedResource() { assertTrue(Resources.unbounded().compareTo( - createResource(Integer.MAX_VALUE, Integer.MAX_VALUE)) == 0); + createResource(Long.MAX_VALUE, Long.MAX_VALUE)) == 0); assertTrue(Resources.unbounded().compareTo( - createResource(Integer.MAX_VALUE, 0)) > 0); + createResource(Long.MAX_VALUE, 0)) > 0); assertTrue(Resources.unbounded().compareTo( - createResource(0, Integer.MAX_VALUE)) > 0); + createResource(0, Long.MAX_VALUE)) > 0); } @Test(timeout=1000) http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 4fdd43c..1d1b186 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -425,7 +425,7 @@ public class BuilderUtils { return report; } - public static Resource newResource(int memory, int vCores) { + public static Resource newResource(long memory, long vCores) { Resource resource = recordFactory.newRecordInstance(Resource.class); resource.setMemory(memory); resource.setVirtualCores(vCores); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
