http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index cad3b2e..ff4519d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -59,8 +59,8 @@ public class AppInfo { protected long elapsedTime; protected String applicationTags; protected int priority; - private int allocatedCpuVcores; - private int allocatedMemoryMB; + private long allocatedCpuVcores; + private long allocatedMemoryMB; protected boolean unmanagedApplication; private String appNodeLabelExpression; private String amNodeLabelExpression; @@ -100,7 +100,7 @@ public class AppInfo { allocatedCpuVcores = app.getApplicationResourceUsageReport() .getUsedResources().getVirtualCores(); allocatedMemoryMB = app.getApplicationResourceUsageReport() - .getUsedResources().getMemory(); + .getUsedResources().getMemorySize(); } } progress = app.getProgress() * 100; // in percent @@ -152,11 +152,11 @@ public class AppInfo { return runningContainers; } - public int getAllocatedCpuVcores() { + public long getAllocatedCpuVcores() { return allocatedCpuVcores; } - public int getAllocatedMemoryMB() { + public long getAllocatedMemoryMB() { return allocatedMemoryMB; }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java index f127f9c..1a5ee85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java @@ -36,8 +36,8 @@ import org.apache.hadoop.yarn.util.Times; public class ContainerInfo { protected String containerId; - protected int allocatedMB; - protected int allocatedVCores; + protected long allocatedMB; + protected long allocatedVCores; protected String assignedNodeId; protected int priority; protected long startedTime; @@ -57,7 +57,7 @@ public class ContainerInfo { public ContainerInfo(ContainerReport container) { containerId = container.getContainerId().toString(); if (container.getAllocatedResource() != null) { - allocatedMB = container.getAllocatedResource().getMemory(); + allocatedMB = container.getAllocatedResource().getMemorySize(); allocatedVCores = container.getAllocatedResource().getVirtualCores(); } if (container.getAssignedNode() != null) { @@ -79,11 +79,11 @@ public class ContainerInfo { return containerId; } - public int getAllocatedMB() { + public long getAllocatedMB() { return allocatedMB; } - public int getAllocatedVCores() { + public long getAllocatedVCores() { return allocatedVCores; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java index c9427dd..e25547d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java @@ -180,7 +180,7 @@ public class TestYarnServerApiClasses { assertEquals(1, copy.getContainersToDecrease().get(0) .getId().getContainerId()); assertEquals(1024, copy.getContainersToDecrease().get(1) - .getResource().getMemory()); + .getResource().getMemorySize()); } /** @@ -201,7 +201,7 @@ public class TestYarnServerApiClasses { assertEquals(8080, copy.getHttpPort()); assertEquals(9090, copy.getNodeId().getPort()); - assertEquals(10000, copy.getResource().getMemory()); + assertEquals(10000, copy.getResource().getMemorySize()); assertEquals(2, copy.getResource().getVirtualCores()); } @@ -273,7 +273,7 @@ public class TestYarnServerApiClasses { assertEquals(1, copy.getIncreasedContainers().get(0) .getId().getContainerId()); assertEquals(4096, copy.getIncreasedContainers().get(1) - .getResource().getMemory()); + .getResource().getMemorySize()); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 8c74bf5..034b13d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -404,7 +404,7 @@ public abstract class ContainerExecutor implements Configurable { .getBoolean( YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED, YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) { - memory = resource.getMemory(); + memory = (int) resource.getMemorySize(); } if (conf.getBoolean( http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index dbc7de8..a983363 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -168,7 +168,7 @@ public class ContainerImpl implements Container { if (recoveredCapability != null && !this.resource.equals(recoveredCapability)) { // resource capability had been updated before NM was down - this.resource = Resource.newInstance(recoveredCapability.getMemory(), + this.resource = Resource.newInstance(recoveredCapability.getMemorySize(), recoveredCapability.getVirtualCores()); } } @@ -516,7 +516,7 @@ public class ContainerImpl implements Container { long launchDuration = clock.getTime() - containerLaunchStartTime; metrics.addContainerLaunchDuration(launchDuration); - long pmemBytes = getResource().getMemory() * 1024 * 1024L; + long pmemBytes = getResource().getMemorySize() * 1024 * 1024L; float pmemRatio = daemonConf.getFloat( YarnConfiguration.NM_VMEM_PMEM_RATIO, YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 8c907c7..a0b80de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -650,7 +650,7 @@ public class ContainersMonitorImpl extends AbstractService implements ChangeMonitoringContainerResourceEvent changeEvent = (ChangeMonitoringContainerResourceEvent) monitoringEvent; Resource resource = changeEvent.getResource(); - pmemLimitMBs = resource.getMemory(); + pmemLimitMBs = (int) resource.getMemorySize(); vmemLimitMBs = (int) (pmemLimitMBs * vmemRatio); cpuVcores = resource.getVirtualCores(); usageMetrics.recordResourceLimit( @@ -748,7 +748,7 @@ public class ContainersMonitorImpl extends AbstractService implements } LOG.info("Changing resource-monitoring for " + containerId); updateContainerMetrics(monitoringEvent); - long pmemLimit = changeEvent.getResource().getMemory() * 1024L * 1024L; + long pmemLimit = changeEvent.getResource().getMemorySize() * 1024L * 1024L; long vmemLimit = (long) (pmemLimit * vmemRatio); int cpuVcores = changeEvent.getResource().getVirtualCores(); processTreeInfo.setResourceLimit(pmemLimit, vmemLimit, cpuVcores); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java index a38d0b7..11e8684 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java @@ -115,9 +115,9 @@ public class NodeManagerMetrics { public void allocateContainer(Resource res) { allocatedContainers.incr(); - allocatedMB = allocatedMB + res.getMemory(); + allocatedMB = allocatedMB + res.getMemorySize(); allocatedGB.set((int)Math.ceil(allocatedMB/1024d)); - availableMB = availableMB - res.getMemory(); + availableMB = availableMB - res.getMemorySize(); availableGB.set((int)Math.floor(availableMB/1024d)); allocatedVCores.incr(res.getVirtualCores()); availableVCores.decr(res.getVirtualCores()); @@ -125,16 +125,16 @@ public class NodeManagerMetrics { public void releaseContainer(Resource res) { allocatedContainers.decr(); - allocatedMB = allocatedMB - res.getMemory(); + allocatedMB = allocatedMB - res.getMemorySize(); allocatedGB.set((int)Math.ceil(allocatedMB/1024d)); - availableMB = availableMB + res.getMemory(); + availableMB = availableMB + res.getMemorySize(); availableGB.set((int)Math.floor(availableMB/1024d)); allocatedVCores.decr(res.getVirtualCores()); availableVCores.incr(res.getVirtualCores()); } public void changeContainer(Resource before, Resource now) { - int deltaMB = now.getMemory() - before.getMemory(); + long deltaMB = now.getMemorySize() - before.getMemorySize(); int deltaVCores = now.getVirtualCores() - before.getVirtualCores(); allocatedMB = allocatedMB + deltaMB; allocatedGB.set((int)Math.ceil(allocatedMB/1024d)); @@ -145,7 +145,7 @@ public class NodeManagerMetrics { } public void addResource(Resource res) { - availableMB = availableMB + res.getMemory(); + availableMB = availableMB + res.getMemorySize(); availableGB.incr((int)Math.floor(availableMB/1024d)); availableVCores.incr(res.getVirtualCores()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java index cf022b9..10a8156 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java @@ -81,7 +81,7 @@ public class ContainerInfo { this.user = container.getUser(); Resource res = container.getResource(); if (res != null) { - this.totalMemoryNeededMB = res.getMemory(); + this.totalMemoryNeededMB = res.getMemorySize(); this.totalVCoresNeeded = res.getVirtualCores(); } this.containerLogsShortLink = ujoin("containerlogs", this.id, http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index b1f826b..94f654a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -188,7 +188,7 @@ public class TestNodeStatusUpdater { InetSocketAddress expected = NetUtils.getConnectAddress( conf.getSocketAddr(YarnConfiguration.NM_ADDRESS, null, -1)); Assert.assertEquals(NetUtils.getHostPortString(expected), nodeId.toString()); - Assert.assertEquals(5 * 1024, resource.getMemory()); + Assert.assertEquals(5 * 1024, resource.getMemorySize()); registeredNodes.add(nodeId); RegisterNodeManagerResponse response = recordFactory @@ -914,7 +914,7 @@ public class TestNodeStatusUpdater { conf.getSocketAddr(YarnConfiguration.NM_ADDRESS, null, -1)); Assert.assertEquals(NetUtils.getHostPortString(expected), nodeId.toString()); - Assert.assertEquals(5 * 1024, resource.getMemory()); + Assert.assertEquals(5 * 1024, resource.getMemorySize()); registeredNodes.add(nodeId); RegisterNodeManagerResponse response = recordFactory http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index 008f7ee..94145e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -228,7 +228,7 @@ public class TestContainersMonitor extends BaseContainerManagerTest { commands.add("/bin/bash"); commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); - Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1); + Resource r = BuilderUtils.newResource(0, 0); ContainerTokenIdentifier containerIdentifier = new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user, r, System.currentTimeMillis() + 120000, 123, DUMMY_RM_IDENTIFIER, http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java index ef4a0d4..f6af030 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java @@ -94,9 +94,9 @@ public class RMNMInfo implements RMNMInfoBeans { ni.getNodeManagerVersion()); if(report != null) { info.put("NumContainers", report.getNumContainers()); - info.put("UsedMemoryMB", report.getUsedResource().getMemory()); + info.put("UsedMemoryMB", report.getUsedResource().getMemorySize()); info.put("AvailableMemoryMB", - report.getAvailableResource().getMemory()); + report.getAvailableResource().getMemorySize()); } nodesInfo.add(info); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 1318d58..95fdb05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -142,7 +142,7 @@ public class RMServerUtils { // example, you cannot request target resource of a <10G, 10> container to // <20G, 8> if (increase) { - if (originalResource.getMemory() > targetResource.getMemory() + if (originalResource.getMemorySize() > targetResource.getMemorySize() || originalResource.getVirtualCores() > targetResource .getVirtualCores()) { String msg = @@ -153,7 +153,7 @@ public class RMServerUtils { throw new InvalidResourceRequestException(msg); } } else { - if (originalResource.getMemory() < targetResource.getMemory() + if (originalResource.getMemorySize() < targetResource.getMemorySize() || originalResource.getVirtualCores() < targetResource .getVirtualCores()) { String msg = @@ -243,15 +243,15 @@ public class RMServerUtils { return; } for (ContainerResourceChangeRequest request : requests) { - if (request.getCapability().getMemory() < 0 - || request.getCapability().getMemory() > maximumAllocation - .getMemory()) { + if (request.getCapability().getMemorySize() < 0 + || request.getCapability().getMemorySize() > maximumAllocation + .getMemorySize()) { throw new InvalidResourceRequestException("Invalid " + (increase ? "increase" : "decrease") + " request" + ", requested memory < 0" + ", or requested memory > max configured" + ", requestedMemory=" - + request.getCapability().getMemory() + ", maxMemory=" - + maximumAllocation.getMemory()); + + request.getCapability().getMemorySize() + ", maxMemory=" + + maximumAllocation.getMemorySize()); } if (request.getCapability().getVirtualCores() < 0 || request.getCapability().getVirtualCores() > maximumAllocation http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index f52b5ab..bf3dc45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -339,7 +339,7 @@ public class ResourceTrackerService extends AbstractService implements } // Check if this node has minimum allocations - if (capability.getMemory() < minAllocMb + if (capability.getMemorySize() < minAllocMb || capability.getVirtualCores() < minAllocVcores) { String message = "NodeManager from " + host http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index 0e3802b..b0b976d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -480,7 +480,7 @@ public class SystemMetricsPublisher extends CompositeService { TimelineEntity entity = createContainerEntity(event.getContainerId()); Map<String, Object> entityInfo = new HashMap<String, Object>(); entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO, - event.getAllocatedResource().getMemory()); + event.getAllocatedResource().getMemorySize()); entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO, event.getAllocatedResource().getVirtualCores()); entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index 6b431b1..ba0bb4d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -1083,17 +1083,17 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic void appendLogString(StringBuilder sb) { sb.append(queueName).append(", ") - .append(current.getMemory()).append(", ") + .append(current.getMemorySize()).append(", ") .append(current.getVirtualCores()).append(", ") - .append(pending.getMemory()).append(", ") + .append(pending.getMemorySize()).append(", ") .append(pending.getVirtualCores()).append(", ") - .append(guaranteed.getMemory()).append(", ") + .append(guaranteed.getMemorySize()).append(", ") .append(guaranteed.getVirtualCores()).append(", ") - .append(idealAssigned.getMemory()).append(", ") + .append(idealAssigned.getMemorySize()).append(", ") .append(idealAssigned.getVirtualCores()).append(", ") - .append(toBePreempted.getMemory()).append(", ") + .append(toBePreempted.getMemorySize()).append(", ") .append(toBePreempted.getVirtualCores() ).append(", ") - .append(actuallyPreempted.getMemory()).append(", ") + .append(actuallyPreempted.getMemorySize()).append(", ") .append(actuallyPreempted.getVirtualCores()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java index eaf2902..90357e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java @@ -160,7 +160,7 @@ public abstract class AbstractSchedulerPlanFollower implements PlanFollower { } Resource capToAssign = res.getResourcesAtTime(now); float targetCapacity = 0f; - if (planResources.getMemory() > 0 + if (planResources.getMemorySize() > 0 && planResources.getVirtualCores() > 0) { if (shouldResize) { capToAssign = http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java index f8b68e3..07bee99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java @@ -246,7 +246,7 @@ public class CapacityOverTimePolicy implements SharingPolicy { long vcores; public IntegralResource(Resource resource) { - this.memory = resource.getMemory(); + this.memory = resource.getMemorySize(); this.vcores = resource.getVirtualCores(); } @@ -256,12 +256,12 @@ public class CapacityOverTimePolicy implements SharingPolicy { } public void add(Resource r) { - memory += r.getMemory(); + memory += r.getMemorySize(); vcores += r.getVirtualCores(); } public void subtract(Resource r) { - memory -= r.getMemory(); + memory -= r.getMemorySize(); vcores -= r.getVirtualCores(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemUtil.java index aba4822..81d32c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemUtil.java @@ -106,7 +106,7 @@ public final class ReservationSystemUtil { public static ResourceProto convertToProtoFormat(Resource e) { return YarnProtos.ResourceProto.newBuilder() - .setMemory(e.getMemory()) + .setMemory(e.getMemorySize()) .setVirtualCores(e.getVirtualCores()) .build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageEarliestStartByDemand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageEarliestStartByDemand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageEarliestStartByDemand.java index 5a46a4e..43d6584 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageEarliestStartByDemand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageEarliestStartByDemand.java @@ -88,7 +88,7 @@ public class StageEarliestStartByDemand implements StageEarliestStart { // Weight = total memory consumption of stage protected double calcWeight(ReservationRequest stage) { - return (stage.getDuration() * stage.getCapability().getMemory()) + return (stage.getDuration() * stage.getCapability().getMemorySize()) * (stage.getNumContainers()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index 16ab55d..e5b1dbc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -708,7 +708,7 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> { } long usedMillis = container.finishTime - container.creationTime; - long memorySeconds = resource.getMemory() + long memorySeconds = resource.getMemorySize() * usedMillis / DateUtils.MILLIS_PER_SECOND; long vcoreSeconds = resource.getVirtualCores() * usedMillis / DateUtils.MILLIS_PER_SECOND; http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 358802b..d708a46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -703,11 +703,11 @@ public abstract class AbstractYarnScheduler maxAllocWriteLock.lock(); try { if (add) { // added node - int nodeMemory = totalResource.getMemory(); + int nodeMemory = (int)totalResource.getMemorySize(); if (nodeMemory > maxNodeMemory) { maxNodeMemory = nodeMemory; maximumAllocation.setMemory(Math.min( - configuredMaximumAllocation.getMemory(), maxNodeMemory)); + configuredMaximumAllocation.getMemorySize(), maxNodeMemory)); } int nodeVCores = totalResource.getVirtualCores(); if (nodeVCores > maxNodeVCores) { @@ -716,7 +716,7 @@ public abstract class AbstractYarnScheduler configuredMaximumAllocation.getVirtualCores(), maxNodeVCores)); } } else { // removed node - if (maxNodeMemory == totalResource.getMemory()) { + if (maxNodeMemory == totalResource.getMemorySize()) { maxNodeMemory = -1; } if (maxNodeVCores == totalResource.getVirtualCores()) { @@ -727,7 +727,7 @@ public abstract class AbstractYarnScheduler if (maxNodeMemory == -1 || maxNodeVCores == -1) { for (Map.Entry<NodeId, N> nodeEntry : nodes.entrySet()) { int nodeMemory = - nodeEntry.getValue().getTotalResource().getMemory(); + (int)nodeEntry.getValue().getTotalResource().getMemorySize(); if (nodeMemory > maxNodeMemory) { maxNodeMemory = nodeMemory; } @@ -738,10 +738,10 @@ public abstract class AbstractYarnScheduler } } if (maxNodeMemory == -1) { // no nodes - maximumAllocation.setMemory(configuredMaximumAllocation.getMemory()); + maximumAllocation.setMemory(configuredMaximumAllocation.getMemorySize()); } else { maximumAllocation.setMemory( - Math.min(configuredMaximumAllocation.getMemory(), maxNodeMemory)); + Math.min(configuredMaximumAllocation.getMemorySize(), maxNodeMemory)); } if (maxNodeVCores == -1) { // no nodes maximumAllocation.setVirtualCores(configuredMaximumAllocation.getVirtualCores()); @@ -760,7 +760,7 @@ public abstract class AbstractYarnScheduler maxAllocWriteLock.lock(); try { configuredMaximumAllocation = Resources.clone(newMaxAlloc); - int maxMemory = newMaxAlloc.getMemory(); + long maxMemory = newMaxAlloc.getMemorySize(); if (maxNodeMemory != -1) { maxMemory = Math.min(maxMemory, maxNodeMemory); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index d9c7283..69c3e8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -38,6 +38,7 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableCounterInt; import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; +import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.metrics2.lib.MutableRate; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; @@ -59,8 +60,8 @@ public class QueueMetrics implements MetricsSource { @Metric("# of apps killed") MutableCounterInt appsKilled; @Metric("# of apps failed") MutableCounterInt appsFailed; - @Metric("Allocated memory in MB") MutableGaugeInt allocatedMB; - @Metric("Allocated CPU in virtual cores") MutableGaugeInt allocatedVCores; + @Metric("Allocated memory in MB") MutableGaugeLong allocatedMB; + @Metric("Allocated CPU in virtual cores") MutableGaugeLong allocatedVCores; @Metric("# of allocated containers") MutableGaugeInt allocatedContainers; @Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated; @Metric("Aggregate # of allocated node-local containers") @@ -70,13 +71,13 @@ public class QueueMetrics implements MetricsSource { @Metric("Aggregate # of allocated off-switch containers") MutableCounterLong aggregateOffSwitchContainersAllocated; @Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased; - @Metric("Available memory in MB") MutableGaugeInt availableMB; - @Metric("Available CPU in virtual cores") MutableGaugeInt availableVCores; - @Metric("Pending memory allocation in MB") MutableGaugeInt pendingMB; - @Metric("Pending CPU allocation in virtual cores") MutableGaugeInt pendingVCores; + @Metric("Available memory in MB") MutableGaugeLong availableMB; + @Metric("Available CPU in virtual cores") MutableGaugeLong availableVCores; + @Metric("Pending memory allocation in MB") MutableGaugeLong pendingMB; + @Metric("Pending CPU allocation in virtual cores") MutableGaugeLong pendingVCores; @Metric("# of pending containers") MutableGaugeInt pendingContainers; - @Metric("# of reserved memory in MB") MutableGaugeInt reservedMB; - @Metric("Reserved CPU in virtual cores") MutableGaugeInt reservedVCores; + @Metric("# of reserved memory in MB") MutableGaugeLong reservedMB; + @Metric("Reserved CPU in virtual cores") MutableGaugeLong reservedVCores; @Metric("# of reserved containers") MutableGaugeInt reservedContainers; @Metric("# of active users") MutableGaugeInt activeUsers; @Metric("# of active applications") MutableGaugeInt activeApplications; @@ -325,7 +326,7 @@ public class QueueMetrics implements MetricsSource { * @param limit resource limit */ public void setAvailableResourcesToQueue(Resource limit) { - availableMB.set(limit.getMemory()); + availableMB.set(limit.getMemorySize()); availableVCores.set(limit.getVirtualCores()); } @@ -362,8 +363,8 @@ public class QueueMetrics implements MetricsSource { private void _incrPendingResources(int containers, Resource res) { pendingContainers.incr(containers); - pendingMB.incr(res.getMemory() * containers); - pendingVCores.incr(res.getVirtualCores() * containers); + pendingMB.incr(res.getMemorySize() * containers); + pendingVCores.incr(res.getVirtualCoresSize() * containers); } public void decrPendingResources(String user, int containers, Resource res) { @@ -379,8 +380,8 @@ public class QueueMetrics implements MetricsSource { private void _decrPendingResources(int containers, Resource res) { pendingContainers.decr(containers); - pendingMB.decr(res.getMemory() * containers); - pendingVCores.decr(res.getVirtualCores() * containers); + pendingMB.decr(res.getMemorySize() * containers); + pendingVCores.decr(res.getVirtualCoresSize() * containers); } public void incrNodeTypeAggregations(String user, NodeType type) { @@ -407,8 +408,8 @@ public class QueueMetrics implements MetricsSource { allocatedContainers.incr(containers); aggregateContainersAllocated.incr(containers); - allocatedMB.incr(res.getMemory() * containers); - allocatedVCores.incr(res.getVirtualCores() * containers); + allocatedMB.incr(res.getMemorySize() * containers); + allocatedVCores.incr(res.getVirtualCoresSize() * containers); if (decrPending) { _decrPendingResources(containers, res); } @@ -428,10 +429,10 @@ public class QueueMetrics implements MetricsSource { * @param res */ public void allocateResources(String user, Resource res) { - allocatedMB.incr(res.getMemory()); + allocatedMB.incr(res.getMemorySize()); allocatedVCores.incr(res.getVirtualCores()); - pendingMB.decr(res.getMemory()); + pendingMB.decr(res.getMemorySize()); pendingVCores.decr(res.getVirtualCores()); QueueMetrics userMetrics = getUserMetrics(user); @@ -446,8 +447,8 @@ public class QueueMetrics implements MetricsSource { public void releaseResources(String user, int containers, Resource res) { allocatedContainers.decr(containers); aggregateContainersReleased.incr(containers); - allocatedMB.decr(res.getMemory() * containers); - allocatedVCores.decr(res.getVirtualCores() * containers); + allocatedMB.decr(res.getMemorySize() * containers); + allocatedVCores.decr(res.getVirtualCoresSize() * containers); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.releaseResources(user, containers, res); @@ -464,7 +465,7 @@ public class QueueMetrics implements MetricsSource { * @param res */ public void releaseResources(String user, Resource res) { - allocatedMB.decr(res.getMemory()); + allocatedMB.decr(res.getMemorySize()); allocatedVCores.decr(res.getVirtualCores()); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { @@ -477,7 +478,7 @@ public class QueueMetrics implements MetricsSource { public void reserveResource(String user, Resource res) { reservedContainers.incr(); - reservedMB.incr(res.getMemory()); + reservedMB.incr(res.getMemorySize()); reservedVCores.incr(res.getVirtualCores()); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { @@ -490,7 +491,7 @@ public class QueueMetrics implements MetricsSource { public void unreserveResource(String user, Resource res) { reservedContainers.decr(); - reservedMB.decr(res.getMemory()); + reservedMB.decr(res.getMemorySize()); reservedVCores.decr(res.getVirtualCores()); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { @@ -563,11 +564,11 @@ public class QueueMetrics implements MetricsSource { return BuilderUtils.newResource(allocatedMB.value(), allocatedVCores.value()); } - public int getAllocatedMB() { + public long getAllocatedMB() { return allocatedMB.value(); } - public int getAllocatedVirtualCores() { + public long getAllocatedVirtualCores() { return allocatedVCores.value(); } @@ -575,19 +576,19 @@ public class QueueMetrics implements MetricsSource { return allocatedContainers.value(); } - public int getAvailableMB() { + public long getAvailableMB() { return availableMB.value(); } - public int getAvailableVirtualCores() { + public long getAvailableVirtualCores() { return availableVCores.value(); } - public int getPendingMB() { + public long getPendingMB() { return pendingMB.value(); } - public int getPendingVirtualCores() { + public long getPendingVirtualCores() { return pendingVCores.value(); } @@ -595,11 +596,11 @@ public class QueueMetrics implements MetricsSource { return pendingContainers.value(); } - public int getReservedMB() { + public long getReservedMB() { return reservedMB.value(); } - public int getReservedVirtualCores() { + public long getReservedVirtualCores() { return reservedVCores.value(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 0cbb88d..8929512 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -442,7 +442,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { */ public synchronized Resource getHeadroom() { // Corner case to deal with applications being slightly over-limit - if (resourceLimit.getMemory() < 0) { + if (resourceLimit.getMemorySize() < 0) { resourceLimit.setMemory(0); } @@ -478,7 +478,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { if (requests != null) { LOG.debug("showRequests:" + " application=" + getApplicationId() + " headRoom=" + getHeadroom() + " currentConsumption=" - + attemptResourceUsage.getUsed().getMemory()); + + attemptResourceUsage.getUsed().getMemorySize()); for (ResourceRequest request : requests.values()) { LOG.debug("showRequests:" + " application=" + getApplicationId() + " request=" + request); @@ -680,7 +680,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { for (RMContainer rmContainer : this.liveContainers.values()) { long usedMillis = currentTimeMillis - rmContainer.getCreationTime(); Resource resource = rmContainer.getContainer().getResource(); - memorySeconds += resource.getMemory() * usedMillis / + memorySeconds += resource.getMemorySize() * usedMillis / DateUtils.MILLIS_PER_SECOND; vcoreSeconds += resource.getVirtualCores() * usedMillis / DateUtils.MILLIS_PER_SECOND; http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index b460964..c999e26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -274,13 +274,13 @@ public class SchedulerUtils { private static void validateResourceRequest(ResourceRequest resReq, Resource maximumResource, QueueInfo queueInfo, RMContext rmContext) throws InvalidResourceRequestException { - if (resReq.getCapability().getMemory() < 0 || - resReq.getCapability().getMemory() > maximumResource.getMemory()) { + if (resReq.getCapability().getMemorySize() < 0 || + resReq.getCapability().getMemorySize() > maximumResource.getMemorySize()) { throw new InvalidResourceRequestException("Invalid resource request" + ", requested memory < 0" + ", or requested memory > max configured" - + ", requestedMemory=" + resReq.getCapability().getMemory() - + ", maxMemory=" + maximumResource.getMemory()); + + ", requestedMemory=" + resReq.getCapability().getMemorySize() + + ", maxMemory=" + maximumResource.getMemorySize()); } if (resReq.getCapability().getVirtualCores() < 0 || resReq.getCapability().getVirtualCores() > http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java index 51d39ff..58c3984 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java @@ -23,7 +23,7 @@ import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.metrics2.lib.MutableGaugeInt; +import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -32,37 +32,37 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; public class CSQueueMetrics extends QueueMetrics { @Metric("AM memory limit in MB") - MutableGaugeInt AMResourceLimitMB; + MutableGaugeLong AMResourceLimitMB; @Metric("AM CPU limit in virtual cores") - MutableGaugeInt AMResourceLimitVCores; + MutableGaugeLong AMResourceLimitVCores; @Metric("Used AM memory limit in MB") - MutableGaugeInt usedAMResourceMB; + MutableGaugeLong usedAMResourceMB; @Metric("Used AM CPU limit in virtual cores") - MutableGaugeInt usedAMResourceVCores; + MutableGaugeLong usedAMResourceVCores; CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { super(ms, queueName, parent, enableUserMetrics, conf); } - public int getAMResourceLimitMB() { + public long getAMResourceLimitMB() { return AMResourceLimitMB.value(); } - public int getAMResourceLimitVCores() { + public long getAMResourceLimitVCores() { return AMResourceLimitVCores.value(); } - public int getUsedAMResourceMB() { + public long getUsedAMResourceMB() { return usedAMResourceMB.value(); } - public int getUsedAMResourceVCores() { + public long getUsedAMResourceVCores() { return usedAMResourceVCores.value(); } public void setAMResouceLimit(Resource res) { - AMResourceLimitMB.set(res.getMemory()); + AMResourceLimitMB.set(res.getMemorySize()); AMResourceLimitVCores.set(res.getVirtualCores()); } @@ -74,7 +74,7 @@ public class CSQueueMetrics extends QueueMetrics { } public void incAMUsed(String user, Resource res) { - usedAMResourceMB.incr(res.getMemory()); + usedAMResourceMB.incr(res.getMemorySize()); usedAMResourceVCores.incr(res.getVirtualCores()); CSQueueMetrics userMetrics = (CSQueueMetrics) getUserMetrics(user); if (userMetrics != null) { @@ -83,7 +83,7 @@ public class CSQueueMetrics extends QueueMetrics { } public void decAMUsed(String user, Resource res) { - usedAMResourceMB.decr(res.getMemory()); + usedAMResourceMB.decr(res.getMemorySize()); usedAMResourceVCores.decr(res.getVirtualCores()); CSQueueMetrics userMetrics = (CSQueueMetrics) getUserMetrics(user); if (userMetrics != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java index 95a12dc..49892f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java @@ -65,7 +65,7 @@ public class CapacityHeadroomProvider { } } // Corner case to deal with applications being slightly over-limit - if (headroom.getMemory() < 0) { + if (headroom.getMemorySize() < 0) { headroom.setMemory(0); } return headroom; http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 3756d9e..c7fe939 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -671,7 +671,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur */ public Resource getMaximumAllocationPerQueue(String queue) { String queuePrefix = getQueuePrefix(queue); - int maxAllocationMbPerQueue = getInt(queuePrefix + MAXIMUM_ALLOCATION_MB, + long maxAllocationMbPerQueue = getInt(queuePrefix + MAXIMUM_ALLOCATION_MB, (int)UNDEFINED); int maxAllocationVcoresPerQueue = getInt( queuePrefix + MAXIMUM_ALLOCATION_VCORES, (int)UNDEFINED); @@ -684,7 +684,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur Resource clusterMax = getMaximumAllocation(); if (maxAllocationMbPerQueue == (int)UNDEFINED) { LOG.info("max alloc mb per queue for " + queue + " is undefined"); - maxAllocationMbPerQueue = clusterMax.getMemory(); + maxAllocationMbPerQueue = clusterMax.getMemorySize(); } if (maxAllocationVcoresPerQueue == (int)UNDEFINED) { LOG.info("max alloc vcore per queue for " + queue + " is undefined"); @@ -692,7 +692,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur } Resource result = Resources.createResource(maxAllocationMbPerQueue, maxAllocationVcoresPerQueue); - if (maxAllocationMbPerQueue > clusterMax.getMemory() + if (maxAllocationMbPerQueue > clusterMax.getMemorySize() || maxAllocationVcoresPerQueue > clusterMax.getVirtualCores()) { throw new IllegalArgumentException( "Queue maximum allocation cannot be larger than the cluster setting" http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index f033adc..df20b3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -445,7 +445,7 @@ public class LeafQueue extends AbstractCSQueue { // since we have already told running AM's the size Resource oldMax = getMaximumAllocation(); Resource newMax = newlyParsedLeafQueue.getMaximumAllocation(); - if (newMax.getMemory() < oldMax.getMemory() + if (newMax.getMemorySize() < oldMax.getMemorySize() || newMax.getVirtualCores() < oldMax.getVirtualCores()) { throw new IOException( "Trying to reinitialize " http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java index 820cccd..ad6ada8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java @@ -442,7 +442,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator { priority, capability); // Can we allocate a container on this node? - int availableContainers = + long availableContainers = rc.computeAvailableContainers(available, capability); // How much need to unreserve equals to: http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index febe050..fe0e3e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -128,7 +128,7 @@ public class FSParentQueue extends FSQueue { public Resource getDemand() { readLock.lock(); try { - return Resource.newInstance(demand.getMemory(), demand.getVirtualCores()); + return Resource.newInstance(demand.getMemorySize(), demand.getVirtualCores()); } finally { readLock.unlock(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 713bdca..c5edbfb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -130,18 +130,18 @@ public abstract class FSQueue implements Queue, Schedulable { QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); queueInfo.setQueueName(getQueueName()); - if (scheduler.getClusterResource().getMemory() == 0) { + if (scheduler.getClusterResource().getMemorySize() == 0) { queueInfo.setCapacity(0.0f); } else { - queueInfo.setCapacity((float) getFairShare().getMemory() / - scheduler.getClusterResource().getMemory()); + queueInfo.setCapacity((float) getFairShare().getMemorySize() / + scheduler.getClusterResource().getMemorySize()); } - if (getFairShare().getMemory() == 0) { + if (getFairShare().getMemorySize() == 0) { queueInfo.setCurrentCapacity(0.0f); } else { - queueInfo.setCurrentCapacity((float) getResourceUsage().getMemory() / - getFairShare().getMemory()); + queueInfo.setCurrentCapacity((float) getResourceUsage().getMemorySize() / + getFairShare().getMemorySize()); } ArrayList<QueueInfo> childQueueInfos = new ArrayList<QueueInfo>(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java index 82c422b..4bdadb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java @@ -24,6 +24,7 @@ import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; +import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -31,69 +32,69 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @Metrics(context="yarn") public class FSQueueMetrics extends QueueMetrics { - @Metric("Fair share of memory in MB") MutableGaugeInt fairShareMB; - @Metric("Fair share of CPU in vcores") MutableGaugeInt fairShareVCores; - @Metric("Steady fair share of memory in MB") MutableGaugeInt steadyFairShareMB; - @Metric("Steady fair share of CPU in vcores") MutableGaugeInt steadyFairShareVCores; - @Metric("Minimum share of memory in MB") MutableGaugeInt minShareMB; - @Metric("Minimum share of CPU in vcores") MutableGaugeInt minShareVCores; - @Metric("Maximum share of memory in MB") MutableGaugeInt maxShareMB; - @Metric("Maximum share of CPU in vcores") MutableGaugeInt maxShareVCores; - + @Metric("Fair share of memory in MB") MutableGaugeLong fairShareMB; + @Metric("Fair share of CPU in vcores") MutableGaugeLong fairShareVCores; + @Metric("Steady fair share of memory in MB") MutableGaugeLong steadyFairShareMB; + @Metric("Steady fair share of CPU in vcores") MutableGaugeLong steadyFairShareVCores; + @Metric("Minimum share of memory in MB") MutableGaugeLong minShareMB; + @Metric("Minimum share of CPU in vcores") MutableGaugeLong minShareVCores; + @Metric("Maximum share of memory in MB") MutableGaugeLong maxShareMB; + @Metric("Maximum share of CPU in vcores") MutableGaugeLong maxShareVCores; + FSQueueMetrics(MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { super(ms, queueName, parent, enableUserMetrics, conf); } public void setFairShare(Resource resource) { - fairShareMB.set(resource.getMemory()); + fairShareMB.set(resource.getMemorySize()); fairShareVCores.set(resource.getVirtualCores()); } - public int getFairShareMB() { + public long getFairShareMB() { return fairShareMB.value(); } - public int getFairShareVirtualCores() { + public long getFairShareVirtualCores() { return fairShareVCores.value(); } public void setSteadyFairShare(Resource resource) { - steadyFairShareMB.set(resource.getMemory()); + steadyFairShareMB.set(resource.getMemorySize()); steadyFairShareVCores.set(resource.getVirtualCores()); } - public int getSteadyFairShareMB() { + public long getSteadyFairShareMB() { return steadyFairShareMB.value(); } - public int getSteadyFairShareVCores() { + public long getSteadyFairShareVCores() { return steadyFairShareVCores.value(); } public void setMinShare(Resource resource) { - minShareMB.set(resource.getMemory()); + minShareMB.set(resource.getMemorySize()); minShareVCores.set(resource.getVirtualCores()); } - public int getMinShareMB() { + public long getMinShareMB() { return minShareMB.value(); } - public int getMinShareVirtualCores() { + public long getMinShareVirtualCores() { return minShareVCores.value(); } public void setMaxShare(Resource resource) { - maxShareMB.set(resource.getMemory()); + maxShareMB.set(resource.getMemorySize()); maxShareVCores.set(resource.getVirtualCores()); } - public int getMaxShareMB() { + public long getMaxShareMB() { return maxShareMB.value(); } - public int getMaxShareVirtualCores() { + public long getMaxShareVirtualCores() { return maxShareVCores.value(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index b0352e5..28ca016 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -478,7 +478,7 @@ public class FairScheduler extends } private boolean isResourceGreaterThanNone(Resource toPreempt) { - return (toPreempt.getMemory() > 0) || (toPreempt.getVirtualCores() > 0); + return (toPreempt.getMemorySize() > 0) || (toPreempt.getVirtualCores() > 0); } protected void warnOrKillContainer(RMContainer container) { @@ -562,7 +562,7 @@ public class FairScheduler extends double weight = 1.0; if (sizeBasedWeight) { // Set weight based on current memory demand - weight = Math.log1p(app.getDemand().getMemory()) / Math.log(2); + weight = Math.log1p(app.getDemand().getMemorySize()) / Math.log(2); } weight *= app.getPriority().getPriority(); if (weightAdjuster != null) { @@ -1238,7 +1238,7 @@ public class FairScheduler extends private boolean shouldAttemptPreemption() { if (preemptionEnabled) { return (preemptionUtilizationThreshold < Math.max( - (float) rootMetrics.getAllocatedMB() / clusterResource.getMemory(), + (float) rootMetrics.getAllocatedMB() / clusterResource.getMemorySize(), (float) rootMetrics.getAllocatedVirtualCores() / clusterResource.getVirtualCores())); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java index f4fad32..5f500cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java @@ -124,15 +124,15 @@ public class ComputeFairShares { // have met all Schedulables' max shares. int totalMaxShare = 0; for (Schedulable sched : schedulables) { - int maxShare = getResourceValue(sched.getMaxShare(), type); - totalMaxShare = (int) Math.min((long)maxShare + (long)totalMaxShare, + long maxShare = getResourceValue(sched.getMaxShare(), type); + totalMaxShare = (int) Math.min(maxShare + (long)totalMaxShare, Integer.MAX_VALUE); if (totalMaxShare == Integer.MAX_VALUE) { break; } } - int totalResource = Math.max((getResourceValue(totalResources, type) - + long totalResource = Math.max((getResourceValue(totalResources, type) - takenResources), 0); totalResource = Math.min(totalMaxShare, totalResource); @@ -207,7 +207,7 @@ public class ComputeFairShares { int totalResource = 0; for (Schedulable sched : schedulables) { - int fixedShare = getFairShareIfFixed(sched, isSteadyShare, type); + long fixedShare = getFairShareIfFixed(sched, isSteadyShare, type); if (fixedShare < 0) { nonFixedSchedulables.add(sched); } else { @@ -229,7 +229,7 @@ public class ComputeFairShares { * The fairshare is fixed if either the maxShare is 0, weight is 0, * or the Schedulable is not active for instantaneous fairshare. */ - private static int getFairShareIfFixed(Schedulable sched, + private static long getFairShareIfFixed(Schedulable sched, boolean isSteadyShare, ResourceType type) { // Check if maxShare is 0 @@ -245,17 +245,17 @@ public class ComputeFairShares { // Check if weight is 0 if (sched.getWeights().getWeight(type) <= 0) { - int minShare = getResourceValue(sched.getMinShare(), type); + long minShare = getResourceValue(sched.getMinShare(), type); return (minShare <= 0) ? 0 : minShare; } return -1; } - private static int getResourceValue(Resource resource, ResourceType type) { + private static long getResourceValue(Resource resource, ResourceType type) { switch (type) { case MEMORY: - return resource.getMemory(); + return resource.getMemorySize(); case CPU: return resource.getVirtualCores(); default: @@ -263,7 +263,7 @@ public class ComputeFairShares { } } - private static void setResourceValue(int val, Resource resource, ResourceType type) { + private static void setResourceValue(long val, Resource resource, ResourceType type) { switch (type) { case MEMORY: resource.setMemory(val); http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java index 45fbf98..623437a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java @@ -101,13 +101,13 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy { @Override public Resource getHeadroom(Resource queueFairShare, Resource queueUsage, Resource maxAvailable) { - int queueAvailableMemory = - Math.max(queueFairShare.getMemory() - queueUsage.getMemory(), 0); + long queueAvailableMemory = + Math.max(queueFairShare.getMemorySize() - queueUsage.getMemorySize(), 0); int queueAvailableCPU = Math.max(queueFairShare.getVirtualCores() - queueUsage .getVirtualCores(), 0); Resource headroom = Resources.createResource( - Math.min(maxAvailable.getMemory(), queueAvailableMemory), + Math.min(maxAvailable.getMemorySize(), queueAvailableMemory), Math.min(maxAvailable.getVirtualCores(), queueAvailableCPU)); return headroom; @@ -180,8 +180,8 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy { */ void calculateShares(Resource resource, Resource pool, ResourceWeights shares, ResourceType[] resourceOrder, ResourceWeights weights) { - shares.setWeight(MEMORY, (float)resource.getMemory() / - (pool.getMemory() * weights.getWeight(MEMORY))); + shares.setWeight(MEMORY, (float)resource.getMemorySize() / + (pool.getMemorySize() * weights.getWeight(MEMORY))); shares.setWeight(CPU, (float)resource.getVirtualCores() / (pool.getVirtualCores() * weights.getWeight(CPU))); // sort order vector by resource share http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java index 3b9f07f..42d0420 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java @@ -82,13 +82,13 @@ public class FairSharePolicy extends SchedulingPolicy { s1.getResourceUsage(), minShare1); boolean s2Needy = Resources.lessThan(RESOURCE_CALCULATOR, null, s2.getResourceUsage(), minShare2); - minShareRatio1 = (double) s1.getResourceUsage().getMemory() - / Resources.max(RESOURCE_CALCULATOR, null, minShare1, ONE).getMemory(); - minShareRatio2 = (double) s2.getResourceUsage().getMemory() - / Resources.max(RESOURCE_CALCULATOR, null, minShare2, ONE).getMemory(); - useToWeightRatio1 = s1.getResourceUsage().getMemory() / + minShareRatio1 = (double) s1.getResourceUsage().getMemorySize() + / Resources.max(RESOURCE_CALCULATOR, null, minShare1, ONE).getMemorySize(); + minShareRatio2 = (double) s2.getResourceUsage().getMemorySize() + / Resources.max(RESOURCE_CALCULATOR, null, minShare2, ONE).getMemorySize(); + useToWeightRatio1 = s1.getResourceUsage().getMemorySize() / s1.getWeights().getWeight(ResourceType.MEMORY); - useToWeightRatio2 = s2.getResourceUsage().getMemory() / + useToWeightRatio2 = s2.getResourceUsage().getMemorySize() / s2.getWeights().getWeight(ResourceType.MEMORY); int res = 0; if (s1Needy && !s2Needy) @@ -124,10 +124,10 @@ public class FairSharePolicy extends SchedulingPolicy { @Override public Resource getHeadroom(Resource queueFairShare, Resource queueUsage, Resource maxAvailable) { - int queueAvailableMemory = Math.max( - queueFairShare.getMemory() - queueUsage.getMemory(), 0); + long queueAvailableMemory = Math.max( + queueFairShare.getMemorySize() - queueUsage.getMemorySize(), 0); Resource headroom = Resources.createResource( - Math.min(maxAvailable.getMemory(), queueAvailableMemory), + Math.min(maxAvailable.getMemorySize(), queueAvailableMemory), maxAvailable.getVirtualCores()); return headroom; } @@ -152,7 +152,7 @@ public class FairSharePolicy extends SchedulingPolicy { @Override public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMResource) { - return usage.getMemory() > maxAMResource.getMemory(); + return usage.getMemorySize() > maxAMResource.getMemorySize(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e57887/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java index a644e58..c277df1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java @@ -115,16 +115,16 @@ public class FifoPolicy extends SchedulingPolicy { @Override public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMResource) { - return usage.getMemory() > maxAMResource.getMemory(); + return usage.getMemorySize() > maxAMResource.getMemorySize(); } @Override public Resource getHeadroom(Resource queueFairShare, Resource queueUsage, Resource maxAvailable) { - int queueAvailableMemory = Math.max( - queueFairShare.getMemory() - queueUsage.getMemory(), 0); + long queueAvailableMemory = Math.max( + queueFairShare.getMemorySize() - queueUsage.getMemorySize(), 0); Resource headroom = Resources.createResource( - Math.min(maxAvailable.getMemory(), queueAvailableMemory), + Math.min(maxAvailable.getMemorySize(), queueAvailableMemory), maxAvailable.getVirtualCores()); return headroom; } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org