Author: szetszwo
Date: Sun Nov 3 17:51:52 2013
New Revision: 1538408
URL: http://svn.apache.org/r1538408
Log:
Merge r1537584 through r1538407 from trunk.
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt Sun Nov 3
17:51:52 2013
@@ -77,6 +77,11 @@ Release 2.3.0 - UNRELEASED
applications so that clients can get information about them post
RM-restart.
(Jian He via vinodkv)
+ YARN-1290. Let continuous scheduling achieve more balanced task assignment
+ (Wei Yan via Sandy Ryza)
+
+ YARN-786. Expose application resource usage in RM REST API (Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
@@ -178,6 +183,9 @@ Release 2.2.1 - UNRELEASED
YARN-1343. NodeManagers additions/restarts are not reported as node
updates
in AllocateResponse responses to AMs. (tucu)
+ YARN-1381. Same relaxLocality appears twice in exception message of
+ AMRMClientImpl#checkLocalityRelaxationConflict() (Ted Yu via Sandy Ryza)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
Sun Nov 3 17:51:52 2013
@@ -499,13 +499,15 @@ public class AMRMClientImpl<T extends Co
for (String location : locations) {
TreeMap<Resource, ResourceRequestInfo> reqs =
remoteRequests.get(location);
- if (reqs != null && !reqs.isEmpty()
- && reqs.values().iterator().next().remoteRequest.getRelaxLocality()
- != relaxLocality) {
- throw new InvalidContainerRequestException("Cannot submit a "
- + "ContainerRequest asking for location " + location
- + " with locality relaxation " + relaxLocality + " when it has "
- + "already been requested with locality relaxation " +
relaxLocality);
+ if (reqs != null && !reqs.isEmpty()) {
+ boolean existingRelaxLocality =
+ reqs.values().iterator().next().remoteRequest.getRelaxLocality();
+ if (relaxLocality != existingRelaxLocality) {
+ throw new InvalidContainerRequestException("Cannot submit a "
+ + "ContainerRequest asking for location " + location
+ + " with locality relaxation " + relaxLocality + " when it has
"
+ + "already been requested with locality relaxation " +
existingRelaxLocality);
+ }
}
}
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
Sun Nov 3 17:51:52 2013
@@ -24,7 +24,6 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
@@ -54,7 +53,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -84,9 +82,7 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
-import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
@@ -98,7 +94,6 @@ import org.apache.hadoop.yarn.state.Mult
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
-import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@SuppressWarnings({"unchecked", "rawtypes"})
@@ -673,38 +668,8 @@ public class RMAppAttemptImpl implements
@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
this.readLock.lock();
-
try {
- int numUsedContainers = 0;
- int numReservedContainers = 0;
- Resource currentConsumption = Resources.createResource(0, 0);
- Resource reservedResources = Resources.createResource(0, 0);
-
- SchedulerAppReport schedApp =
- scheduler.getSchedulerAppInfo(this.getAppAttemptId());
- Collection<RMContainer> liveContainers;
- Collection<RMContainer> reservedContainers;
- if (schedApp != null) {
- liveContainers = schedApp.getLiveContainers();
- reservedContainers = schedApp.getReservedContainers();
- if (liveContainers != null) {
- numUsedContainers = liveContainers.size();
- for (RMContainer lc : liveContainers) {
- Resources.addTo(currentConsumption,
lc.getContainer().getResource());
- }
- }
- if (reservedContainers != null) {
- numReservedContainers = reservedContainers.size();
- for (RMContainer rc : reservedContainers) {
- Resources.addTo(reservedResources,
rc.getContainer().getResource());
- }
- }
- }
-
- return BuilderUtils.newApplicationResourceUsageReport(
- numUsedContainers, numReservedContainers,
- currentConsumption, reservedResources,
- Resources.add(currentConsumption, reservedResources));
+ return scheduler.getAppResourceUsageReport(this.getAppAttemptId());
} finally {
this.readLock.unlock();
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
Sun Nov 3 17:51:52 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -397,5 +398,12 @@ public abstract class SchedulerApplicati
lastScheduledContainer.put(priority, currentTimeMs);
schedulingOpportunities.setCount(priority, 0);
}
+
+ public synchronized ApplicationResourceUsageReport getResourceUsageReport() {
+ return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
+ reservedContainers.size(), Resources.clone(currentConsumption),
+ Resources.clone(currentReservation),
+ Resources.add(currentConsumption, currentReservation));
+ }
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
Sun Nov 3 17:51:52 2013
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -130,6 +131,16 @@ public interface YarnScheduler extends E
SchedulerAppReport getSchedulerAppInfo(ApplicationAttemptId appAttemptId);
/**
+ * Get a resource usage report from a given app attempt ID.
+ * @param appAttemptId the id of the application attempt
+ * @return resource usage report for this given attempt
+ */
+ @LimitedPrivate("yarn")
+ @Evolving
+ ApplicationResourceUsageReport getAppResourceUsageReport(
+ ApplicationAttemptId appAttemptId);
+
+ /**
* Get the root queue for the scheduler.
* @return the root queue for the scheduler.
*/
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
Sun Nov 3 17:51:52 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -856,6 +857,13 @@ public class CapacityScheduler
return app == null ? null : new SchedulerAppReport(app);
}
+ @Override
+ public ApplicationResourceUsageReport getAppResourceUsageReport(
+ ApplicationAttemptId applicationAttemptId) {
+ FiCaSchedulerApp app = getApplication(applicationAttemptId);
+ return app == null ? null : app.getResourceUsageReport();
+ }
+
@Lock(Lock.NoLock.class)
FiCaSchedulerNode getNode(NodeId nodeId) {
return nodes.get(nodeId);
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
Sun Nov 3 17:51:52 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -181,6 +182,8 @@ public class FairScheduler implements Re
protected WeightAdjuster weightAdjuster; // Can be null for no weight
adjuster
protected boolean continuousSchedulingEnabled; // Continuous Scheduling
enabled or not
protected int continuousSchedulingSleepMs; // Sleep time for each pass in
continuous scheduling
+ private Comparator nodeAvailableResourceComparator =
+ new NodeAvailableResourceComparator(); // Node available resource
comparator
protected double nodeLocalityThreshold; // Cluster threshold for node
locality
protected double rackLocalityThreshold; // Cluster threshold for rack
locality
protected long nodeLocalityDelayMs; // Delay for node locality
@@ -948,14 +951,22 @@ public class FairScheduler implements Re
private void continuousScheduling() {
while (true) {
- for (FSSchedulerNode node : nodes.values()) {
- try {
- if (Resources.fitsIn(minimumAllocation,
node.getAvailableResource())) {
- attemptScheduling(node);
+ List<NodeId> nodeIdList = new ArrayList<NodeId>(nodes.keySet());
+ Collections.sort(nodeIdList, nodeAvailableResourceComparator);
+
+ // iterate all nodes
+ for (NodeId nodeId : nodeIdList) {
+ if (nodes.containsKey(nodeId)) {
+ FSSchedulerNode node = nodes.get(nodeId);
+ try {
+ if (Resources.fitsIn(minimumAllocation,
+ node.getAvailableResource())) {
+ attemptScheduling(node);
+ }
+ } catch (Throwable ex) {
+ LOG.warn("Error while attempting scheduling for node " + node +
+ ": " + ex.toString(), ex);
}
- } catch (Throwable ex) {
- LOG.warn("Error while attempting scheduling for node " + node + ": "
+
- ex.toString(), ex);
}
}
try {
@@ -966,6 +977,17 @@ public class FairScheduler implements Re
}
}
}
+
+ /** Sort nodes by available resource */
+ private class NodeAvailableResourceComparator implements Comparator<NodeId> {
+
+ @Override
+ public int compare(NodeId n1, NodeId n2) {
+ return RESOURCE_CALCULATOR.compare(clusterCapacity,
+ nodes.get(n2).getAvailableResource(),
+ nodes.get(n1).getAvailableResource());
+ }
+ }
private synchronized void attemptScheduling(FSSchedulerNode node) {
// Assign new containers...
@@ -1030,6 +1052,17 @@ public class FairScheduler implements Re
return new SchedulerAppReport(applications.get(appAttemptId));
}
+ @Override
+ public ApplicationResourceUsageReport getAppResourceUsageReport(
+ ApplicationAttemptId appAttemptId) {
+ FSSchedulerApp app = applications.get(appAttemptId);
+ if (app == null) {
+ LOG.error("Request for appInfo of unknown attempt" + appAttemptId);
+ return null;
+ }
+ return app.getResourceUsageReport();
+ }
+
/**
* Subqueue metrics might be a little out of date because fair shares are
* recalculated at the update interval, but the root queue metrics needs to
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
Sun Nov 3 17:51:52 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -327,6 +328,13 @@ public class FifoScheduler implements Re
return app == null ? null : new SchedulerAppReport(app);
}
+ @Override
+ public ApplicationResourceUsageReport getAppResourceUsageReport(
+ ApplicationAttemptId applicationAttemptId) {
+ FiCaSchedulerApp app = getApplication(applicationAttemptId);
+ return app == null ? null : app.getResourceUsageReport();
+ }
+
private FiCaSchedulerNode getNode(NodeId nodeId) {
return nodes.get(nodeId);
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
Sun Nov 3 17:51:52 2013
@@ -26,8 +26,10 @@ import javax.xml.bind.annotation.XmlTran
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -71,6 +73,9 @@ public class AppInfo {
protected long elapsedTime;
protected String amContainerLogs;
protected String amHostHttpAddress;
+ protected int allocatedMB;
+ protected int allocatedVCores;
+ protected int runningContainers;
public AppInfo() {
} // JAXB needs this
@@ -132,6 +137,15 @@ public class AppInfo {
this.amContainerLogs = url;
this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
}
+
+ ApplicationResourceUsageReport resourceReport = attempt
+ .getApplicationResourceUsageReport();
+ if (resourceReport != null) {
+ Resource usedResources = resourceReport.getUsedResources();
+ allocatedMB = usedResources.getMemory();
+ allocatedVCores = usedResources.getVirtualCores();
+ runningContainers = resourceReport.getNumUsedContainers();
+ }
}
}
}
@@ -224,5 +238,17 @@ public class AppInfo {
public String getApplicationType() {
return this.applicationType;
}
-
+
+ public int getRunningContainers() {
+ return this.runningContainers;
+ }
+
+ public int getAllocatedMB() {
+ return this.allocatedMB;
+ }
+
+ public int getAllocatedVCores() {
+ return this.allocatedVCores;
+ }
+
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Sun Nov 3 17:51:52 2013
@@ -33,6 +33,9 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -53,6 +56,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import
org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -2348,7 +2352,7 @@ public class TestFairScheduler {
fs.applications, FSSchedulerApp.class);
}
- @Test (timeout = 5000)
+ @Test (timeout = 10000)
public void testContinuousScheduling() throws Exception {
// set continuous scheduling enabled
FairScheduler fs = new FairScheduler();
@@ -2359,16 +2363,21 @@ public class TestFairScheduler {
Assert.assertTrue("Continuous scheduling should be enabled.",
fs.isContinuousSchedulingEnabled());
- // Add one node
+ // Add two nodes
RMNode node1 =
MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
"127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
fs.handle(nodeEvent1);
+ RMNode node2 =
+ MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 2,
+ "127.0.0.2");
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ fs.handle(nodeEvent2);
// available resource
- Assert.assertEquals(fs.getClusterCapacity().getMemory(), 8 * 1024);
- Assert.assertEquals(fs.getClusterCapacity().getVirtualCores(), 8);
+ Assert.assertEquals(fs.getClusterCapacity().getMemory(), 16 * 1024);
+ Assert.assertEquals(fs.getClusterCapacity().getVirtualCores(), 16);
// send application request
ApplicationAttemptId appAttemptId =
@@ -2387,10 +2396,32 @@ public class TestFairScheduler {
FSSchedulerApp app = fs.applications.get(appAttemptId);
// Wait until app gets resources.
while (app.getCurrentConsumption().equals(Resources.none())) { }
-
+
// check consumption
Assert.assertEquals(1024, app.getCurrentConsumption().getMemory());
Assert.assertEquals(1, app.getCurrentConsumption().getVirtualCores());
+
+ // another request
+ request =
+ createResourceRequest(1024, 1, ResourceRequest.ANY, 2, 1, true);
+ ask.clear();
+ ask.add(request);
+ fs.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);
+
+ // Wait until app gets resources
+ while (app.getCurrentConsumption()
+ .equals(Resources.createResource(1024, 1))) { }
+
+ Assert.assertEquals(2048, app.getCurrentConsumption().getMemory());
+ Assert.assertEquals(2, app.getCurrentConsumption().getVirtualCores());
+
+ // 2 containers should be assigned to 2 nodes
+ Set<NodeId> nodes = new HashSet<NodeId>();
+ Iterator<RMContainer> it = app.getLiveContainers().iterator();
+ while (it.hasNext()) {
+ nodes.add(it.next().getContainer().getNodeId());
+ }
+ Assert.assertEquals(2, nodes.size());
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
Sun Nov 3 17:51:52 2013
@@ -78,6 +78,8 @@ import com.sun.jersey.test.framework.Web
public class TestRMWebServicesApps extends JerseyTest {
private static MockRM rm;
+
+ private static final int CONTAINER_MB = 1024;
private Injector injector = Guice.createInjector(new ServletModule() {
@Override
@@ -126,7 +128,7 @@ public class TestRMWebServicesApps exten
public void testApps() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
testAppsHelper("apps", app1, MediaType.APPLICATION_JSON);
rm.stop();
@@ -136,7 +138,7 @@ public class TestRMWebServicesApps exten
public void testAppsSlash() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
testAppsHelper("apps/", app1, MediaType.APPLICATION_JSON);
rm.stop();
@@ -146,7 +148,7 @@ public class TestRMWebServicesApps exten
public void testAppsDefault() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
testAppsHelper("apps/", app1, "");
rm.stop();
@@ -156,7 +158,7 @@ public class TestRMWebServicesApps exten
public void testAppsXML() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024, "testwordcount", "user1");
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
@@ -181,7 +183,7 @@ public class TestRMWebServicesApps exten
public void testAppsXMLMulti() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024, "testwordcount", "user1");
+ rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
rm.submitApp(2048, "testwordcount2", "user1");
amNodeManager.nodeHeartbeat(true);
@@ -225,7 +227,7 @@ public class TestRMWebServicesApps exten
public void testAppsQueryState() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -248,8 +250,8 @@ public class TestRMWebServicesApps exten
public void testAppsQueryStates() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
- RMApp killedApp = rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
+ RMApp killedApp = rm.submitApp(CONTAINER_MB);
rm.killApp(killedApp.getApplicationId());
amNodeManager.nodeHeartbeat(true);
@@ -297,8 +299,8 @@ public class TestRMWebServicesApps exten
public void testAppsQueryStatesComma() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
- RMApp killedApp = rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
+ RMApp killedApp = rm.submitApp(CONTAINER_MB);
rm.killApp(killedApp.getApplicationId());
amNodeManager.nodeHeartbeat(true);
@@ -346,7 +348,7 @@ public class TestRMWebServicesApps exten
public void testAppsQueryStatesNone() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -365,7 +367,7 @@ public class TestRMWebServicesApps exten
public void testAppsQueryStateNone() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -384,7 +386,7 @@ public class TestRMWebServicesApps exten
public void testAppsQueryStatesInvalid() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -421,7 +423,7 @@ public class TestRMWebServicesApps exten
public void testAppsQueryStateInvalid() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -458,7 +460,7 @@ public class TestRMWebServicesApps exten
public void testAppsQueryFinalStatus() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -481,7 +483,7 @@ public class TestRMWebServicesApps exten
public void testAppsQueryFinalStatusNone() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -499,7 +501,7 @@ public class TestRMWebServicesApps exten
public void testAppsQueryFinalStatusInvalid() throws JSONException,
Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -537,8 +539,8 @@ public class TestRMWebServicesApps exten
public void testAppsQueryUser() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -565,8 +567,8 @@ public class TestRMWebServicesApps exten
public void testAppsQueryQueue() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -588,9 +590,9 @@ public class TestRMWebServicesApps exten
public void testAppsQueryLimit() throws JSONException, Exception {
rm.start();
rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
- rm.submitApp(1024);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParam("limit", "2")
@@ -611,9 +613,9 @@ public class TestRMWebServicesApps exten
long start = System.currentTimeMillis();
Thread.sleep(1);
rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
- rm.submitApp(1024);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParam("startedTimeBegin", String.valueOf(start))
@@ -632,11 +634,11 @@ public class TestRMWebServicesApps exten
public void testAppsQueryStartBeginSome() throws JSONException, Exception {
rm.start();
rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
long start = System.currentTimeMillis();
Thread.sleep(1);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParam("startedTimeBegin", String.valueOf(start))
@@ -657,9 +659,9 @@ public class TestRMWebServicesApps exten
rm.registerNode("127.0.0.1:1234", 2048);
long end = System.currentTimeMillis();
Thread.sleep(1);
- rm.submitApp(1024);
- rm.submitApp(1024);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParam("startedTimeEnd", String.valueOf(end))
@@ -677,11 +679,11 @@ public class TestRMWebServicesApps exten
rm.registerNode("127.0.0.1:1234", 2048);
long start = System.currentTimeMillis();
Thread.sleep(1);
- rm.submitApp(1024);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
long end = System.currentTimeMillis();
Thread.sleep(1);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParam("startedTimeBegin", String.valueOf(start))
@@ -703,7 +705,7 @@ public class TestRMWebServicesApps exten
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
long start = System.currentTimeMillis();
Thread.sleep(1);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
// finish App
MockAM am = rm
@@ -712,8 +714,8 @@ public class TestRMWebServicesApps exten
am.unregisterAppAttempt();
amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
1, ContainerState.COMPLETE);
- rm.submitApp(1024);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
@@ -733,7 +735,7 @@ public class TestRMWebServicesApps exten
public void testAppsQueryFinishEnd() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
// finish App
MockAM am = rm
@@ -743,8 +745,8 @@ public class TestRMWebServicesApps exten
amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
1, ContainerState.COMPLETE);
- rm.submitApp(1024);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
long end = System.currentTimeMillis();
WebResource r = resource();
@@ -767,7 +769,7 @@ public class TestRMWebServicesApps exten
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
long start = System.currentTimeMillis();
Thread.sleep(1);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
// finish App
MockAM am = rm
@@ -777,8 +779,8 @@ public class TestRMWebServicesApps exten
amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
1, ContainerState.COMPLETE);
- rm.submitApp(1024);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
+ rm.submitApp(CONTAINER_MB);
long end = System.currentTimeMillis();
WebResource r = resource();
@@ -801,7 +803,7 @@ public class TestRMWebServicesApps exten
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
Thread.sleep(1);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
// finish App
MockAM am = rm
@@ -811,9 +813,9 @@ public class TestRMWebServicesApps exten
amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
1, ContainerState.COMPLETE);
- rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+ rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser()
.getShortUserName(), null, false, null, 2, null, "MAPREDUCE");
- rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+ rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser()
.getShortUserName(), null, false, null, 2, null, "NON-YARN");
WebResource r = resource();
@@ -987,7 +989,7 @@ public class TestRMWebServicesApps exten
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 4096);
Thread.sleep(1);
- RMApp app1 = rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "",
UserGroupInformation.getCurrentUser()
.getShortUserName(), null, false, null, 2, null, "MAPREDUCE");
amNodeManager.nodeHeartbeat(true);
// finish App
@@ -998,9 +1000,9 @@ public class TestRMWebServicesApps exten
amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
1, ContainerState.COMPLETE);
- rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+ rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser()
.getShortUserName(), null, false, null, 2, null, "MAPREDUCE");
- rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+ rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser()
.getShortUserName(), null, false, null, 2, null, "OTHER");
// zero type, zero state
@@ -1148,7 +1150,7 @@ public class TestRMWebServicesApps exten
public void testSingleApp() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024, "testwordcount", "user1");
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
amNodeManager.nodeHeartbeat(true);
testSingleAppsHelper(app1.getApplicationId().toString(), app1,
MediaType.APPLICATION_JSON);
@@ -1159,7 +1161,7 @@ public class TestRMWebServicesApps exten
public void testSingleAppsSlash() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
testSingleAppsHelper(app1.getApplicationId().toString() + "/", app1,
MediaType.APPLICATION_JSON);
@@ -1170,7 +1172,7 @@ public class TestRMWebServicesApps exten
public void testSingleAppsDefault() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
testSingleAppsHelper(app1.getApplicationId().toString() + "/", app1, "");
rm.stop();
@@ -1180,7 +1182,7 @@ public class TestRMWebServicesApps exten
public void testInvalidApp() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -1216,7 +1218,7 @@ public class TestRMWebServicesApps exten
public void testNonexistApp() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024, "testwordcount", "user1");
+ rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -1265,7 +1267,7 @@ public class TestRMWebServicesApps exten
public void testSingleAppsXML() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024, "testwordcount", "user1");
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
@@ -1307,7 +1309,10 @@ public class TestRMWebServicesApps exten
WebServicesTestUtils.getXmlLong(element, "finishedTime"),
WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
WebServicesTestUtils.getXmlString(element, "amHostHttpAddress"),
- WebServicesTestUtils.getXmlString(element, "amContainerLogs"));
+ WebServicesTestUtils.getXmlString(element, "amContainerLogs"),
+ WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
+ WebServicesTestUtils.getXmlInt(element, "allocatedVCores"),
+ WebServicesTestUtils.getXmlInt(element, "runningContainers"));
}
}
@@ -1315,7 +1320,7 @@ public class TestRMWebServicesApps exten
Exception {
// 15 because trackingUrl not assigned yet
- assertEquals("incorrect number of elements", 16, info.length());
+ assertEquals("incorrect number of elements", 19, info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
info.getString("name"), info.getString("applicationType"),
info.getString("queue"),
@@ -1324,14 +1329,16 @@ public class TestRMWebServicesApps exten
info.getString("diagnostics"), info.getLong("clusterId"),
info.getLong("startedTime"), info.getLong("finishedTime"),
info.getLong("elapsedTime"), info.getString("amHostHttpAddress"),
- info.getString("amContainerLogs"));
+ info.getString("amContainerLogs"), info.getInt("allocatedMB"),
+ info.getInt("allocatedVCores"), info.getInt("runningContainers"));
}
public void verifyAppInfoGeneric(RMApp app, String id, String user,
String name, String applicationType, String queue, String state, String
finalStatus,
float progress, String trackingUI, String diagnostics, long clusterId,
long startedTime, long finishedTime, long elapsedTime,
- String amHostHttpAddress, String amContainerLogs) throws JSONException,
+ String amHostHttpAddress, String amContainerLogs, int allocatedMB,
+ int allocatedVCores, int numContainers) throws JSONException,
Exception {
WebServicesTestUtils.checkStringMatch("id", app.getApplicationId()
@@ -1363,13 +1370,16 @@ public class TestRMWebServicesApps exten
amContainerLogs.startsWith("http://";));
assertTrue("amContainerLogs doesn't contain user info",
amContainerLogs.endsWith("/" + app.getUser()));
+ assertEquals("allocatedMB doesn't match", 1024, allocatedMB);
+ assertEquals("allocatedVCores doesn't match", 1, allocatedVCores);
+ assertEquals("numContainers doesn't match", 1, numContainers);
}
@Test
public void testAppAttempts() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024, "testwordcount", "user1");
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
amNodeManager.nodeHeartbeat(true);
testAppAttemptsHelper(app1.getApplicationId().toString(), app1,
MediaType.APPLICATION_JSON);
@@ -1380,7 +1390,7 @@ public class TestRMWebServicesApps exten
public void testMultipleAppAttempts() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024, "testwordcount", "user1");
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
amNodeManager.nodeHeartbeat(true);
int maxAppAttempts = rm.getConfig().getInt(
YarnConfiguration.RM_AM_MAX_ATTEMPTS,
@@ -1406,7 +1416,7 @@ public class TestRMWebServicesApps exten
public void testAppAttemptsSlash() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1,
MediaType.APPLICATION_JSON);
@@ -1417,7 +1427,7 @@ public class TestRMWebServicesApps exten
public void testAppAttemtpsDefault() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, "");
rm.stop();
@@ -1427,7 +1437,7 @@ public class TestRMWebServicesApps exten
public void testInvalidAppAttempts() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024);
+ rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -1463,7 +1473,7 @@ public class TestRMWebServicesApps exten
public void testNonexistAppAttempts() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(1024, "testwordcount", "user1");
+ rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
@@ -1526,7 +1536,7 @@ public class TestRMWebServicesApps exten
rm.start();
String user = "user1";
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(1024, "testwordcount", user);
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", user);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1538408&r1=1538407&r2=1538408&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Sun Nov 3 17:51:52 2013
@@ -1177,7 +1177,10 @@ ResourceManager REST API's.
"elapsedTime" : 25196,
"diagnostics" : "",
"trackingUrl" :
"http://host.domain.com:8088/proxy/application_1326815542473_0001/jobhistory/job/job_1326815542473_1_1";,
- "queue" : "default"
+ "queue" : "default",
+ "allocatedMB" : 0,
+ "allocatedVCores" : 0,
+ "runningContainers" : 0
},
{
"finishedTime" : 1326815789546,
@@ -1195,7 +1198,10 @@ ResourceManager REST API's.
"elapsedTime" : 148166,
"diagnostics" : "",
"trackingUrl" :
"http://host.domain.com:8088/proxy/application_1326815542473_0002/jobhistory/job/job_1326815542473_2_2";,
- "queue" : "default"
+ "queue" : "default",
+ "allocatedMB" : 0,
+ "allocatedVCores" : 0,
+ "runningContainers" : 1
}
]
}
@@ -1245,6 +1251,9 @@ ResourceManager REST API's.
<amContainerLogs>http://host.domain.com:8042/node/containerlogs/container_1326815542473_0001
_01_000001</amContainerLogs>
<amHostHttpAddress>host.domain.com:8042</amHostHttpAddress>
+ <allocatedMB>0</allocatedMB>
+ <allocatedVCores>0</allocatedVCores>
+ <runningContainers>0</runningContainers>
</app>
<app>
<id>application_1326815542473_0002</id>
@@ -1264,6 +1273,9 @@ _01_000001</amContainerLogs>
<elapsedTime>148166</elapsedTime>
<amContainerLogs>http://host.domain.com:8042/node/containerlogs/container_1326815542473_0002_01_000001</amContainerLogs>
<amHostHttpAddress>host.domain.com:8042</amHostHttpAddress>
+ <allocatedMB>0</allocatedMB>
+ <allocatedVCores>0</allocatedVCores>
+ <runningContainers>0</runningContainers>
</app>
</apps>
@@ -1457,6 +1469,12 @@ _01_000001</amContainerLogs>
*---------------+--------------+--------------------------------+
| amHostHttpAddress | string | The nodes http address of the application
master |
*---------------+--------------+--------------------------------+
+| allocatedMB | int | The sum of memory in MB allocated to the application's
running containers |
+*---------------------------------------------------------------+
+| allocatedVCores | int | The sum of virtual cores allocated to the
application's running containers |
++---------------------------------------------------------------+
+| runningContainers | int | The number of containers currently running for the
application |
++---------------------------------------------------------------+
** Response Examples
