Author: umamahesh
Date: Thu May 15 13:03:22 2014
New Revision: 1594906
URL: http://svn.apache.org/r1594906
Log:
Merge from trunk to HDFS-2006
Added:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestGetApplicationsRequest.java
- copied unchanged from r1594886,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestGetApplicationsRequest.java
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt Thu May 15
13:03:22 2014
@@ -26,6 +26,9 @@ Release 2.5.0 - UNRELEASED
YARN-1864. Fair Scheduler Dynamic Hierarchical User Queues (Ashwin Shankar
via Sandy Ryza)
+ YARN-1362. Distinguish between nodemanager shutdown for decommission vs
shutdown
+ for restart. (Jason Lowe via junping_du)
+
IMPROVEMENTS
YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via
@@ -76,6 +79,12 @@ Release 2.5.0 - UNRELEASED
YARN-1987. Wrapper for leveldb DBIterator to aid in handling database
exceptions.
(Jason Lowe via kasha)
+ YARN-1751. Improve MiniYarnCluster for log aggregation testing (Ming Ma
+ via jlowe)
+
+ YARN-1981. Nodemanager version is not updated when a node reconnects (Jason
+ Lowe via jeagles)
+
OPTIMIZATIONS
BUG FIXES
@@ -102,9 +111,15 @@ Release 2.5.0 - UNRELEASED
YARN-2011. Fix typo and warning in TestLeafQueue (Chen He via junping_du)
- YARN-1975. Fix yarn application CLI to print the scheme of the tracking url
+ YARN-1976. Fix yarn application CLI to print the scheme of the tracking url
of failed/killed applications. (Junping Du via jianhe)
+ YARN-2016. Fix a bug in GetApplicationsRequestPBImpl to add the missed
fields
+ to proto. (Junping Du via jianhe)
+
+ YARN-2042. String shouldn't be compared using == in
+ QueuePlacementRule#NestedUserQueue#getQueueForApp (Chen He via Sandy Ryza)
+
Release 2.4.1 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -203,6 +218,16 @@ Release 2.4.1 - UNRELEASED
YARN-1201. TestAMAuthorization fails with local hostname cannot be
resolved.
(Wangda Tan via junping_du)
+ YARN-1861. Fixed a bug in RM to reset leader-election on fencing that was
+ causing both RMs to be stuck in standby mode when automatic failover is
+ enabled. (Karthik Kambatla and Xuan Gong via vinodkv)
+
+ YARN-1957. Consider the max capacity of the queue when computing the ideal
+ capacity for preemption. (Carlo Curino via cdouglas)
+
+ YARN-1986. In Fifo Scheduler, node heartbeat in between creating app and
+ attempt causes NPE (Hong Zhiguo via Sandy Ryza)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
Thu May 15 13:03:22 2014
@@ -34,6 +34,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.ClientBaseWithFixes;
import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.client.api.YarnClient;
@@ -42,6 +43,9 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
+import org.apache.hadoop.yarn.server.resourcemanager.RMFatalEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.RMFatalEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
import org.junit.After;
import org.junit.Assert;
@@ -169,6 +173,7 @@ public class TestRMFailover extends Clie
verifyConnections();
}
+ @SuppressWarnings("unchecked")
@Test
public void testAutomaticFailover()
throws YarnException, InterruptedException, IOException {
@@ -186,6 +191,25 @@ public class TestRMFailover extends Clie
failover();
verifyConnections();
+
+ // Make the current Active handle an RMFatalEvent,
+ // so it transitions to standby.
+ ResourceManager rm = cluster.getResourceManager(
+ cluster.getActiveRMIndex());
+ RMFatalEvent event =
+ new RMFatalEvent(RMFatalEventType.STATE_STORE_FENCED,
+ "Fake RMFatalEvent");
+ rm.getRMContext().getDispatcher().getEventHandler().handle(event);
+ int maxWaitingAttempts = 2000;
+ while (maxWaitingAttempts-- > 0 ) {
+ if (rm.getRMContext().getHAServiceState() == HAServiceState.STANDBY) {
+ break;
+ }
+ Thread.sleep(1);
+ }
+ Assert.assertFalse("RM didn't transition to Standby ",
+ maxWaitingAttempts == 0);
+ verifyConnections();
}
@Test
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
Thu May 15 13:03:22 2014
@@ -121,6 +121,25 @@ public class GetApplicationsRequestPBImp
if (this.scope != null) {
builder.setScope(ProtoUtils.convertToProtoFormat(scope));
}
+ if (this.start != null) {
+ builder.setStartBegin(start.getMinimumLong());
+ builder.setStartEnd(start.getMaximumLong());
+ }
+
+ if (this.finish != null) {
+ builder.setFinishBegin(start.getMinimumLong());
+ builder.setFinishEnd(start.getMaximumLong());
+ }
+
+ builder.setLimit(limit);
+
+ if (this.users != null && !this.users.isEmpty()) {
+ builder.addAllUsers(this.users);
+ }
+
+ if (this.queues != null && !this.queues.isEmpty()) {
+ builder.addAllQueues(this.queues);
+ }
}
private void addLocalApplicationTypesToProto() {
@@ -326,7 +345,7 @@ public class GetApplicationsRequestPBImp
public LongRange getStartRange() {
if (this.start == null) {
GetApplicationsRequestProtoOrBuilder p = viaProto ? proto: builder;
- if (p.hasStartBegin() || p.hasFinishBegin()) {
+ if (p.hasStartBegin() || p.hasStartEnd()) {
long begin = p.hasStartBegin() ? p.getStartBegin() : 0L;
long end = p.hasStartEnd() ? p.getStartEnd() : Long.MAX_VALUE;
this.start = new LongRange(begin, end);
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
Thu May 15 13:03:22 2014
@@ -173,7 +173,7 @@ public class ApplicationClassLoader exte
}
@VisibleForTesting
- static boolean isSystemClass(String name, List<String> systemClasses) {
+ public static boolean isSystemClass(String name, List<String> systemClasses)
{
if (systemClasses != null) {
String canonicalName = name.replace('/', '.');
while (canonicalName.startsWith(".")) {
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
Thu May 15 13:03:22 2014
@@ -66,4 +66,8 @@ public interface Context {
LocalDirsHandlerService getLocalDirsHandler();
ApplicationACLsManager getApplicationACLsManager();
+
+ boolean getDecommissioned();
+
+ void setDecommissioned(boolean isDecommissioned);
}
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
Thu May 15 13:03:22 2014
@@ -272,7 +272,8 @@ public class NodeManager extends Composi
private WebServer webServer;
private final NodeHealthStatus nodeHealthStatus = RecordFactoryProvider
.getRecordFactory(null).newRecordInstance(NodeHealthStatus.class);
-
+ private boolean isDecommissioned = false;
+
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager,
LocalDirsHandlerService dirsHandler, ApplicationACLsManager
aclsManager) {
@@ -349,6 +350,16 @@ public class NodeManager extends Composi
public ApplicationACLsManager getApplicationACLsManager() {
return aclsManager;
}
+
+ @Override
+ public boolean getDecommissioned() {
+ return isDecommissioned;
+ }
+
+ @Override
+ public void setDecommissioned(boolean isDecommissioned) {
+ this.isDecommissioned = isDecommissioned;
+ }
}
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
Thu May 15 13:03:22 2014
@@ -493,6 +493,7 @@ public class NodeStatusUpdaterImpl exten
+ " hence shutting down.");
LOG.warn("Message from ResourceManager: "
+ response.getDiagnosticsMessage());
+ context.setDecommissioned(true);
dispatcher.getEventHandler().handle(
new NodeManagerEvent(NodeManagerEventType.SHUTDOWN));
break;
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
Thu May 15 13:03:22 2014
@@ -930,6 +930,7 @@ public class TestNodeStatusUpdater {
Thread.sleep(500);
}
Assert.assertFalse(heartBeatID < 1);
+ Assert.assertTrue(nm.getNMContext().getDecommissioned());
// NM takes a while to reach the STOPPED state.
waitCount = 0;
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
Thu May 15 13:03:22 2014
@@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ha.HAServiceProtocol;
@@ -86,6 +87,7 @@ public class AdminService extends Compos
private String rmId;
private boolean autoFailoverEnabled;
+ private EmbeddedElectorService embeddedElector;
private Server server;
private InetSocketAddress masterServiceAddress;
@@ -106,7 +108,8 @@ public class AdminService extends Compos
autoFailoverEnabled = HAUtil.isAutomaticFailoverEnabled(conf);
if (autoFailoverEnabled) {
if (HAUtil.isAutomaticFailoverEmbedded(conf)) {
- addIfService(createEmbeddedElectorService());
+ embeddedElector = createEmbeddedElectorService();
+ addIfService(embeddedElector);
}
}
}
@@ -181,6 +184,13 @@ public class AdminService extends Compos
return new EmbeddedElectorService(rmContext);
}
+ @InterfaceAudience.Private
+ void resetLeaderElection() {
+ if (embeddedElector != null) {
+ embeddedElector.resetLeaderElection();
+ }
+ }
+
private UserGroupInformation checkAccess(String method) throws IOException {
return RMServerUtils.verifyAccess(adminAcl, method, LOG);
}
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
Thu May 15 13:03:22 2014
@@ -194,4 +194,9 @@ public class EmbeddedElectorService exte
}
return true;
}
+
+ public void resetLeaderElection() {
+ elector.quitElection(false);
+ elector.joinElection(localActiveNodeInfo);
+ }
}
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Thu May 15 13:03:22 2014
@@ -664,6 +664,7 @@ public class ResourceManager extends Com
// Transition to standby and reinit active services
LOG.info("Transitioning RM to Standby mode");
rm.transitionToStandby(true);
+ rm.adminService.resetLeaderElection();
return;
} catch (Exception e) {
LOG.fatal("Failed to transition RM to Standby mode.");
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
Thu May 15 13:03:22 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -293,34 +294,31 @@ public class ProportionalCapacityPreempt
// with the total capacity for this set of queues
Resource unassigned = Resources.clone(tot_guarant);
- //assign all cluster resources until no more demand, or no resources are
left
- while (!qAlloc.isEmpty() && Resources.greaterThan(rc, tot_guarant,
- unassigned, Resources.none())) {
- Resource wQassigned = Resource.newInstance(0, 0);
-
- // we compute normalizedGuarantees capacity based on currently active
- // queues
- resetCapacity(rc, unassigned, qAlloc);
-
- // offer for each queue their capacity first and in following invocations
- // their share of over-capacity
- for (Iterator<TempQueue> i = qAlloc.iterator(); i.hasNext();) {
- TempQueue sub = i.next();
- Resource wQavail =
- Resources.multiply(unassigned, sub.normalizedGuarantee);
- Resource wQidle = sub.offer(wQavail, rc, tot_guarant);
- Resource wQdone = Resources.subtract(wQavail, wQidle);
- // if the queue returned a value > 0 it means it is fully satisfied
- // and it is removed from the list of active queues qAlloc
- if (!Resources.greaterThan(rc, tot_guarant,
- wQdone, Resources.none())) {
- i.remove();
- }
- Resources.addTo(wQassigned, wQdone);
+ // group queues based on whether they have non-zero guaranteed capacity
+ Set<TempQueue> nonZeroGuarQueues = new HashSet<TempQueue>();
+ Set<TempQueue> zeroGuarQueues = new HashSet<TempQueue>();
+
+ for (TempQueue q : qAlloc) {
+ if (Resources
+ .greaterThan(rc, tot_guarant, q.guaranteed, Resources.none())) {
+ nonZeroGuarQueues.add(q);
+ } else {
+ zeroGuarQueues.add(q);
}
- Resources.subtractFrom(unassigned, wQassigned);
}
+ // first compute the allocation as a fixpoint based on guaranteed capacity
+ computeFixpointAllocation(rc, tot_guarant, nonZeroGuarQueues, unassigned,
+ false);
+
+ // if any capacity is left unassigned, distributed among zero-guarantee
+ // queues uniformly (i.e., not based on guaranteed capacity, as this is
zero)
+ if (!zeroGuarQueues.isEmpty()
+ && Resources.greaterThan(rc, tot_guarant, unassigned,
Resources.none())) {
+ computeFixpointAllocation(rc, tot_guarant, zeroGuarQueues, unassigned,
+ true);
+ }
+
// based on ideal assignment computed above and current assignment we
derive
// how much preemption is required overall
Resource totPreemptionNeeded = Resource.newInstance(0, 0);
@@ -353,6 +351,46 @@ public class ProportionalCapacityPreempt
}
}
+
+ /**
+ * Given a set of queues compute the fix-point distribution of unassigned
+ * resources among them. As pending request of a queue are exhausted, the
+ * queue is removed from the set and remaining capacity redistributed among
+ * remaining queues. The distribution is weighted based on guaranteed
+ * capacity, unless asked to ignoreGuarantee, in which case resources are
+ * distributed uniformly.
+ */
+ private void computeFixpointAllocation(ResourceCalculator rc,
+ Resource tot_guarant, Collection<TempQueue> qAlloc, Resource unassigned,
+ boolean ignoreGuarantee) {
+ //assign all cluster resources until no more demand, or no resources are
left
+ while (!qAlloc.isEmpty() && Resources.greaterThan(rc, tot_guarant,
+ unassigned, Resources.none())) {
+ Resource wQassigned = Resource.newInstance(0, 0);
+
+ // we compute normalizedGuarantees capacity based on currently active
+ // queues
+ resetCapacity(rc, unassigned, qAlloc, ignoreGuarantee);
+
+ // offer for each queue their capacity first and in following invocations
+ // their share of over-capacity
+ for (Iterator<TempQueue> i = qAlloc.iterator(); i.hasNext();) {
+ TempQueue sub = i.next();
+ Resource wQavail =
+ Resources.multiply(unassigned, sub.normalizedGuarantee);
+ Resource wQidle = sub.offer(wQavail, rc, tot_guarant);
+ Resource wQdone = Resources.subtract(wQavail, wQidle);
+ // if the queue returned a value > 0 it means it is fully satisfied
+ // and it is removed from the list of active queues qAlloc
+ if (!Resources.greaterThan(rc, tot_guarant,
+ wQdone, Resources.none())) {
+ i.remove();
+ }
+ Resources.addTo(wQassigned, wQdone);
+ }
+ Resources.subtractFrom(unassigned, wQassigned);
+ }
+ }
/**
* Computes a normalizedGuaranteed capacity based on active queues
@@ -361,14 +399,21 @@ public class ProportionalCapacityPreempt
* @param queues the list of queues to consider
*/
private void resetCapacity(ResourceCalculator rc, Resource clusterResource,
- List<TempQueue> queues) {
+ Collection<TempQueue> queues, boolean ignoreGuar) {
Resource activeCap = Resource.newInstance(0, 0);
- for (TempQueue q : queues) {
- Resources.addTo(activeCap, q.guaranteed);
- }
- for (TempQueue q : queues) {
- q.normalizedGuarantee = Resources.divide(rc, clusterResource,
- q.guaranteed, activeCap);
+
+ if (ignoreGuar) {
+ for (TempQueue q : queues) {
+ q.normalizedGuarantee = (float) 1.0f / ((float) queues.size());
+ }
+ } else {
+ for (TempQueue q : queues) {
+ Resources.addTo(activeCap, q.guaranteed);
+ }
+ for (TempQueue q : queues) {
+ q.normalizedGuarantee = Resources.divide(rc, clusterResource,
+ q.guaranteed, activeCap);
+ }
}
}
@@ -515,18 +560,25 @@ public class ProportionalCapacityPreempt
private TempQueue cloneQueues(CSQueue root, Resource clusterResources) {
TempQueue ret;
synchronized (root) {
- float absUsed = root.getAbsoluteUsedCapacity();
+ String queueName = root.getQueueName();
+ float absUsed = root.getAbsoluteUsedCapacity();
+ float absCap = root.getAbsoluteCapacity();
+ float absMaxCap = root.getAbsoluteMaximumCapacity();
+
Resource current = Resources.multiply(clusterResources, absUsed);
- Resource guaranteed =
- Resources.multiply(clusterResources, root.getAbsoluteCapacity());
+ Resource guaranteed = Resources.multiply(clusterResources, absCap);
+ Resource maxCapacity = Resources.multiply(clusterResources, absMaxCap);
if (root instanceof LeafQueue) {
LeafQueue l = (LeafQueue) root;
Resource pending = l.getTotalResourcePending();
- ret = new TempQueue(root.getQueueName(), current, pending, guaranteed);
+ ret = new TempQueue(queueName, current, pending, guaranteed,
+ maxCapacity);
+
ret.setLeafQueue(l);
} else {
Resource pending = Resource.newInstance(0, 0);
- ret = new TempQueue(root.getQueueName(), current, pending, guaranteed);
+ ret = new TempQueue(root.getQueueName(), current, pending, guaranteed,
+ maxCapacity);
for (CSQueue c : root.getChildQueues()) {
ret.addChild(cloneQueues(c, clusterResources));
}
@@ -563,6 +615,7 @@ public class ProportionalCapacityPreempt
final Resource current;
final Resource pending;
final Resource guaranteed;
+ final Resource maxCapacity;
Resource idealAssigned;
Resource toBePreempted;
Resource actuallyPreempted;
@@ -573,11 +626,12 @@ public class ProportionalCapacityPreempt
LeafQueue leafQueue;
TempQueue(String queueName, Resource current, Resource pending,
- Resource guaranteed) {
+ Resource guaranteed, Resource maxCapacity) {
this.queueName = queueName;
this.current = current;
this.pending = pending;
this.guaranteed = guaranteed;
+ this.maxCapacity = maxCapacity;
this.idealAssigned = Resource.newInstance(0, 0);
this.actuallyPreempted = Resource.newInstance(0, 0);
this.toBePreempted = Resource.newInstance(0, 0);
@@ -614,12 +668,12 @@ public class ProportionalCapacityPreempt
// the unused ones
Resource offer(Resource avail, ResourceCalculator rc,
Resource clusterResource) {
- // remain = avail - min(avail, current + pending - assigned)
- Resource accepted = Resources.min(rc, clusterResource,
- avail,
- Resources.subtract(
- Resources.add(current, pending),
- idealAssigned));
+ // remain = avail - min(avail, (max - assigned), (current + pending -
assigned))
+ Resource accepted =
+ Resources.min(rc, clusterResource,
+ Resources.subtract(maxCapacity, idealAssigned),
+ Resources.min(rc, clusterResource, avail, Resources.subtract(
+ Resources.add(current, pending), idealAssigned)));
Resource remain = Resources.subtract(avail, accepted);
Resources.addTo(idealAssigned, accepted);
return remain;
@@ -628,13 +682,15 @@ public class ProportionalCapacityPreempt
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
- sb.append("CUR: ").append(current)
+ sb.append(" NAME: " + queueName)
+ .append(" CUR: ").append(current)
.append(" PEN: ").append(pending)
.append(" GAR: ").append(guaranteed)
.append(" NORM: ").append(normalizedGuarantee)
.append(" IDEAL_ASSIGNED: ").append(idealAssigned)
.append(" IDEAL_PREEMPT: ").append(toBePreempted)
- .append(" ACTUAL_PREEMPT: ").append(actuallyPreempted);
+ .append(" ACTUAL_PREEMPT: ").append(actuallyPreempted)
+ .append("\n");
return sb.toString();
}
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
Thu May 15 13:03:22 2014
@@ -491,6 +491,7 @@ public class RMNodeImpl implements RMNod
new NodeRemovedSchedulerEvent(rmNode));
RMNode newNode = ((RMNodeReconnectEvent)event).getReconnectedNode();
+ rmNode.nodeManagerVersion = newNode.getNodeManagerVersion();
if (rmNode.getTotalCapability().equals(newNode.getTotalCapability())
&& rmNode.getHttpPort() == newNode.getHttpPort()) {
// Reset heartbeat ID since node just restarted.
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
Thu May 15 13:03:22 2014
@@ -227,7 +227,7 @@ public abstract class QueuePlacementRule
String queueName = nestedRule.assignAppToQueue(requestedQueue, user,
groups, configuredQueues);
- if (queueName != null && queueName != "") {
+ if (queueName != null && queueName.length() != 0) {
if (!queueName.startsWith("root.")) {
queueName = "root." + queueName;
}
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
Thu May 15 13:03:22 2014
@@ -360,7 +360,8 @@ public class FifoScheduler extends Abstr
return nodes.get(nodeId);
}
- private synchronized void addApplication(ApplicationId applicationId,
+ @VisibleForTesting
+ public synchronized void addApplication(ApplicationId applicationId,
String queue, String user) {
SchedulerApplication application =
new SchedulerApplication(DEFAULT_QUEUE, user);
@@ -372,7 +373,8 @@ public class FifoScheduler extends Abstr
.handle(new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED));
}
- private synchronized void
+ @VisibleForTesting
+ public synchronized void
addApplicationAttempt(ApplicationAttemptId appAttemptId,
boolean transferStateFromPreviousAttempt) {
SchedulerApplication application =
@@ -458,6 +460,9 @@ public class FifoScheduler extends Abstr
.entrySet()) {
FiCaSchedulerApp application =
(FiCaSchedulerApp) e.getValue().getCurrentAppAttempt();
+ if (application == null) {
+ continue;
+ }
LOG.debug("pre-assignContainers");
application.showRequests();
synchronized (application) {
@@ -497,6 +502,9 @@ public class FifoScheduler extends Abstr
for (SchedulerApplication application : applications.values()) {
FiCaSchedulerApp attempt =
(FiCaSchedulerApp) application.getCurrentAppAttempt();
+ if (attempt == null) {
+ continue;
+ }
attempt.setHeadroom(Resources.subtract(clusterResource, usedResource));
}
}
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
Thu May 15 13:03:22 2014
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -66,7 +67,7 @@ public class TestFifoScheduler {
private final int GB = 1024;
private static YarnConfiguration conf;
-
+
@BeforeClass
public static void setup() {
conf = new YarnConfiguration();
@@ -213,6 +214,32 @@ public class TestFifoScheduler {
rm.stop();
}
+ @Test
+ public void testNodeUpdateBeforeAppAttemptInit() throws Exception {
+ FifoScheduler scheduler = new FifoScheduler();
+ MockRM rm = new MockRM(conf);
+ scheduler.reinitialize(conf, rm.getRMContext());
+
+ RMNode node = MockNodes.newNodeInfo(1,
+ Resources.createResource(1024, 4), 1, "127.0.0.1");
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ scheduler.addApplication(appId, "queue1", "user1");
+
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ try {
+ scheduler.handle(updateEvent);
+ } catch (NullPointerException e) {
+ Assert.fail();
+ }
+
+ ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 1);
+ scheduler.addApplicationAttempt(attId, false);
+
+ rm.stop();
+ }
+
private void testMinimumAllocation(YarnConfiguration conf, int testAlloc)
throws Exception {
MockRM rm = new MockRM(conf);
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
Thu May 15 13:03:22 2014
@@ -455,11 +455,15 @@ public class TestRMNodeTransitions {
}
private RMNodeImpl getRunningNode() {
+ return getRunningNode(null);
+ }
+
+ private RMNodeImpl getRunningNode(String nmVersion) {
NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
Resource capability = Resource.newInstance(4096, 4);
RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
null, ResourceOption.newInstance(capability,
- RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT), null);
+ RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT), nmVersion);
node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.STARTED));
Assert.assertEquals(NodeState.RUNNING, node.getState());
return node;
@@ -530,4 +534,14 @@ public class TestRMNodeTransitions {
nodesListManagerEvent.getType());
}
+ @Test
+ public void testReconnnectUpdate() {
+ final String nmVersion1 = "nm version 1";
+ final String nmVersion2 = "nm version 2";
+ RMNodeImpl node = getRunningNode(nmVersion1);
+ Assert.assertEquals(nmVersion1, node.getNodeManagerVersion());
+ RMNodeImpl reconnectingNode = getRunningNode(nmVersion2);
+ node.handle(new RMNodeReconnectEvent(node.getNodeID(), reconnectingNode));
+ Assert.assertEquals(nmVersion2, node.getNodeManagerVersion());
+ }
}
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
Thu May 15 13:03:22 2014
@@ -115,6 +115,7 @@ public class TestProportionalCapacityPre
int[][] qData = new int[][]{
// / A B C
{ 100, 40, 40, 20 }, // abs
+ { 100, 100, 100, 100 }, // maxCap
{ 100, 0, 60, 40 }, // used
{ 0, 0, 0, 0 }, // pending
{ 0, 0, 0, 0 }, // reserved
@@ -133,6 +134,7 @@ public class TestProportionalCapacityPre
int[][] qData = new int[][]{
// / A B C D
{ 100, 10, 40, 20, 30 }, // abs
+ { 100, 100, 100, 100, 100 }, // maxCap
{ 100, 30, 60, 10, 0 }, // used
{ 45, 20, 5, 20, 0 }, // pending
{ 0, 0, 0, 0, 0 }, // reserved
@@ -144,12 +146,33 @@ public class TestProportionalCapacityPre
policy.editSchedule();
verify(mDisp, times(16)).handle(argThat(new IsPreemptionRequestFor(appA)));
}
+
+ @Test
+ public void testMaxCap() {
+ int[][] qData = new int[][]{
+ // / A B C
+ { 100, 40, 40, 20 }, // abs
+ { 100, 100, 45, 100 }, // maxCap
+ { 100, 55, 45, 0 }, // used
+ { 20, 10, 10, 0 }, // pending
+ { 0, 0, 0, 0 }, // reserved
+ { 2, 1, 1, 0 }, // apps
+ { -1, 1, 1, 0 }, // req granularity
+ { 3, 0, 0, 0 }, // subqueues
+ };
+ ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
+ policy.editSchedule();
+ // despite the imbalance, since B is at maxCap, do not correct
+ verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA)));
+ }
+
@Test
public void testPreemptCycle() {
int[][] qData = new int[][]{
// / A B C
{ 100, 40, 40, 20 }, // abs
+ { 100, 100, 100, 100 }, // maxCap
{ 100, 0, 60, 40 }, // used
{ 10, 10, 0, 0 }, // pending
{ 0, 0, 0, 0 }, // reserved
@@ -169,6 +192,7 @@ public class TestProportionalCapacityPre
int[][] qData = new int[][]{
// / A B C
{ 100, 40, 40, 20 }, // abs
+ { 100, 100, 100, 100 }, // maxCap
{ 100, 0, 60, 40 }, // used
{ 10, 10, 0, 0 }, // pending
{ 0, 0, 0, 0 }, // reserved
@@ -205,6 +229,7 @@ public class TestProportionalCapacityPre
int[][] qData = new int[][]{
// / A B C
{ 100, 40, 40, 20 }, // abs
+ { 100, 100, 100, 100 }, // maxCap
{ 100, 39, 43, 21 }, // used
{ 10, 10, 0, 0 }, // pending
{ 0, 0, 0, 0 }, // reserved
@@ -224,6 +249,7 @@ public class TestProportionalCapacityPre
int[][] qData = new int[][]{
// / A B C
{ 100, 40, 40, 20 }, // abs
+ { 100, 100, 100, 100 }, // maxCap
{ 100, 55, 45, 0 }, // used
{ 20, 10, 10, 0 }, // pending
{ 0, 0, 0, 0 }, // reserved
@@ -242,6 +268,7 @@ public class TestProportionalCapacityPre
int[][] qData = new int[][]{
// / A B C
{ 100, 40, 40, 20 }, // abs
+ { 100, 100, 100, 100 }, // maxCap
{ 100, 55, 45, 0 }, // used
{ 20, 10, 10, 0 }, // pending
{ 0, 0, 0, 0 }, // reserved
@@ -261,6 +288,7 @@ public class TestProportionalCapacityPre
int[][] qData = new int[][]{
// / A B C
{ 100, 40, 40, 20 }, // abs
+ { 100, 100, 100, 100 }, // maxCap
{ 100, 90, 10, 0 }, // used
{ 80, 10, 20, 50 }, // pending
{ 0, 0, 0, 0 }, // reserved
@@ -280,6 +308,7 @@ public class TestProportionalCapacityPre
int[][] qData = new int[][] {
// / A B C D E F
{ 200, 100, 50, 50, 100, 10, 90 }, // abs
+ { 200, 200, 200, 200, 200, 200, 200 }, // maxCap
{ 200, 110, 60, 50, 90, 90, 0 }, // used
{ 10, 0, 0, 0, 10, 0, 10 }, // pending
{ 0, 0, 0, 0, 0, 0, 0 }, // reserved
@@ -295,10 +324,54 @@ public class TestProportionalCapacityPre
}
@Test
+ public void testZeroGuar() {
+ int[][] qData = new int[][] {
+ // / A B C D E F
+ { 200, 100, 0, 99, 100, 10, 90 }, // abs
+ { 200, 200, 200, 200, 200, 200, 200 }, // maxCap
+ { 170, 80, 60, 20, 90, 90, 0 }, // used
+ { 10, 0, 0, 0, 10, 0, 10 }, // pending
+ { 0, 0, 0, 0, 0, 0, 0 }, // reserved
+ { 4, 2, 1, 1, 2, 1, 1 }, // apps
+ { -1, -1, 1, 1, -1, 1, 1 }, // req granularity
+ { 2, 2, 0, 0, 2, 0, 0 }, // subqueues
+ };
+ ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
+ policy.editSchedule();
+ // verify capacity taken from A1, not B1 despite B1 being far over
+ // its absolute guaranteed capacity
+ verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA)));
+ }
+
+ @Test
+ public void testZeroGuarOverCap() {
+ int[][] qData = new int[][] {
+ // / A B C D E F
+ { 200, 100, 0, 99, 0, 100, 100 }, // abs
+ { 200, 200, 200, 200, 200, 200, 200 }, // maxCap
+ { 170, 170, 60, 20, 90, 0, 0 }, // used
+ { 85, 50, 30, 10, 10, 20, 20 }, // pending
+ { 0, 0, 0, 0, 0, 0, 0 }, // reserved
+ { 4, 3, 1, 1, 1, 1, 1 }, // apps
+ { -1, -1, 1, 1, 1, -1, 1 }, // req granularity
+ { 2, 3, 0, 0, 0, 1, 0 }, // subqueues
+ };
+ ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
+ policy.editSchedule();
+ // we verify both that C has priority on B and D (has it has >0 guarantees)
+ // and that B and D are force to share their over capacity fairly (as they
+ // are both zero-guarantees) hence D sees some of its containers preempted
+ verify(mDisp, times(14)).handle(argThat(new IsPreemptionRequestFor(appC)));
+ }
+
+
+
+ @Test
public void testHierarchicalLarge() {
int[][] qData = new int[][] {
// / A B C D E F G H I
- { 400, 200, 60,140, 100, 70, 30, 100, 10, 90 }, // abs
+ { 400, 200, 60, 140, 100, 70, 30, 100, 10, 90 }, // abs
+ { 400, 400, 400, 400, 400, 400, 400, 400, 400, 400, }, // maxCap
{ 400, 210, 70,140, 100, 50, 50, 90, 90, 0 }, // used
{ 10, 0, 0, 0, 0, 0, 0, 0, 0, 15 }, // pending
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, // reserved
@@ -382,24 +455,25 @@ public class TestProportionalCapacityPre
when(mCS.getRootQueue()).thenReturn(mRoot);
Resource clusterResources =
- Resource.newInstance(leafAbsCapacities(qData[0], qData[6]), 0);
+ Resource.newInstance(leafAbsCapacities(qData[0], qData[7]), 0);
when(mCS.getClusterResources()).thenReturn(clusterResources);
return policy;
}
ParentQueue buildMockRootQueue(Random r, int[]... queueData) {
int[] abs = queueData[0];
- int[] used = queueData[1];
- int[] pending = queueData[2];
- int[] reserved = queueData[3];
- int[] apps = queueData[4];
- int[] gran = queueData[5];
- int[] queues = queueData[6];
+ int[] maxCap = queueData[1];
+ int[] used = queueData[2];
+ int[] pending = queueData[3];
+ int[] reserved = queueData[4];
+ int[] apps = queueData[5];
+ int[] gran = queueData[6];
+ int[] queues = queueData[7];
- return mockNested(abs, used, pending, reserved, apps, gran, queues);
+ return mockNested(abs, maxCap, used, pending, reserved, apps, gran,
queues);
}
- ParentQueue mockNested(int[] abs, int[] used,
+ ParentQueue mockNested(int[] abs, int[] maxCap, int[] used,
int[] pending, int[] reserved, int[] apps, int[] gran, int[] queues) {
float tot = leafAbsCapacities(abs, queues);
Deque<ParentQueue> pqs = new LinkedList<ParentQueue>();
@@ -407,6 +481,8 @@ public class TestProportionalCapacityPre
when(root.getQueueName()).thenReturn("/");
when(root.getAbsoluteUsedCapacity()).thenReturn(used[0] / tot);
when(root.getAbsoluteCapacity()).thenReturn(abs[0] / tot);
+ when(root.getAbsoluteMaximumCapacity()).thenReturn(maxCap[0] / tot);
+
for (int i = 1; i < queues.length; ++i) {
final CSQueue q;
final ParentQueue p = pqs.removeLast();
@@ -420,6 +496,7 @@ public class TestProportionalCapacityPre
when(q.getQueueName()).thenReturn(queueName);
when(q.getAbsoluteUsedCapacity()).thenReturn(used[i] / tot);
when(q.getAbsoluteCapacity()).thenReturn(abs[i] / tot);
+ when(q.getAbsoluteMaximumCapacity()).thenReturn(maxCap[i] / tot);
}
assert 0 == pqs.size();
return root;
@@ -439,7 +516,7 @@ public class TestProportionalCapacityPre
return pq;
}
- LeafQueue mockLeafQueue(ParentQueue p, float tot, int i, int[] abs,
+ LeafQueue mockLeafQueue(ParentQueue p, float tot, int i, int[] abs,
int[] used, int[] pending, int[] reserved, int[] apps, int[] gran) {
LeafQueue lq = mock(LeafQueue.class);
when(lq.getTotalResourcePending()).thenReturn(
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1594906&r1=1594905&r2=1594906&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
(original)
+++
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Thu May 15 13:03:22 2014
@@ -504,12 +504,6 @@ public class MiniYARNCluster extends Com
String logDirsString = prepareDirs("log", numLogDirs);
config.set(YarnConfiguration.NM_LOG_DIRS, logDirsString);
- File remoteLogDir =
- new File(testWorkDir, MiniYARNCluster.this.getName()
- + "-remoteLogDir-nm-" + index);
- remoteLogDir.mkdir();
- config.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
- remoteLogDir.getAbsolutePath());
// By default AM + 2 containers
config.setInt(YarnConfiguration.NM_PMEM_MB, 4*1024);
config.set(YarnConfiguration.NM_ADDRESS,
@@ -658,12 +652,14 @@ public class MiniYARNCluster extends Com
*/
public boolean waitForNodeManagersToConnect(long timeout)
throws YarnException, InterruptedException {
- ResourceManager rm = getResourceManager();
GetClusterMetricsRequest req = GetClusterMetricsRequest.newInstance();
-
for (int i = 0; i < timeout / 100; i++) {
- if (nodeManagers.length == rm.getClientRMService().getClusterMetrics(req)
- .getClusterMetrics().getNumNodeManagers()) {
+ ResourceManager rm = getResourceManager();
+ if (rm == null) {
+ throw new YarnException("Can not find the active RM.");
+ }
+ else if (nodeManagers.length == rm.getClientRMService()
+ .getClusterMetrics(req).getClusterMetrics().getNumNodeManagers()) {
return true;
}
Thread.sleep(100);
