Author: szetszwo
Date: Wed Jul 23 17:30:06 2014
New Revision: 1612883
URL: http://svn.apache.org/r1612883
Log:
Merge r1609845 through r1612880 from trunk.
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt Wed Jul 23
17:30:06 2014
@@ -59,6 +59,9 @@ Release 2.6.0 - UNRELEASED
YARN-2013. The diagnostics is always the ExitCodeException stack when the
container
crashes. (Tsuyoshi OZAWA via junping_du)
+ YARN-2295. Refactored DistributedShell to use public APIs of protocol
records.
+ (Li Lu via jianhe)
+
OPTIMIZATIONS
BUG FIXES
@@ -85,6 +88,12 @@ Release 2.6.0 - UNRELEASED
YARN-2321. NodeManager web UI can incorrectly report Pmem enforcement
(Leitao Guo via jlowe)
+ YARN-2273. NPE in ContinuousScheduling thread when we lose a node.
+ (Wei Yan via kasha)
+
+ YARN-2313. Livelock can occur in FairScheduler when there are lots of
+ running apps (Tsuyoshi Ozawa via Sandy Ryza)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -415,6 +424,9 @@ Release 2.5.0 - UNRELEASED
YARN-2270. Made TestFSDownload#testDownloadPublicWithStatCache be skipped
when thereâs no ancestor permissions. (Akira Ajisaka via zjshen)
+ YARN-2319. Made the MiniKdc instance start/close before/after the class of
+ TestRMWebServicesDelegationTokens. (Wenwu Peng via zjshen)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
Wed Jul 23 17:30:06 2014
@@ -194,6 +194,12 @@
<Field name="scheduleAsynchronously" />
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+ <!-- Inconsistent sync warning - updateInterval is only initialized once and
never changed -->
+ <Match>
+ <Class
name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"
/>
+ <Field name="updateInterval" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
<!-- Inconsistent sync warning - numRetries is only initialized once and
never changed -->
<Match>
<Class
name="org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore" />
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
Wed Jul 23 17:30:06 2014
@@ -83,6 +83,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
@@ -94,7 +95,6 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.LogManager;
import com.google.common.annotations.VisibleForTesting;
@@ -522,6 +522,8 @@ public class ApplicationMaster {
+ appAttemptID.toString(), e);
}
+ // Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class
+ // are marked as LimitedPrivate
Credentials credentials =
UserGroupInformation.getCurrentUser().getCredentials();
DataOutputBuffer dob = new DataOutputBuffer();
@@ -900,11 +902,6 @@ public class ApplicationMaster {
public void run() {
LOG.info("Setting up container launch container for containerid="
+ container.getId());
- ContainerLaunchContext ctx = Records
- .newRecord(ContainerLaunchContext.class);
-
- // Set the environment
- ctx.setEnvironment(shellEnv);
// Set the local resources
Map<String, LocalResource> localResources = new HashMap<String,
LocalResource>();
@@ -935,16 +932,13 @@ public class ApplicationMaster {
return;
}
- LocalResource shellRsrc = Records.newRecord(LocalResource.class);
- shellRsrc.setType(LocalResourceType.FILE);
- shellRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
+ URL yarnUrl = null;
try {
- shellRsrc.setResource(ConverterUtils.getYarnUrlFromURI(new URI(
- renamedScriptPath.toString())));
+ yarnUrl = ConverterUtils.getYarnUrlFromURI(
+ new URI(renamedScriptPath.toString()));
} catch (URISyntaxException e) {
LOG.error("Error when trying to use shell script path specified"
+ " in env, path=" + renamedScriptPath, e);
-
// A failure scenario on bad input such as invalid shell script path
// We know we cannot continue launching the container
// so we should release it.
@@ -953,13 +947,13 @@ public class ApplicationMaster {
numFailedContainers.incrementAndGet();
return;
}
- shellRsrc.setTimestamp(shellScriptPathTimestamp);
- shellRsrc.setSize(shellScriptPathLen);
+ LocalResource shellRsrc = LocalResource.newInstance(yarnUrl,
+ LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
+ shellScriptPathLen, shellScriptPathTimestamp);
localResources.put(Shell.WINDOWS ? ExecBatScripStringtPath :
ExecShellStringPath, shellRsrc);
shellCommand = Shell.WINDOWS ? windows_command : linux_bash_command;
}
- ctx.setLocalResources(localResources);
// Set the necessary command to execute on the allocated container
Vector<CharSequence> vargs = new Vector<CharSequence>(5);
@@ -986,16 +980,18 @@ public class ApplicationMaster {
List<String> commands = new ArrayList<String>();
commands.add(command.toString());
- ctx.setCommands(commands);
- // Set up tokens for the container too. Today, for normal shell commands,
- // the container in distribute-shell doesn't need any tokens. We are
- // populating them mainly for NodeManagers to be able to download any
- // files in the distributed file-system. The tokens are otherwise also
- // useful in cases, for e.g., when one is running a "hadoop dfs" command
- // inside the distributed shell.
- ctx.setTokens(allTokens.duplicate());
+ // Set up ContainerLaunchContext, setting local resource, environment,
+ // command and token for constructor.
+ // Note for tokens: Set up tokens for the container too. Today, for
normal
+ // shell commands, the container in distribute-shell doesn't need any
+ // tokens. We are populating them mainly for NodeManagers to be able to
+ // download anyfiles in the distributed file-system. The tokens are
+ // otherwise also useful in cases, for e.g., when one is running a
+ // "hadoop dfs" command inside the distributed shell.
+ ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(
+ localResources, shellEnv, commands, null, allTokens.duplicate(), null);
containerListener.addContainer(container.getId(), container);
nmClientAsync.startContainerAsync(container, ctx);
}
@@ -1024,15 +1020,13 @@ public class ApplicationMaster {
// setup requirements for hosts
// using * as any host will do for the distributed shell app
// set the priority for the request
- Priority pri = Records.newRecord(Priority.class);
// TODO - what is the range for priority? how to decide?
- pri.setPriority(requestPriority);
+ Priority pri = Priority.newInstance(requestPriority);
// Set up resource type requirements
// For now, memory and CPU are supported so we set memory and cpu
requirements
- Resource capability = Records.newRecord(Resource.class);
- capability.setMemory(containerMemory);
- capability.setVirtualCores(containerVirtualCores);
+ Resource capability = Resource.newInstance(containerMemory,
+ containerVirtualCores);
ContainerRequest request = new ContainerRequest(capability, null, null,
pri);
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
Wed Jul 23 17:30:06 2014
@@ -456,9 +456,6 @@ public class Client {
appContext.setKeepContainersAcrossApplicationAttempts(keepContainers);
appContext.setApplicationName(appName);
- // Set up the container launch context for the application master
- ContainerLaunchContext amContainer =
Records.newRecord(ContainerLaunchContext.class);
-
// set local resources for the application master
// local files or archives as needed
// In this scenario, the jar file for the application master is part of
the local resources
@@ -508,8 +505,6 @@ public class Client {
addToLocalResources(fs, null, shellArgsPath, appId.toString(),
localResources, StringUtils.join(shellArgs, " "));
}
- // Set local resource info into app master container launch context
- amContainer.setLocalResources(localResources);
// Set the necessary security tokens as needed
//amContainer.setContainerTokens(containerToken);
@@ -550,8 +545,6 @@ public class Client {
env.put("CLASSPATH", classPathEnv.toString());
- amContainer.setEnvironment(env);
-
// Set the necessary command to execute the application master
Vector<CharSequence> vargs = new Vector<CharSequence>(30);
@@ -587,14 +580,15 @@ public class Client {
LOG.info("Completed setting up app master command " + command.toString());
List<String> commands = new ArrayList<String>();
commands.add(command.toString());
- amContainer.setCommands(commands);
+
+ // Set up the container launch context for the application master
+ ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(
+ localResources, env, commands, null, null, null);
// Set up resource type requirements
// For now, both memory and vcores are supported, so we set memory and
// vcores requirements
- Resource capability = Records.newRecord(Resource.class);
- capability.setMemory(amMemory);
- capability.setVirtualCores(amVCores);
+ Resource capability = Resource.newInstance(amMemory, amVCores);
appContext.setResource(capability);
// Service data is a binary blob that can be passed to the application
@@ -603,6 +597,7 @@ public class Client {
// Setup security tokens
if (UserGroupInformation.isSecurityEnabled()) {
+ // Note: Credentials class is marked as LimitedPrivate for HDFS and
MapReduce
Credentials credentials = new Credentials();
String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
if (tokenRenewer == null || tokenRenewer.length() == 0) {
@@ -627,9 +622,8 @@ public class Client {
appContext.setAMContainerSpec(amContainer);
// Set the priority for the application master
- Priority pri = Records.newRecord(Priority.class);
// TODO - what is the range for priority? how to decide?
- pri.setPriority(amPriority);
+ Priority pri = Priority.newInstance(amPriority);
appContext.setPriority(pri);
// Set the queue to which this application is to be submitted in the RM
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
Wed Jul 23 17:30:06 2014
@@ -1294,12 +1294,20 @@ public class RMAppAttemptImpl implements
private String getAMContainerCrashedDiagnostics(
RMAppAttemptContainerFinishedEvent finishEvent) {
ContainerStatus status = finishEvent.getContainerStatus();
- String diagnostics =
- "AM Container for " + finishEvent.getApplicationAttemptId()
- + " exited with " + " exitCode: " + status.getExitStatus() + ". "
- + "Check application tracking page: " + this.getTrackingUrl()
- + " . Then, click on links to logs of each attempt for detailed
output. ";
- return diagnostics;
+ StringBuilder diagnosticsBuilder = new StringBuilder();
+ diagnosticsBuilder.append("AM Container for ").append(
+ finishEvent.getApplicationAttemptId()).append(
+ " exited with ").append(" exitCode: ").append(status.getExitStatus()).
+ append("\n");
+ if (this.getTrackingUrl() != null) {
+ diagnosticsBuilder.append("For more detailed output,").append(
+ " check application tracking page:").append(
+ this.getTrackingUrl()).append(
+ "Then, click on links to logs of each attempt.\n");
+ }
+ diagnosticsBuilder.append("Diagnostics: ").append(status.getDiagnostics())
+ .append("Failing this attempt");
+ return diagnosticsBuilder.toString();
}
private static class FinalTransition extends BaseFinalTransition {
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
Wed Jul 23 17:30:06 2014
@@ -135,7 +135,7 @@ public class FairScheduler extends
public static final Resource CONTAINER_RESERVED =
Resources.createResource(-1);
// How often fair shares are re-calculated (ms)
- protected long UPDATE_INTERVAL = 500;
+ protected long updateInterval;
private final int UPDATE_DEBUG_FREQUENCY = 5;
private int updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY;
@@ -244,13 +244,13 @@ public class FairScheduler extends
/**
* A runnable which calls {@link FairScheduler#update()} every
- * <code>UPDATE_INTERVAL</code> milliseconds.
+ * <code>updateInterval</code> milliseconds.
*/
private class UpdateThread implements Runnable {
public void run() {
while (true) {
try {
- Thread.sleep(UPDATE_INTERVAL);
+ Thread.sleep(updateInterval);
update();
preemptTasksIfNecessary();
} catch (Exception e) {
@@ -970,37 +970,27 @@ public class FairScheduler extends
}
}
- private void continuousScheduling() {
- while (true) {
- List<NodeId> nodeIdList = new ArrayList<NodeId>(nodes.keySet());
- // Sort the nodes by space available on them, so that we offer
- // containers on emptier nodes first, facilitating an even spread. This
- // requires holding the scheduler lock, so that the space available on a
- // node doesn't change during the sort.
- synchronized (this) {
- Collections.sort(nodeIdList, nodeAvailableResourceComparator);
- }
-
- // iterate all nodes
- for (NodeId nodeId : nodeIdList) {
- if (nodes.containsKey(nodeId)) {
- FSSchedulerNode node = getFSSchedulerNode(nodeId);
- try {
- if (Resources.fitsIn(minimumAllocation,
- node.getAvailableResource())) {
- attemptScheduling(node);
- }
- } catch (Throwable ex) {
- LOG.warn("Error while attempting scheduling for node " + node +
- ": " + ex.toString(), ex);
- }
- }
- }
+ void continuousSchedulingAttempt() {
+ List<NodeId> nodeIdList = new ArrayList<NodeId>(nodes.keySet());
+ // Sort the nodes by space available on them, so that we offer
+ // containers on emptier nodes first, facilitating an even spread. This
+ // requires holding the scheduler lock, so that the space available on a
+ // node doesn't change during the sort.
+ synchronized (this) {
+ Collections.sort(nodeIdList, nodeAvailableResourceComparator);
+ }
+
+ // iterate all nodes
+ for (NodeId nodeId : nodeIdList) {
+ FSSchedulerNode node = getFSSchedulerNode(nodeId);
try {
- Thread.sleep(getContinuousSchedulingSleepMs());
- } catch (InterruptedException e) {
- LOG.warn("Error while doing sleep in continuous scheduling: " +
- e.toString(), e);
+ if (node != null && Resources.fitsIn(minimumAllocation,
+ node.getAvailableResource())) {
+ attemptScheduling(node);
+ }
+ } catch (Throwable ex) {
+ LOG.error("Error while attempting scheduling for node " + node +
+ ": " + ex.toString(), ex);
}
}
}
@@ -1010,6 +1000,12 @@ public class FairScheduler extends
@Override
public int compare(NodeId n1, NodeId n2) {
+ if (!nodes.containsKey(n1)) {
+ return 1;
+ }
+ if (!nodes.containsKey(n2)) {
+ return -1;
+ }
return RESOURCE_CALCULATOR.compare(clusterResource,
nodes.get(n2).getAvailableResource(),
nodes.get(n1).getAvailableResource());
@@ -1210,6 +1206,15 @@ public class FairScheduler extends
waitTimeBeforeKill = this.conf.getWaitTimeBeforeKill();
usePortForNodeName = this.conf.getUsePortForNodeName();
+ updateInterval = this.conf.getUpdateInterval();
+ if (updateInterval < 0) {
+ updateInterval = FairSchedulerConfiguration.DEFAULT_UPDATE_INTERVAL_MS;
+ LOG.warn(FairSchedulerConfiguration.UPDATE_INTERVAL_MS
+ + " is invalid, so using default value " +
+ + FairSchedulerConfiguration.DEFAULT_UPDATE_INTERVAL_MS
+ + " ms instead");
+ }
+
rootMetrics = FSQueueMetrics.forQueue("root", null, true, conf);
// This stores per-application scheduling information
this.applications =
@@ -1234,7 +1239,16 @@ public class FairScheduler extends
new Runnable() {
@Override
public void run() {
- continuousScheduling();
+ while (!Thread.currentThread().isInterrupted()) {
+ try {
+ continuousSchedulingAttempt();
+ Thread.sleep(getContinuousSchedulingSleepMs());
+ } catch (InterruptedException e) {
+ LOG.error("Continuous scheduling thread interrupted.
Exiting. ",
+ e);
+ return;
+ }
+ }
}
}
);
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
Wed Jul 23 17:30:06 2014
@@ -123,6 +123,11 @@ public class FairSchedulerConfiguration
protected static final String MAX_ASSIGN = CONF_PREFIX + "max.assign";
protected static final int DEFAULT_MAX_ASSIGN = -1;
+ /** The update interval for calculating resources in FairScheduler .*/
+ public static final String UPDATE_INTERVAL_MS =
+ CONF_PREFIX + "update-interval-ms";
+ public static final int DEFAULT_UPDATE_INTERVAL_MS = 500;
+
public FairSchedulerConfiguration() {
super();
}
@@ -246,6 +251,10 @@ public class FairSchedulerConfiguration
"Error reading resource config", ex);
}
}
+
+ public long getUpdateInterval() {
+ return getLong(UPDATE_INTERVAL_MS, DEFAULT_UPDATE_INTERVAL_MS);
+ }
private static int findResource(String val, String units)
throws AllocationConfigurationException {
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
Wed Jul 23 17:30:06 2014
@@ -823,7 +823,9 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptState());
verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
-
verifyAMCrashAtAllocatedDiagnosticInfo(applicationAttempt.getDiagnostics());
+ boolean shouldCheckURL = (applicationAttempt.getTrackingUrl() != null);
+ verifyAMCrashAtAllocatedDiagnosticInfo(applicationAttempt.getDiagnostics(),
+ exitCode, shouldCheckURL);
}
@Test
@@ -1241,11 +1243,18 @@ public class TestRMAppAttemptTransitions
verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}
- private void verifyAMCrashAtAllocatedDiagnosticInfo(String diagnostics) {
- assertTrue("Diagnostic information does not contain application proxy URL",
- diagnostics.contains(applicationAttempt.getWebProxyBase()));
+ private void verifyAMCrashAtAllocatedDiagnosticInfo(String diagnostics,
+ int exitCode, boolean shouldCheckURL) {
assertTrue("Diagnostic information does not point the logs to the users",
diagnostics.contains("logs"));
+ assertTrue("Diagnostic information does not contain application attempt
id",
+ diagnostics.contains(applicationAttempt.getAppAttemptId().toString()));
+ assertTrue("Diagnostic information does not contain application exit code",
+ diagnostics.contains("exitCode: " + exitCode));
+ if (shouldCheckURL) {
+ assertTrue("Diagnostic information does not contain application proxy
URL",
+ diagnostics.contains(applicationAttempt.getWebProxyBase()));
+ }
}
private void verifyTokenCount(ApplicationAttemptId appAttemptId, int count) {
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Wed Jul 23 17:30:06 2014
@@ -2763,7 +2763,43 @@ public class TestFairScheduler extends F
Assert.assertEquals(2, nodes.size());
}
-
+ @Test
+ public void testContinuousSchedulingWithNodeRemoved() throws Exception {
+ // Disable continuous scheduling, will invoke continuous scheduling once
manually
+ scheduler.init(conf);
+ scheduler.start();
+ Assert.assertTrue("Continuous scheduling should be disabled.",
+ !scheduler.isContinuousSchedulingEnabled());
+
+ // Add two nodes
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+ RMNode node2 =
+ MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 2,
+ "127.0.0.2");
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+ Assert.assertEquals("We should have two alive nodes.",
+ 2, scheduler.getNumClusterNodes());
+
+ // Remove one node
+ NodeRemovedSchedulerEvent removeNode1 = new
NodeRemovedSchedulerEvent(node1);
+ scheduler.handle(removeNode1);
+ Assert.assertEquals("We should only have one alive node.",
+ 1, scheduler.getNumClusterNodes());
+
+ // Invoke the continuous scheduling once
+ try {
+ scheduler.continuousSchedulingAttempt();
+ } catch (Exception e) {
+ fail("Exception happened when doing continuous scheduling. " +
+ e.toString());
+ }
+ }
+
@Test
public void testDontAllowUndeclaredPools() throws Exception{
conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
Wed Jul 23 17:30:06 2014
@@ -94,7 +94,7 @@ public class TestFairSchedulerPreemption
scheduler = (FairScheduler)resourceManager.getResourceScheduler();
scheduler.setClock(clock);
- scheduler.UPDATE_INTERVAL = 60 * 1000;
+ scheduler.updateInterval = 60 * 1000;
}
private void registerNodeAndSubmitApp(
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java
Wed Jul 23 17:30:06 2014
@@ -60,7 +60,9 @@ import org.apache.hadoop.yarn.webapp.Web
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -90,28 +92,14 @@ import com.sun.jersey.test.framework.Web
@RunWith(Parameterized.class)
public class TestRMWebServicesDelegationTokens extends JerseyTest {
- private static final File testRootDir = new File("target",
- TestRMWebServicesDelegationTokens.class.getName() + "-root");
+ private static File testRootDir;
private static File httpSpnegoKeytabFile = new File(
KerberosTestUtils.getKeytabFile());
-
private static String httpSpnegoPrincipal = KerberosTestUtils
.getServerPrincipal();
-
- private static boolean miniKDCStarted = false;
private static MiniKdc testMiniKDC;
- static {
- try {
- testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir);
- } catch (Exception e) {
- assertTrue("Couldn't create MiniKDC", false);
- }
- }
-
private static MockRM rm;
-
private Injector injector;
-
private boolean isKerberosAuth = false;
// Make sure the test uses the published header string
@@ -237,7 +225,6 @@ public class TestRMWebServicesDelegation
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
- setupKDC();
switch (run) {
case 0:
default:
@@ -249,17 +236,14 @@ public class TestRMWebServicesDelegation
}
}
- private void setupKDC() throws Exception {
- if (miniKDCStarted == false) {
- testMiniKDC.start();
- getKdc().createPrincipal(httpSpnegoKeytabFile, "HTTP/localhost",
- "client", "client2", "client3");
- miniKDCStarted = true;
- }
- }
-
- private MiniKdc getKdc() {
- return testMiniKDC;
+ @BeforeClass
+ public static void setupKDC() throws Exception {
+ testRootDir = new File("target",
+ TestRMWebServicesDelegationTokens.class.getName() + "-root");
+ testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir);
+ testMiniKDC.start();
+ testMiniKDC.createPrincipal(httpSpnegoKeytabFile, "HTTP/localhost",
+ "client", "client2", "client3");
}
@Before
@@ -270,6 +254,13 @@ public class TestRMWebServicesDelegation
testRootDir.deleteOnExit();
}
+ @AfterClass
+ public static void shutdownKdc() {
+ if (testMiniKDC != null) {
+ testMiniKDC.stop();
+ }
+ }
+
@After
@Override
public void tearDown() throws Exception {
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Wed Jul 23 17:30:06 2014
@@ -205,6 +205,12 @@ Properties that can be placed in yarn-si
instead. Defaults to true. If a queue placement policy is given in the
allocations file, this property is ignored.
+ * <<<yarn.scheduler.fair.update-interval-ms>>>
+
+ * The interval at which to lock the scheduler and recalculate fair shares,
+ recalculate demand, and check whether anything is due for preemption.
+ Defaults to 500 ms.
+
Allocation file format
The allocation file must be in XML format. The format contains five types of
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
Wed Jul 23 17:30:06 2014
@@ -157,15 +157,16 @@ Usage: yarn [--config confdir] COMMAND
Start the ResourceManager
-------
- Usage: yarn resourcemanager [-format]
+ Usage: yarn resourcemanager [-format-state-store]
-------
*---------------+--------------+
|| COMMAND_OPTIONS || Description |
*---------------+--------------+
-| -format | Formats the RMStateStore. This will clear the RMStateStore and is
-| | useful if past applications are no longer needed. This should be
run
-| | only when the ResourceManager is not running.
+| -format-state-store | Formats the RMStateStore. This will clear the
+| | RMStateStore and is useful if past applications are no
+| | longer needed. This should be run only when the
+| | ResourceManager is not running.
*---------------+--------------+
** nodemanager
