Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh Wed Aug 20 01:34:29 2014 @@ -13,118 +13,115 @@ # See the License for the specific language governing permissions and # limitations under the License. +## +## THIS FILE ACTS AS AN OVERRIDE FOR hadoop-env.sh FOR ALL +## WORK DONE BY THE yarn AND RELATED COMMANDS. +## +## Precedence rules: +## +## yarn-env.sh > hadoop-env.sh > hard-coded defaults +## +## YARN_xyz > HADOOP_xyz > hard-coded defaults +## + +### +# Generic settings for YARN +### + # User for YARN daemons export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} -# resolve links - $0 may be a softlink -export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}" +# +# By default, YARN will use HADOOP_CONF_DIR. Specify a custom +# YARN_CONF_DIR here +# export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}" +# + +# Override Hadoop's log directory & file +# export YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" +# export YARN_LOGFILE='yarn.log' + +# Need a custom-to-YARN service-level authorization policy file? +# export YARN_POLICYFILE="yarn-policy.xml" -# some Java parameters -# export JAVA_HOME=/home/y/libexec/jdk1.6.0/ -if [ "$JAVA_HOME" != "" ]; then - #echo "run java in $JAVA_HOME" - JAVA_HOME=$JAVA_HOME -fi - -if [ "$JAVA_HOME" = "" ]; then - echo "Error: JAVA_HOME is not set." - exit 1 -fi - -JAVA=$JAVA_HOME/bin/java -JAVA_HEAP_MAX=-Xmx1000m - -# For setting YARN specific HEAP sizes please use this -# Parameter and set appropriately -# YARN_HEAPSIZE=1000 - -# check envvars which might override default args -if [ "$YARN_HEAPSIZE" != "" ]; then - JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m" -fi +#Override the log4j settings for all YARN apps +# export YARN_ROOT_LOGGER="INFO,console" +### # Resource Manager specific parameters +### -# Specify the max Heapsize for the ResourceManager using a numerical value +# Specify the max heapsize for the ResourceManager using a numerical value # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set # the value to 1000. -# This value will be overridden by an Xmx setting specified in either YARN_OPTS -# and/or YARN_RESOURCEMANAGER_OPTS. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS, +# HADOOP_OPTS, and/or YARN_RESOURCEMANAGER_OPTS. # If not specified, the default value will be picked from either YARN_HEAPMAX # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +# #export YARN_RESOURCEMANAGER_HEAPSIZE=1000 -# Specify the max Heapsize for the timeline server using a numerical value -# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set -# the value to 1000. -# This value will be overridden by an Xmx setting specified in either YARN_OPTS -# and/or YARN_TIMELINESERVER_OPTS. -# If not specified, the default value will be picked from either YARN_HEAPMAX -# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. -#export YARN_TIMELINESERVER_HEAPSIZE=1000 - # Specify the JVM options to be used when starting the ResourceManager. # These options will be appended to the options specified as YARN_OPTS # and therefore may override any similar flags set in YARN_OPTS -#export YARN_RESOURCEMANAGER_OPTS= +# +# Examples for a Sun/Oracle JDK: +# a) override the appsummary log file: +# export YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log -Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY" +# +# b) Set JMX options +# export YARN_RESOURCEMANAGER_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026" +# +# c) Set garbage collection logs from hadoop-env.sh +# export YARN_RESOURCE_MANAGER_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" +# +# d) ... or set them directly +# export YARN_RESOURCEMANAGER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" +# +# +# export YARN_RESOURCEMANAGER_OPTS= +### # Node Manager specific parameters +### # Specify the max Heapsize for the NodeManager using a numerical value # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set # the value to 1000. -# This value will be overridden by an Xmx setting specified in either YARN_OPTS -# and/or YARN_NODEMANAGER_OPTS. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS, +# HADOOP_OPTS, and/or YARN_NODEMANAGER_OPTS. # If not specified, the default value will be picked from either YARN_HEAPMAX # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +# #export YARN_NODEMANAGER_HEAPSIZE=1000 # Specify the JVM options to be used when starting the NodeManager. # These options will be appended to the options specified as YARN_OPTS # and therefore may override any similar flags set in YARN_OPTS +# +# See ResourceManager for some examples +# #export YARN_NODEMANAGER_OPTS= -# so that filenames w/ spaces are handled correctly in loops below -IFS= +### +# TimeLineServer specifc parameters +### +# Specify the max Heapsize for the timeline server using a numerical value +# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set +# the value to 1000. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS, +# HADOOP_OPTS, and/or YARN_TIMELINESERVER_OPTS. +# If not specified, the default value will be picked from either YARN_HEAPMAX +# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +# +#export YARN_TIMELINESERVER_HEAPSIZE=1000 -# default log directory & file -if [ "$YARN_LOG_DIR" = "" ]; then - YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" -fi -if [ "$YARN_LOGFILE" = "" ]; then - YARN_LOGFILE='yarn.log' -fi - -# default policy file for service-level authorization -if [ "$YARN_POLICYFILE" = "" ]; then - YARN_POLICYFILE="hadoop-policy.xml" -fi - -# restore ordinary behaviour -unset IFS - -MAC_OSX=false -case "`uname`" in -Darwin*) MAC_OSX=true;; -esac - -if $MAC_OSX; then - YARN_OPTS="$YARN_OPTS -Djava.security.krb5.realm= -Djava.security.krb5.kdc=" -fi - - -YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" -YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" -YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" -YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE" -YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME" -YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING" -YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" -YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" -if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" -fi -YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" - +# Specify the JVM options to be used when starting the TimeLineServer. +# These options will be appended to the options specified as YARN_OPTS +# and therefore may override any similar flags set in YARN_OPTS +# +# See ResourceManager for some examples +# +#export YARN_TIMELINESERVER_OPTS=
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml Wed Aug 20 01:34:29 2014 @@ -200,6 +200,13 @@ <Field name="updateInterval" /> <Bug pattern="IS2_INCONSISTENT_SYNC" /> </Match> + <!-- Inconsistent sync warning - callDurationMetrics is only initialized once and never changed --> + <Match> + <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" /> + <Field name="fsOpDurations" /> + <Bug pattern="IS2_INCONSISTENT_SYNC" /> + </Match> + <!-- Inconsistent sync warning - numRetries is only initialized once and never changed --> <Match> <Class name="org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore" /> Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java Wed Aug 20 01:34:29 2014 @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.PreemptionMessage; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.util.Records; /** @@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.util.Recor * <li>A list of nodes whose status has been updated.</li> * <li>The number of available nodes in a cluster.</li> * <li>A description of resources requested back by the cluster</li> + * <li>AMRMToken, if AMRMToken has been rolled over</li> * </ul> * </p> * @@ -102,6 +104,23 @@ public abstract class AllocateResponse { return response; } + @Private + @Unstable + public static AllocateResponse newInstance(int responseId, + List<ContainerStatus> completedContainers, + List<Container> allocatedContainers, List<NodeReport> updatedNodes, + Resource availResources, AMCommand command, int numClusterNodes, + PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken, + List<ContainerResourceIncrease> increasedContainers, + List<ContainerResourceDecrease> decreasedContainers) { + AllocateResponse response = + newInstance(responseId, completedContainers, allocatedContainers, + updatedNodes, availResources, command, numClusterNodes, preempt, + nmTokens, increasedContainers, decreasedContainers); + response.setAMRMToken(amRMToken); + return response; + } + /** * If the <code>ResourceManager</code> needs the * <code>ApplicationMaster</code> to take some action then it will send an @@ -270,4 +289,17 @@ public abstract class AllocateResponse { @Unstable public abstract void setDecreasedContainers( List<ContainerResourceDecrease> decreasedContainers); + + /** + * The AMRMToken that belong to this attempt + * + * @return The AMRMToken that belong to this attempt + */ + @Public + @Unstable + public abstract Token getAMRMToken(); + + @Private + @Unstable + public abstract void setAMRMToken(Token amRMToken); } Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java Wed Aug 20 01:34:29 2014 @@ -308,6 +308,15 @@ public abstract class GetApplicationsReq /** * Set the range of start times to filter applications on * + * @param range + */ + @Private + @Unstable + public abstract void setStartRange(LongRange range); + + /** + * Set the range of start times to filter applications on + * * @param begin beginning of the range * @param end end of the range * @throws IllegalArgumentException @@ -329,6 +338,15 @@ public abstract class GetApplicationsReq /** * Set the range of finish times to filter applications on * + * @param range + */ + @Private + @Unstable + public abstract void setFinishRange(LongRange range); + + /** + * Set the range of finish times to filter applications on + * * @param begin beginning of the range * @param end end of the range * @throws IllegalArgumentException Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java Wed Aug 20 01:34:29 2014 @@ -31,6 +31,8 @@ public abstract class ResourceOption { int overCommitTimeout){ ResourceOption resourceOption = Records.newRecord(ResourceOption.class); resourceOption.setResource(resource); + resourceOption.setOverCommitTimeout(overCommitTimeout); + resourceOption.build(); return resourceOption; } Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Wed Aug 20 01:34:29 2014 @@ -126,6 +126,10 @@ public class YarnConfiguration extends C public static final String DEFAULT_RM_ADDRESS = "0.0.0.0:" + DEFAULT_RM_PORT; + /** The actual bind address for the RM.*/ + public static final String RM_BIND_HOST = + RM_PREFIX + "bind-host"; + /** The number of threads used to handle applications manager requests.*/ public static final String RM_CLIENT_THREAD_COUNT = RM_PREFIX + "client.thread-count"; @@ -263,6 +267,17 @@ public class YarnConfiguration extends C public static final String RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = RM_PREFIX + "webapp.spnego-keytab-file"; + /** + * Flag to enable override of the default kerberos authentication filter with + * the RM authentication filter to allow authentication using delegation + * tokens(fallback to kerberos if the tokens are missing). Only applicable + * when the http authentication type is kerberos. + */ + public static final String RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = RM_PREFIX + + "webapp.delegation-token-auth-filter.enabled"; + public static final boolean DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = + true; + /** How long to wait until a container is considered dead.*/ public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = RM_PREFIX + "rm.container-allocation.expiry-interval-ms"; @@ -534,6 +549,10 @@ public class YarnConfiguration extends C public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:" + DEFAULT_NM_PORT; + /** The actual bind address or the NM.*/ + public static final String NM_BIND_HOST = + NM_PREFIX + "bind-host"; + /** who will execute(launch) the containers.*/ public static final String NM_CONTAINER_EXECUTOR = NM_PREFIX + "container-executor.class"; @@ -1121,6 +1140,10 @@ public class YarnConfiguration extends C public static final String DEFAULT_TIMELINE_SERVICE_ADDRESS = "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_PORT; + /** The listening endpoint for the timeline service application.*/ + public static final String TIMELINE_SERVICE_BIND_HOST = + TIMELINE_SERVICE_PREFIX + "bind-host"; + /** The number of threads to handle client RPC API requests. */ public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT = TIMELINE_SERVICE_PREFIX + "handler-thread-count"; @@ -1347,7 +1370,7 @@ public class YarnConfiguration extends C public static String getClusterId(Configuration conf) { String clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID); if (clusterId == null) { - throw new HadoopIllegalArgumentException("Configuration doesn't specify" + + throw new HadoopIllegalArgumentException("Configuration doesn't specify " + YarnConfiguration.RM_CLUSTER_ID); } return clusterId; Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto Wed Aug 20 01:34:29 2014 @@ -130,11 +130,6 @@ message ApplicationAttemptStateDataProto optional int32 am_container_exit_status = 9 [default = -1000]; } -message RMStateVersionProto { - optional int32 major_version = 1; - optional int32 minor_version = 2; -} - message EpochProto { optional int64 epoch = 1; } Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Wed Aug 20 01:34:29 2014 @@ -85,6 +85,7 @@ message AllocateResponseProto { repeated NMTokenProto nm_tokens = 9; repeated ContainerResourceIncreaseProto increased_containers = 10; repeated ContainerResourceDecreaseProto decreased_containers = 11; + optional hadoop.common.TokenProto am_rm_token = 12; } ////////////////////////////////////////////////////// Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Wed Aug 20 01:34:29 2014 @@ -208,7 +208,8 @@ public class ApplicationMaster { // App Master configuration // No. of containers to run shell command on - private int numTotalContainers = 1; + @VisibleForTesting + protected int numTotalContainers = 1; // Memory to request for the container on which the shell command will run private int containerMemory = 10; // VirtualCores to request for the container on which the shell command will run @@ -594,8 +595,8 @@ public class ApplicationMaster { List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts(); - LOG.info("Received " + previousAMRunningContainers.size() - + " previous AM's running containers on AM registration."); + LOG.info(appAttemptID + " received " + previousAMRunningContainers.size() + + " previous attempts' running containers on AM registration."); numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); int numTotalContainersToRequest = @@ -610,7 +611,7 @@ public class ApplicationMaster { ContainerRequest containerAsk = setupContainerAskForRM(); amRMClient.addContainerRequest(containerAsk); } - numRequestedContainers.set(numTotalContainersToRequest); + numRequestedContainers.set(numTotalContainers); try { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_END); @@ -689,7 +690,7 @@ public class ApplicationMaster { LOG.info("Got response from RM for container ask, completedCnt=" + completedContainers.size()); for (ContainerStatus containerStatus : completedContainers) { - LOG.info("Got container status for containerID=" + LOG.info(appAttemptID + " got container status for containerID=" + containerStatus.getContainerId() + ", state=" + containerStatus.getState() + ", exitStatus=" + containerStatus.getExitStatus() + ", diagnostics=" @@ -1053,8 +1054,8 @@ public class ApplicationMaster { TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); - entity.addPrimaryFilter("user", UserGroupInformation.getCurrentUser() - .toString()); + entity.addPrimaryFilter("user", + UserGroupInformation.getCurrentUser().getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_START.toString()); @@ -1070,8 +1071,8 @@ public class ApplicationMaster { TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getContainerId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); - entity.addPrimaryFilter("user", UserGroupInformation.getCurrentUser() - .toString()); + entity.addPrimaryFilter("user", + UserGroupInformation.getCurrentUser().getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_END.toString()); @@ -1088,8 +1089,8 @@ public class ApplicationMaster { TimelineEntity entity = new TimelineEntity(); entity.setEntityId(appAttemptId); entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString()); - entity.addPrimaryFilter("user", UserGroupInformation.getCurrentUser() - .toString()); + entity.addPrimaryFilter("user", + UserGroupInformation.getCurrentUser().getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setEventType(appEvent.toString()); event.setTimestamp(System.currentTimeMillis()); Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java Wed Aug 20 01:34:29 2014 @@ -36,9 +36,11 @@ public class TestDSFailedAppMaster exten if (appAttemptID.getAttemptId() == 2) { // should reuse the earlier running container, so numAllocatedContainers // should be set to 1. And should ask no more containers, so - // numRequestedContainers should be set to 0. + // numRequestedContainers should be the same as numTotalContainers. + // The only container is the container requested by the AM in the first + // attempt. if (numAllocatedContainers.get() != 1 - || numRequestedContainers.get() != 0) { + || numRequestedContainers.get() != numTotalContainers) { LOG.info("NumAllocatedContainers is " + numAllocatedContainers.get() + " and NumRequestedContainers is " + numAllocatedContainers.get() + ".Application Master failed. exiting"); Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java Wed Aug 20 01:34:29 2014 @@ -26,13 +26,13 @@ import java.io.FileReader; import java.io.IOException; import java.io.OutputStream; import java.io.PrintWriter; +import java.net.InetAddress; import java.net.URL; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -169,7 +169,9 @@ public class TestDistributedShell { yarnClient.init(new Configuration(yarnCluster.getConfig())); yarnClient.start(); String hostName = NetUtils.getHostname(); + boolean verified = false; + String errorMessage = ""; while(!verified) { List<ApplicationReport> apps = yarnClient.getApplications(); if (apps.size() == 0 ) { @@ -177,15 +179,22 @@ public class TestDistributedShell { continue; } ApplicationReport appReport = apps.get(0); - if (appReport.getHost().startsWith(hostName) - && appReport.getRpcPort() == -1) { + if(appReport.getHost().equals("N/A")) { + Thread.sleep(10); + continue; + } + errorMessage = + "Expected host name to start with '" + hostName + "', was '" + + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + + appReport.getRpcPort() + "'."; + if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) { verified = true; } if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) { break; } } - Assert.assertTrue(verified); + Assert.assertTrue(errorMessage, verified); t.join(); LOG.info("Client run completed. Result=" + result); Assert.assertTrue(result.get()); @@ -212,6 +221,64 @@ public class TestDistributedShell { .toString(), ApplicationMaster.DSEntity.DS_CONTAINER.toString()); } + /* + * NetUtils.getHostname() returns a string in the form "hostname/ip". + * Sometimes the hostname we get is the FQDN and sometimes the short name. In + * addition, on machines with multiple network interfaces, it runs any one of + * the ips. The function below compares the returns values for + * NetUtils.getHostname() accounting for the conditions mentioned. + */ + private boolean checkHostname(String appHostname) throws Exception { + + String hostname = NetUtils.getHostname(); + if (hostname.equals(appHostname)) { + return true; + } + + Assert.assertTrue("Unknown format for hostname " + appHostname, + appHostname.contains("/")); + Assert.assertTrue("Unknown format for hostname " + hostname, + hostname.contains("/")); + + String[] appHostnameParts = appHostname.split("/"); + String[] hostnameParts = hostname.split("/"); + + return (compareFQDNs(appHostnameParts[0], hostnameParts[0]) && checkIPs( + hostnameParts[0], hostnameParts[1], appHostnameParts[1])); + } + + private boolean compareFQDNs(String appHostname, String hostname) + throws Exception { + if (appHostname.equals(hostname)) { + return true; + } + String appFQDN = InetAddress.getByName(appHostname).getCanonicalHostName(); + String localFQDN = InetAddress.getByName(hostname).getCanonicalHostName(); + return appFQDN.equals(localFQDN); + } + + private boolean checkIPs(String hostname, String localIP, String appIP) + throws Exception { + + if (localIP.equals(appIP)) { + return true; + } + boolean appIPCheck = false; + boolean localIPCheck = false; + InetAddress[] addresses = InetAddress.getAllByName(hostname); + for (InetAddress ia : addresses) { + if (ia.getHostAddress().equals(appIP)) { + appIPCheck = true; + continue; + } + if (ia.getHostAddress().equals(localIP)) { + localIPCheck = true; + } + } + return (appIPCheck && localIPCheck); + + } + @Test(timeout=90000) public void testDSRestartWithPreviousRunningContainers() throws Exception { String[] args = { Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml Wed Aug 20 01:34:29 2014 @@ -57,14 +57,6 @@ <groupId>log4j</groupId> <artifactId>log4j</artifactId> </dependency> - <dependency> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty-util</artifactId> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-client</artifactId> - </dependency> <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> <dependency> Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java Wed Aug 20 01:34:29 2014 @@ -22,6 +22,8 @@ import java.io.IOException; import java.util.Collection; import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -37,12 +39,14 @@ import org.apache.hadoop.yarn.client.api import org.apache.hadoop.yarn.exceptions.YarnException; import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; @InterfaceAudience.Public @InterfaceStability.Stable public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends AbstractService { + private static final Log LOG = LogFactory.getLog(AMRMClient.class); /** * Create a new instance of AMRMClient. @@ -336,4 +340,63 @@ public abstract class AMRMClient<T exten return nmTokenCache; } + /** + * Wait for <code>check</code> to return true for each 1000 ms. + * See also {@link #waitFor(com.google.common.base.Supplier, int)} + * and {@link #waitFor(com.google.common.base.Supplier, int, int)} + * @param check + */ + public void waitFor(Supplier<Boolean> check) throws InterruptedException { + waitFor(check, 1000); + } + + /** + * Wait for <code>check</code> to return true for each + * <code>checkEveryMillis</code> ms. + * See also {@link #waitFor(com.google.common.base.Supplier, int, int)} + * @param check user defined checker + * @param checkEveryMillis interval to call <code>check</code> + */ + public void waitFor(Supplier<Boolean> check, int checkEveryMillis) + throws InterruptedException { + waitFor(check, checkEveryMillis, 1); + } + + /** + * Wait for <code>check</code> to return true for each + * <code>checkEveryMillis</code> ms. In the main loop, this method will log + * the message "waiting in main loop" for each <code>logInterval</code> times + * iteration to confirm the thread is alive. + * @param check user defined checker + * @param checkEveryMillis interval to call <code>check</code> + * @param logInterval interval to log for each + */ + public void waitFor(Supplier<Boolean> check, int checkEveryMillis, + int logInterval) throws InterruptedException { + Preconditions.checkNotNull(check, "check should not be null"); + Preconditions.checkArgument(checkEveryMillis >= 0, + "checkEveryMillis should be positive value"); + Preconditions.checkArgument(logInterval >= 0, + "logInterval should be positive value"); + + int loggingCounter = logInterval; + do { + if (LOG.isDebugEnabled()) { + LOG.debug("Check the condition for main loop."); + } + + boolean result = check.get(); + if (result) { + LOG.info("Exits the main loop."); + return; + } + if (--loggingCounter <= 0) { + LOG.info("Waiting in main loop."); + loggingCounter = logInterval; + } + + Thread.sleep(checkEveryMillis); + } while (true); + } + } Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java Wed Aug 20 01:34:29 2014 @@ -18,11 +18,15 @@ package org.apache.hadoop.yarn.client.api.async; +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; @@ -90,6 +94,7 @@ import com.google.common.annotations.Vis @Stable public abstract class AMRMClientAsync<T extends ContainerRequest> extends AbstractService { + private static final Log LOG = LogFactory.getLog(AMRMClientAsync.class); protected final AMRMClient<T> client; protected final CallbackHandler handler; @@ -189,6 +194,65 @@ extends AbstractService { */ public abstract int getClusterNodeCount(); + /** + * Wait for <code>check</code> to return true for each 1000 ms. + * See also {@link #waitFor(com.google.common.base.Supplier, int)} + * and {@link #waitFor(com.google.common.base.Supplier, int, int)} + * @param check + */ + public void waitFor(Supplier<Boolean> check) throws InterruptedException { + waitFor(check, 1000); + } + + /** + * Wait for <code>check</code> to return true for each + * <code>checkEveryMillis</code> ms. + * See also {@link #waitFor(com.google.common.base.Supplier, int, int)} + * @param check user defined checker + * @param checkEveryMillis interval to call <code>check</code> + */ + public void waitFor(Supplier<Boolean> check, int checkEveryMillis) + throws InterruptedException { + waitFor(check, checkEveryMillis, 1); + }; + + /** + * Wait for <code>check</code> to return true for each + * <code>checkEveryMillis</code> ms. In the main loop, this method will log + * the message "waiting in main loop" for each <code>logInterval</code> times + * iteration to confirm the thread is alive. + * @param check user defined checker + * @param checkEveryMillis interval to call <code>check</code> + * @param logInterval interval to log for each + */ + public void waitFor(Supplier<Boolean> check, int checkEveryMillis, + int logInterval) throws InterruptedException { + Preconditions.checkNotNull(check, "check should not be null"); + Preconditions.checkArgument(checkEveryMillis >= 0, + "checkEveryMillis should be positive value"); + Preconditions.checkArgument(logInterval >= 0, + "logInterval should be positive value"); + + int loggingCounter = logInterval; + do { + if (LOG.isDebugEnabled()) { + LOG.debug("Check the condition for main loop."); + } + + boolean result = check.get(); + if (result) { + LOG.info("Exits the main loop."); + return; + } + if (--loggingCounter <= 0) { + LOG.info("Waiting in main loop."); + loggingCounter = logInterval; + } + + Thread.sleep(checkEveryMillis); + } while (true); + } + public interface CallbackHandler { /** Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java Wed Aug 20 01:34:29 2014 @@ -39,7 +39,9 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; @@ -56,6 +58,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; @@ -64,6 +67,7 @@ import org.apache.hadoop.yarn.conf.YarnC import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.util.RackResolver; import com.google.common.annotations.VisibleForTesting; @@ -300,6 +304,9 @@ public class AMRMClientImpl<T extends Co if (!allocateResponse.getNMTokens().isEmpty()) { populateNMTokens(allocateResponse.getNMTokens()); } + if (allocateResponse.getAMRMToken() != null) { + updateAMRMToken(allocateResponse.getAMRMToken()); + } if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) { removePendingReleaseRequests(allocateResponse @@ -743,4 +750,16 @@ public class AMRMClientImpl<T extends Co "blacklistRemovals in updateBlacklist."); } } + + private void updateAMRMToken(Token token) throws IOException { + org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken = + new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token + .getIdentifier().array(), token.getPassword().array(), new Text( + token.getKind()), new Text(token.getService())); + UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser(); + if (UserGroupInformation.isSecurityEnabled()) { + currentUGI = UserGroupInformation.getLoginUser(); + } + currentUGI.addToken(amrmToken); + } } Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java Wed Aug 20 01:34:29 2014 @@ -267,6 +267,7 @@ public abstract class ProtocolHATestBase protected void startHACluster(int numOfNMs, boolean overrideClientRMService, boolean overrideRTS, boolean overrideApplicationMasterService) throws Exception { + conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); cluster = new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2, Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java Wed Aug 20 01:34:29 2014 @@ -54,11 +54,9 @@ public class TestApplicationMasterServic amClient = ClientRMProxy .createRMProxy(this.conf, ApplicationMasterProtocol.class); - AMRMTokenIdentifier id = - new AMRMTokenIdentifier(attemptId); Token<AMRMTokenIdentifier> appToken = - new Token<AMRMTokenIdentifier>(id, this.cluster.getResourceManager() - .getRMContext().getAMRMTokenSecretManager()); + this.cluster.getResourceManager().getRMContext() + .getAMRMTokenSecretManager().createAndGetAMRMToken(attemptId); appToken.setService(new Text("appToken service")); UserGroupInformation.setLoginUser(UserGroupInformation .createRemoteUser(UserGroupInformation.getCurrentUser() Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java Wed Aug 20 01:34:29 2014 @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.client.api.async.impl; +import com.google.common.base.Supplier; import static org.mockito.Matchers.anyFloat; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyString; @@ -180,7 +181,7 @@ public class TestAMRMClientAsync { AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class); when(client.allocate(anyFloat())).thenThrow(ex); - AMRMClientAsync<ContainerRequest> asyncClient = + AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler); asyncClient.init(conf); asyncClient.start(); @@ -228,6 +229,41 @@ public class TestAMRMClientAsync { asyncClient.stop(); } + @Test (timeout = 10000) + public void testAMRMClientAsyncShutDownWithWaitFor() throws Exception { + Configuration conf = new Configuration(); + final TestCallbackHandler callbackHandler = new TestCallbackHandler(); + @SuppressWarnings("unchecked") + AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class); + + final AllocateResponse shutDownResponse = createAllocateResponse( + new ArrayList<ContainerStatus>(), new ArrayList<Container>(), null); + shutDownResponse.setAMCommand(AMCommand.AM_SHUTDOWN); + when(client.allocate(anyFloat())).thenReturn(shutDownResponse); + + AMRMClientAsync<ContainerRequest> asyncClient = + AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler); + asyncClient.init(conf); + asyncClient.start(); + + Supplier<Boolean> checker = new Supplier<Boolean>() { + @Override + public Boolean get() { + return callbackHandler.reboot; + } + }; + + asyncClient.registerApplicationMaster("localhost", 1234, null); + asyncClient.waitFor(checker); + + asyncClient.stop(); + // stopping should have joined all threads and completed all callbacks + Assert.assertTrue(callbackHandler.callbackCount == 0); + + verify(client, times(1)).allocate(anyFloat()); + asyncClient.stop(); + } + @Test (timeout = 5000) public void testCallAMRMClientAsyncStopFromCallbackHandler() throws YarnException, IOException, InterruptedException { @@ -262,6 +298,40 @@ public class TestAMRMClientAsync { } } + @Test (timeout = 5000) + public void testCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor() + throws YarnException, IOException, InterruptedException { + Configuration conf = new Configuration(); + final TestCallbackHandler2 callbackHandler = new TestCallbackHandler2(); + @SuppressWarnings("unchecked") + AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class); + + List<ContainerStatus> completed = Arrays.asList( + ContainerStatus.newInstance(newContainerId(0, 0, 0, 0), + ContainerState.COMPLETE, "", 0)); + final AllocateResponse response = createAllocateResponse(completed, + new ArrayList<Container>(), null); + + when(client.allocate(anyFloat())).thenReturn(response); + + AMRMClientAsync<ContainerRequest> asyncClient = + AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler); + callbackHandler.asynClient = asyncClient; + asyncClient.init(conf); + asyncClient.start(); + + Supplier<Boolean> checker = new Supplier<Boolean>() { + @Override + public Boolean get() { + return callbackHandler.notify; + } + }; + + asyncClient.registerApplicationMaster("localhost", 1234, null); + asyncClient.waitFor(checker); + Assert.assertTrue(checker.get()); + } + void runCallBackThrowOutException(TestCallbackHandler2 callbackHandler) throws InterruptedException, YarnException, IOException { Configuration conf = new Configuration(); @@ -342,7 +412,7 @@ public class TestAMRMClientAsync { private volatile List<ContainerStatus> completedContainers; private volatile List<Container> allocatedContainers; Exception savedException = null; - boolean reboot = false; + volatile boolean reboot = false; Object notifier = new Object(); int callbackCount = 0; @@ -432,7 +502,7 @@ public class TestAMRMClientAsync { @SuppressWarnings("rawtypes") AMRMClientAsync asynClient; boolean stop = true; - boolean notify = false; + volatile boolean notify = false; boolean throwOutException = false; @Override Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java Wed Aug 20 01:34:29 2014 @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.client.api.impl; +import com.google.common.base.Supplier; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -27,19 +28,23 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.nio.ByteBuffer; +import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.TreeSet; import org.junit.Assert; - import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; @@ -71,9 +76,12 @@ import org.apache.hadoop.yarn.client.api import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.After; @@ -93,6 +101,9 @@ public class TestAMRMClient { static ApplicationAttemptId attemptId = null; static int nodeCount = 3; + static final int rolling_interval_sec = 13; + static final long am_expire_ms = 4000; + static Resource capability; static Priority priority; static Priority priority2; @@ -106,6 +117,10 @@ public class TestAMRMClient { public static void setup() throws Exception { // start minicluster conf = new YarnConfiguration(); + conf.setLong( + YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, + rolling_interval_sec); + conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms); conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100); conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1); yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1); @@ -800,6 +815,40 @@ public class TestAMRMClient { assertEquals(0, amClient.ask.size()); assertEquals(0, amClient.release.size()); } + + class CountDownSupplier implements Supplier<Boolean> { + int counter = 0; + @Override + public Boolean get() { + counter++; + if (counter >= 3) { + return true; + } else { + return false; + } + } + }; + + @Test + public void testWaitFor() throws InterruptedException { + AMRMClientImpl<ContainerRequest> amClient = null; + CountDownSupplier countDownChecker = new CountDownSupplier(); + + try { + // start am rm client + amClient = + (AMRMClientImpl<ContainerRequest>) AMRMClient + .<ContainerRequest> createAMRMClient(); + amClient.init(new YarnConfiguration()); + amClient.start(); + amClient.waitFor(countDownChecker, 1000); + assertEquals(3, countDownChecker.counter); + } finally { + if (amClient != null) { + amClient.stop(); + } + } + } private void sleep(int sleepTime) { try { @@ -809,4 +858,123 @@ public class TestAMRMClient { } } + @Test(timeout = 60000) + public void testAMRMClientOnAMRMTokenRollOver() throws YarnException, + IOException { + AMRMClient<ContainerRequest> amClient = null; + try { + AMRMTokenSecretManager amrmTokenSecretManager = + yarnCluster.getResourceManager().getRMContext() + .getAMRMTokenSecretManager(); + + // start am rm client + amClient = AMRMClient.<ContainerRequest> createAMRMClient(); + + amClient.init(conf); + amClient.start(); + + Long startTime = System.currentTimeMillis(); + amClient.registerApplicationMaster("Host", 10000, ""); + + org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_1 = + getAMRMToken(); + Assert.assertNotNull(amrmToken_1); + Assert.assertEquals(amrmToken_1.decodeIdentifier().getKeyId(), + amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId()); + + // Wait for enough time and make sure the roll_over happens + // At mean time, the old AMRMToken should continue to work + while (System.currentTimeMillis() - startTime < + rolling_interval_sec * 1000) { + amClient.allocate(0.1f); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + amClient.allocate(0.1f); + + org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_2 = + getAMRMToken(); + Assert.assertNotNull(amrmToken_2); + Assert.assertEquals(amrmToken_2.decodeIdentifier().getKeyId(), + amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId()); + + Assert.assertNotEquals(amrmToken_1, amrmToken_2); + + // can do the allocate call with latest AMRMToken + amClient.allocate(0.1f); + + // Make sure previous token has been rolled-over + // and can not use this rolled-over token to make a allocate all. + while (true) { + if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager + .getCurrnetMasterKeyData().getMasterKey().getKeyId()) { + if (amrmTokenSecretManager.getNextMasterKeyData() == null) { + break; + } else if (amrmToken_2.decodeIdentifier().getKeyId() != + amrmTokenSecretManager.getNextMasterKeyData().getMasterKey() + .getKeyId()) { + break; + } + } + amClient.allocate(0.1f); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // DO NOTHING + } + } + + try { + UserGroupInformation testUser = + UserGroupInformation.createRemoteUser("testUser"); + SecurityUtil.setTokenService(amrmToken_2, yarnCluster + .getResourceManager().getApplicationMasterService().getBindAddress()); + testUser.addToken(amrmToken_2); + testUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() { + @Override + public ApplicationMasterProtocol run() { + return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy( + ApplicationMasterProtocol.class, + yarnCluster.getResourceManager().getApplicationMasterService() + .getBindAddress(), conf); + } + }).allocate(Records.newRecord(AllocateRequest.class)); + Assert.fail("The old Token should not work"); + } catch (Exception ex) { + Assert.assertTrue(ex instanceof InvalidToken); + Assert.assertTrue(ex.getMessage().contains( + "Invalid AMRMToken from " + + amrmToken_2.decodeIdentifier().getApplicationAttemptId())); + } + + amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, + null, null); + + } finally { + if (amClient != null && amClient.getServiceState() == STATE.STARTED) { + amClient.stop(); + } + } + } + + @SuppressWarnings("unchecked") + private org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> + getAMRMToken() throws IOException { + Credentials credentials = + UserGroupInformation.getCurrentUser().getCredentials(); + Iterator<org.apache.hadoop.security.token.Token<?>> iter = + credentials.getAllTokens().iterator(); + while (iter.hasNext()) { + org.apache.hadoop.security.token.Token<?> token = iter.next(); + if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { + return (org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>) + token; + } + } + return null; + } } Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java Wed Aug 20 01:34:29 2014 @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.client.api.impl; import java.io.IOException; +import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -26,7 +27,11 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; +import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; @@ -42,10 +47,12 @@ import org.apache.hadoop.yarn.conf.YarnC import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.NodeAction; +import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -58,12 +65,16 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; +import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; public class TestAMRMClientOnRMRestart { static Configuration conf = null; + static final int rolling_interval_sec = 13; + static final long am_expire_ms = 4000; @BeforeClass public static void setup() throws Exception { @@ -362,6 +373,134 @@ public class TestAMRMClientOnRMRestart { } + + // Test verify for AM issued with rolled-over AMRMToken + // is still able to communicate with restarted RM. + @Test(timeout = 30000) + public void testAMRMClientOnAMRMTokenRollOverOnRMRestart() throws Exception { + conf.setLong( + YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, + rolling_interval_sec); + conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms); + MemoryRMStateStore memStore = new MemoryRMStateStore(); + memStore.init(conf); + + // start first RM + MyResourceManager2 rm1 = new MyResourceManager2(conf, memStore); + rm1.start(); + DrainDispatcher dispatcher = + (DrainDispatcher) rm1.getRMContext().getDispatcher(); + Long startTime = System.currentTimeMillis(); + // Submit the application + RMApp app = rm1.submitApp(1024); + dispatcher.await(); + + MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); + nm1.registerNode(); + nm1.nodeHeartbeat(true); // Node heartbeat + dispatcher.await(); + + ApplicationAttemptId appAttemptId = + app.getCurrentAppAttempt().getAppAttemptId(); + rm1.sendAMLaunched(appAttemptId); + dispatcher.await(); + + AMRMTokenSecretManager amrmTokenSecretManagerForRM1 = + rm1.getRMContext().getAMRMTokenSecretManager(); + org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token = + amrmTokenSecretManagerForRM1.createAndGetAMRMToken(appAttemptId); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + ugi.addTokenIdentifier(token.decodeIdentifier()); + + AMRMClient<ContainerRequest> amClient = new MyAMRMClientImpl(rm1); + amClient.init(conf); + amClient.start(); + + amClient.registerApplicationMaster("h1", 10000, ""); + amClient.allocate(0.1f); + + // Wait for enough time and make sure the roll_over happens + // At mean time, the old AMRMToken should continue to work + while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) { + amClient.allocate(0.1f); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // DO NOTHING + } + } + Assert.assertTrue(amrmTokenSecretManagerForRM1.getMasterKey() + .getMasterKey().getKeyId() != token.decodeIdentifier().getKeyId()); + + amClient.allocate(0.1f); + + // active the nextMasterKey, and replace the currentMasterKey + org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newToken = + amrmTokenSecretManagerForRM1.createAndGetAMRMToken(appAttemptId); + int waitCount = 0; + while (waitCount++ <= 50) { + if (amrmTokenSecretManagerForRM1.getCurrnetMasterKeyData().getMasterKey() + .getKeyId() != token.decodeIdentifier().getKeyId()) { + break; + } + try { + amClient.allocate(0.1f); + } catch (Exception ex) { + break; + } + Thread.sleep(500); + } + Assert + .assertTrue(amrmTokenSecretManagerForRM1.getNextMasterKeyData() == null); + Assert.assertTrue(amrmTokenSecretManagerForRM1.getCurrnetMasterKeyData() + .getMasterKey().getKeyId() == newToken.decodeIdentifier().getKeyId()); + + // start 2nd RM + conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:9030"); + final MyResourceManager2 rm2 = new MyResourceManager2(conf, memStore); + rm2.start(); + nm1.setResourceTrackerService(rm2.getResourceTrackerService()); + ((MyAMRMClientImpl) amClient).updateRMProxy(rm2); + dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher(); + + AMRMTokenSecretManager amrmTokenSecretManagerForRM2 = + rm2.getRMContext().getAMRMTokenSecretManager(); + Assert.assertTrue(amrmTokenSecretManagerForRM2.getCurrnetMasterKeyData() + .getMasterKey().getKeyId() == newToken.decodeIdentifier().getKeyId()); + Assert + .assertTrue(amrmTokenSecretManagerForRM2.getNextMasterKeyData() == null); + + try { + UserGroupInformation testUser = + UserGroupInformation.createRemoteUser("testUser"); + SecurityUtil.setTokenService(token, rm2.getApplicationMasterService() + .getBindAddress()); + testUser.addToken(token); + testUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() { + @Override + public ApplicationMasterProtocol run() { + return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy( + ApplicationMasterProtocol.class, + rm2.getApplicationMasterService().getBindAddress(), conf); + } + }).allocate(Records.newRecord(AllocateRequest.class)); + Assert.fail("The old Token should not work"); + } catch (Exception ex) { + Assert.assertTrue(ex instanceof InvalidToken); + Assert.assertTrue(ex.getMessage().contains( + "Invalid AMRMToken from " + + token.decodeIdentifier().getApplicationAttemptId())); + } + + // make sure the recovered AMRMToken works for new RM + amClient.allocate(0.1f); + amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, + null, null); + amClient.stop(); + rm1.stop(); + rm2.stop(); + } + private static class MyFifoScheduler extends FifoScheduler { public MyFifoScheduler(RMContext rmContext) { @@ -445,6 +584,18 @@ public class TestAMRMClientOnRMRestart { } } + private static class MyResourceManager2 extends MyResourceManager { + + public MyResourceManager2(Configuration conf, RMStateStore store) { + super(conf, store); + } + + @Override + protected ApplicationMasterService createApplicationMasterService() { + return new ApplicationMasterService(getRMContext(), scheduler); + } + } + private static class MyAMRMClientImpl extends AMRMClientImpl<ContainerRequest> { private MyResourceManager rm; Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml Wed Aug 20 01:34:29 2014 @@ -67,10 +67,18 @@ <artifactId>commons-codec</artifactId> </dependency> <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jetty-util</artifactId> + </dependency> + <dependency> <groupId>com.sun.jersey</groupId> <artifactId>jersey-core</artifactId> </dependency> <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-client</artifactId> + </dependency> + <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-core-asl</artifactId> </dependency> Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java Wed Aug 20 01:34:29 2014 @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.AMCommand; import org.apache.hadoop.yarn.api.records.Container; @@ -35,6 +36,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.PreemptionMessage; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceDecreasePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceIncreasePBImpl; @@ -44,6 +46,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseProto; @@ -74,7 +77,7 @@ public class AllocateResponsePBImpl exte private List<NodeReport> updatedNodes = null; private PreemptionMessage preempt; - + private Token amrmToken = null; public AllocateResponsePBImpl() { builder = AllocateResponseProto.newBuilder(); @@ -154,6 +157,9 @@ public class AllocateResponsePBImpl exte getChangeProtoIterable(this.decreasedContainers); builder.addAllDecreasedContainers(iterable); } + if (this.amrmToken != null) { + builder.setAmRmToken(convertToProtoFormat(this.amrmToken)); + } } private synchronized void mergeLocalToProto() { @@ -357,6 +363,28 @@ public class AllocateResponsePBImpl exte this.decreasedContainers.addAll(decreasedContainers); } + @Override + public synchronized Token getAMRMToken() { + AllocateResponseProtoOrBuilder p = viaProto ? proto : builder; + if (amrmToken != null) { + return amrmToken; + } + if (!p.hasAmRmToken()) { + return null; + } + this.amrmToken = convertFromProtoFormat(p.getAmRmToken()); + return amrmToken; + } + + @Override + public synchronized void setAMRMToken(Token amRMToken) { + maybeInitBuilder(); + if (amRMToken == null) { + builder.clearAmRmToken(); + } + this.amrmToken = amRMToken; + } + private synchronized void initLocalIncreasedContainerList() { if (this.increasedContainers != null) { return; @@ -699,4 +727,12 @@ public class AllocateResponsePBImpl exte private synchronized NMToken convertFromProtoFormat(NMTokenProto proto) { return new NMTokenPBImpl(proto); } + + private TokenPBImpl convertFromProtoFormat(TokenProto p) { + return new TokenPBImpl(p); + } + + private TokenProto convertToProtoFormat(Token t) { + return ((TokenPBImpl)t).getProto(); + } } Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java Wed Aug 20 01:34:29 2014 @@ -327,6 +327,11 @@ public class GetApplicationsRequestPBImp } @Override + public void setStartRange(LongRange range) { + this.start = range; + } + + @Override public void setStartRange(long begin, long end) throws IllegalArgumentException { if (begin > end) { @@ -350,6 +355,11 @@ public class GetApplicationsRequestPBImp } @Override + public void setFinishRange(LongRange range) { + this.finish = range; + } + + @Override public void setFinishRange(long begin, long end) { if (begin > end) { throw new IllegalArgumentException("begin > end in range (begin, " + Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java Wed Aug 20 01:34:29 2014 @@ -479,6 +479,7 @@ public class ApplicationReportPBImpl ext builder.setAmRmToken(convertToProtoFormat(this.amRmToken)); } if (this.applicationTags != null && !this.applicationTags.isEmpty()) { + builder.clearApplicationTags(); builder.addAllApplicationTags(this.applicationTags); } } Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java Wed Aug 20 01:34:29 2014 @@ -107,6 +107,7 @@ extends ApplicationSubmissionContext { builder.setResource(convertToProtoFormat(this.resource)); } if (this.applicationTags != null && !this.applicationTags.isEmpty()) { + builder.clearApplicationTags(); builder.addAllApplicationTags(this.applicationTags); } } Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java Wed Aug 20 01:34:29 2014 @@ -90,7 +90,7 @@ public class ResourceBlacklistRequestPBI private void addBlacklistRemovalsToProto() { maybeInitBuilder(); - builder.clearBlacklistAdditions(); + builder.clearBlacklistRemovals(); if (this.blacklistRemovals == null) { return; } @@ -159,5 +159,14 @@ public class ResourceBlacklistRequestPBI public int hashCode() { return getProto().hashCode(); } - + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } } Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java Wed Aug 20 01:34:29 2014 @@ -86,4 +86,19 @@ public class ResourceOptionPBImpl extend builder = null; } + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + } Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java Wed Aug 20 01:34:29 2014 @@ -48,7 +48,7 @@ public class TokenPBImpl extends Token { } public synchronized TokenProto getProto() { - mergeLocalToProto(); + mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; return proto; Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java Wed Aug 20 01:34:29 2014 @@ -162,5 +162,19 @@ public class UpdateNodeResourceRequestPB }; this.builder.addAllNodeResourceMap(values); } - + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } } Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java Wed Aug 20 01:34:29 2014 @@ -42,6 +42,13 @@ public class DefaultResourceCalculator e Resource numerator, Resource denominator) { return ratio(numerator, denominator); } + + public boolean isInvalidDivisor(Resource r) { + if (r.getMemory() == 0.0f) { + return true; + } + return false; + } @Override public float ratio(Resource a, Resource b) { Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java?rev=1619019&r1=1619018&r2=1619019&view=diff ============================================================================== --- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java (original) +++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java Wed Aug 20 01:34:29 2014 @@ -109,6 +109,14 @@ public class DominantResourceCalculator getResourceAsValue(clusterResource, numerator, true) / getResourceAsValue(clusterResource, denominator, true); } + + @Override + public boolean isInvalidDivisor(Resource r) { + if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f) { + return true; + } + return false; + } @Override public float ratio(Resource a, Resource b) {
