Author: arp Date: Thu Sep 26 03:41:09 2013 New Revision: 1526366 URL: http://svn.apache.org/r1526366 Log: Merging r1525759 through r1526365 from trunk to branch HDFS-2832
Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/ (props changed) hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/INSTALL hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/conf/ (props changed) hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed) hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobClient.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1525759-1526365 Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt Thu Sep 26 03:41:09 2013 @@ -193,6 +193,18 @@ Release 2.2.0 - UNRELEASED BUG FIXES +Release 2.1.2 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + MAPREDUCE-5504. mapred queue -info inconsistent with types (Kousuke Saruta via tgraves) @@ -204,6 +216,11 @@ Release 2.2.0 - UNRELEASED needs to set up its own certificates etc and not depend on clusters'. (Omkar Vinit Joshi via vinodkv) + MAPREDUCE-5505. Clients should be notified job finished only after job + successfully unregistered (Zhijie Shen via bikas) + + MAPREDUCE-5503. Fixed a test issue in TestMRJobClient. (Jian He via vinodkv) + Release 2.1.1-beta - 2013-09-23 INCOMPATIBLE CHANGES @@ -221,6 +238,10 @@ Release 2.1.1-beta - 2013-09-23 MAPREDUCE-5379. Include token tracking ids in jobconf. (kkambatl via tucu) + MAPREDUCE-5523. Added separate configuration properties for https for JHS + without which even when https is enabled, it starts on http port itself. + (Omkar Vinit Joshi via vinodkv) + OPTIMIZATIONS MAPREDUCE-5446. TestJobHistoryEvents and TestJobHistoryParsing have race Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1525759-1526365 Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/INSTALL URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/INSTALL?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/INSTALL (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/INSTALL Thu Sep 26 03:41:09 2013 @@ -39,11 +39,11 @@ export YARN_CONF_DIR=$HADOOP_CONF_DIR Step 7) Setup config: for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager. <property> <name>yarn.nodemanager.aux-services</name> - <value>mapreduce.shuffle</value> + <value>mapreduce_shuffle</value> </property> <property> - <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name> + <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name> <value>org.apache.hadoop.mapred.ShuffleHandler</value> </property> Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/conf/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1525759-1526365 Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java Thu Sep 26 03:41:09 2013 @@ -63,4 +63,7 @@ public interface AppContext { ClientToAMTokenSecretManager getClientToAMTokenSecretManager(); boolean isLastAMRetry(); + + boolean safeToReportTerminationToUser(); + } Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Thu Sep 26 03:41:09 2013 @@ -31,6 +31,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; @@ -209,6 +210,10 @@ public class MRAppMaster extends Composi private long recoveredJobStartTime = 0; + @VisibleForTesting + protected AtomicBoolean safeToReportTerminationToUser = + new AtomicBoolean(false); + public MRAppMaster(ApplicationAttemptId applicationAttemptId, ContainerId containerId, String nmHost, int nmPort, int nmHttpPort, long appSubmitTime, int maxAppAttempts) { @@ -554,8 +559,10 @@ public class MRAppMaster extends Composi LOG.info("Calling stop for all the services"); MRAppMaster.this.stop(); - // TODO: Stop ClientService last, since only ClientService should wait for - // some time so clients can know the final states. Will be removed once RM come on. + // Except ClientService, other services are already stopped, it is safe to + // let clients know the final states. ClientService should wait for some + // time so clients have enough time to know the final states. + safeToReportTerminationToUser.set(true); try { Thread.sleep(5000); } catch (InterruptedException e) { @@ -964,6 +971,11 @@ public class MRAppMaster extends Composi public boolean isLastAMRetry(){ return isLastAMRetry; } + + @Override + public boolean safeToReportTerminationToUser() { + return safeToReportTerminationToUser.get(); + } } @SuppressWarnings("unchecked") Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Thu Sep 26 03:41:09 2013 @@ -641,6 +641,8 @@ public class JobImpl implements org.apac private ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); private ScheduledFuture failWaitTriggerScheduledFuture; + private JobState lastNonFinalState = JobState.NEW; + public JobImpl(JobId jobId, ApplicationAttemptId applicationAttemptId, Configuration conf, EventHandler eventHandler, TaskAttemptListener taskAttemptListener, @@ -928,7 +930,14 @@ public class JobImpl implements org.apac public JobState getState() { readLock.lock(); try { - return getExternalState(getInternalState()); + JobState state = getExternalState(getInternalState()); + if (!appContext.safeToReportTerminationToUser() + && (state == JobState.SUCCEEDED || state == JobState.FAILED + || state == JobState.KILLED || state == JobState.ERROR)) { + return lastNonFinalState; + } else { + return state; + } } finally { readLock.unlock(); } @@ -972,6 +981,7 @@ public class JobImpl implements org.apac if (oldState != getInternalState()) { LOG.info(jobId + "Job Transitioned from " + oldState + " to " + getInternalState()); + rememberLastNonFinalState(oldState); } } @@ -980,6 +990,15 @@ public class JobImpl implements org.apac } } + private void rememberLastNonFinalState(JobStateInternal stateInternal) { + JobState state = getExternalState(stateInternal); + // if state is not the final state, set lastNonFinalState + if (state != JobState.SUCCEEDED && state != JobState.FAILED + && state != JobState.KILLED && state != JobState.ERROR) { + lastNonFinalState = state; + } + } + @Private public JobStateInternal getInternalState() { readLock.lock(); Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java Thu Sep 26 03:41:09 2013 @@ -37,11 +37,11 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AppInfo; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.Controller; import org.apache.hadoop.yarn.webapp.View; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.base.Joiner; import com.google.inject.Inject; @@ -62,7 +62,7 @@ public class AppController extends Contr set(APP_ID, app.context.getApplicationID().toString()); set(RM_WEB, JOINER.join(WebAppUtil.getSchemePrefix(), - YarnConfiguration.getRMWebAppHostAndPort(conf))); + WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf))); } @Inject Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/WebAppUtil.java Thu Sep 26 03:41:09 2013 @@ -17,6 +17,10 @@ */ package org.apache.hadoop.mapreduce.v2.app.webapp; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; + public class WebAppUtil { private static boolean isSSLEnabledInYARN; @@ -36,4 +40,21 @@ public class WebAppUtil { return "http://"; } } -} + + public static void setJHSWebAppURLWithoutScheme(Configuration conf, + String hostAddress) { + if (HttpConfig.isSecure()) { + conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS, hostAddress); + } else { + conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, hostAddress); + } + } + + public static String getJHSWebAppURLWithoutScheme(Configuration conf) { + if (HttpConfig.isSecure()) { + return conf.get(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS); + } else { + return conf.get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS); + } + } +} \ No newline at end of file Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java Thu Sep 26 03:41:09 2013 @@ -136,10 +136,21 @@ public class MRApp extends MRAppMaster { } public MRApp(int maps, int reduces, boolean autoComplete, String testName, + boolean cleanOnStart, Clock clock, boolean shutdown) { + this(maps, reduces, autoComplete, testName, cleanOnStart, 1, clock, + shutdown); + } + + public MRApp(int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart) { this(maps, reduces, autoComplete, testName, cleanOnStart, 1); } - + + public MRApp(int maps, int reduces, boolean autoComplete, String testName, + boolean cleanOnStart, boolean shutdown) { + this(maps, reduces, autoComplete, testName, cleanOnStart, 1, shutdown); + } + @Override protected void initJobCredentialsAndUGI(Configuration conf) { // Fake a shuffle secret that normally is provided by the job client. @@ -170,22 +181,42 @@ public class MRApp extends MRAppMaster { } public MRApp(int maps, int reduces, boolean autoComplete, String testName, + boolean cleanOnStart, int startCount, boolean shutdown) { + this(maps, reduces, autoComplete, testName, cleanOnStart, startCount, + new SystemClock(), shutdown); + } + + public MRApp(int maps, int reduces, boolean autoComplete, String testName, + boolean cleanOnStart, int startCount, Clock clock, boolean shutdown) { + this(getApplicationAttemptId(applicationId, startCount), getContainerId( + applicationId, startCount), maps, reduces, autoComplete, testName, + cleanOnStart, startCount, clock, shutdown); + } + + public MRApp(int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart, int startCount, Clock clock) { this(getApplicationAttemptId(applicationId, startCount), getContainerId( applicationId, startCount), maps, reduces, autoComplete, testName, - cleanOnStart, startCount, clock); + cleanOnStart, startCount, clock, true); + } + + public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId, + int maps, int reduces, boolean autoComplete, String testName, + boolean cleanOnStart, int startCount, boolean shutdown) { + this(appAttemptId, amContainerId, maps, reduces, autoComplete, testName, + cleanOnStart, startCount, new SystemClock(), shutdown); } public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId, int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart, int startCount) { this(appAttemptId, amContainerId, maps, reduces, autoComplete, testName, - cleanOnStart, startCount, new SystemClock()); + cleanOnStart, startCount, new SystemClock(), true); } public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId, int maps, int reduces, boolean autoComplete, String testName, - boolean cleanOnStart, int startCount, Clock clock) { + boolean cleanOnStart, int startCount, Clock clock, boolean shutdown) { super(appAttemptId, amContainerId, NM_HOST, NM_PORT, NM_HTTP_PORT, clock, System .currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); this.testWorkDir = new File("target", testName); @@ -204,6 +235,9 @@ public class MRApp extends MRAppMaster { this.maps = maps; this.reduces = reduces; this.autoComplete = autoComplete; + // If safeToReportTerminationToUser is set to true, we can verify whether + // the job can reaches the final state when MRAppMaster shuts down. + this.safeToReportTerminationToUser.set(shutdown); } @Override Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java Thu Sep 26 03:41:09 2013 @@ -135,4 +135,11 @@ public class MockAppContext implements A public boolean isLastAMRetry() { return false; } + + @Override + public boolean safeToReportTerminationToUser() { + // bogus - Not Required + return true; + } + } Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java Thu Sep 26 03:41:09 2013 @@ -44,6 +44,7 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType; +import org.apache.hadoop.mapreduce.v2.app.job.event.JobStartEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobUpdatedNodesEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; @@ -374,6 +375,19 @@ public class TestMRApp { app.waitForState(job, JobState.ERROR); } + @SuppressWarnings("resource") + @Test + public void testJobSuccess() throws Exception { + MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true, false); + JobImpl job = (JobImpl) app.submit(new Configuration()); + app.waitForInternalState(job, JobStateInternal.SUCCEEDED); + // AM is not unregistered + Assert.assertEquals(JobState.RUNNING, job.getState()); + // imitate that AM is unregistered + app.safeToReportTerminationToUser.set(true); + app.waitForState(job, JobState.SUCCEEDED); + } + @Test public void testJobRebootNotLastRetry() throws Exception { MRApp app = new MRApp(1, 0, false, this.getClass().getName(), true); Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java Thu Sep 26 03:41:09 2013 @@ -867,5 +867,12 @@ public class TestRuntimeEstimators { public boolean isLastAMRetry() { return false; } + + @Override + public boolean safeToReportTerminationToUser() { + // bogus - Not Required + return true; + } + } } Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java Thu Sep 26 03:41:09 2013 @@ -275,6 +275,7 @@ public class TestJobImpl { AppContext mockContext = mock(AppContext.class); when(mockContext.isLastAMRetry()).thenReturn(true); + when(mockContext.safeToReportTerminationToUser()).thenReturn(false); JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, mockContext); completeJobTasks(job); assertJobState(job, JobStateInternal.COMMITTING); @@ -282,7 +283,9 @@ public class TestJobImpl { syncBarrier.await(); job.handle(new JobEvent(job.getID(), JobEventType.JOB_AM_REBOOT)); assertJobState(job, JobStateInternal.REBOOT); - // return the external state as FAILED since this is last retry. + // return the external state as ERROR since this is last retry. + Assert.assertEquals(JobState.RUNNING, job.getState()); + when(mockContext.safeToReportTerminationToUser()).thenReturn(true); Assert.assertEquals(JobState.ERROR, job.getState()); dispatcher.stop(); @@ -590,12 +593,14 @@ public class TestJobImpl { final JobDiagnosticsUpdateEvent diagUpdateEvent = new JobDiagnosticsUpdateEvent(jobId, diagMsg); MRAppMetrics mrAppMetrics = MRAppMetrics.create(); + AppContext mockContext = mock(AppContext.class); + when(mockContext.safeToReportTerminationToUser()).thenReturn(true); JobImpl job = new JobImpl(jobId, Records .newRecord(ApplicationAttemptId.class), new Configuration(), mock(EventHandler.class), null, mock(JobTokenSecretManager.class), null, new SystemClock(), null, - mrAppMetrics, null, true, null, 0, null, null, null, null); + mrAppMetrics, null, true, null, 0, null, mockContext, null, null); job.handle(diagUpdateEvent); String diagnostics = job.getReport().getDiagnostics(); Assert.assertNotNull(diagnostics); @@ -606,7 +611,7 @@ public class TestJobImpl { mock(EventHandler.class), null, mock(JobTokenSecretManager.class), null, new SystemClock(), null, - mrAppMetrics, null, true, null, 0, null, null, null, null); + mrAppMetrics, null, true, null, 0, null, mockContext, null, null); job.handle(new JobEvent(jobId, JobEventType.JOB_KILL)); job.handle(diagUpdateEvent); diagnostics = job.getReport().getDiagnostics(); @@ -699,7 +704,9 @@ public class TestJobImpl { commitHandler.init(conf); commitHandler.start(); - JobImpl job = createStubbedJob(conf, dispatcher, 2, null); + AppContext mockContext = mock(AppContext.class); + when(mockContext.safeToReportTerminationToUser()).thenReturn(false); + JobImpl job = createStubbedJob(conf, dispatcher, 2, mockContext); JobId jobId = job.getID(); job.handle(new JobEvent(jobId, JobEventType.JOB_INIT)); assertJobState(job, JobStateInternal.INITED); @@ -707,12 +714,15 @@ public class TestJobImpl { assertJobState(job, JobStateInternal.FAILED); job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_COMPLETED)); - Assert.assertEquals(JobState.FAILED, job.getState()); + assertJobState(job, JobStateInternal.FAILED); job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_ATTEMPT_COMPLETED)); - Assert.assertEquals(JobState.FAILED, job.getState()); + assertJobState(job, JobStateInternal.FAILED); job.handle(new JobEvent(jobId, JobEventType.JOB_MAP_TASK_RESCHEDULED)); - Assert.assertEquals(JobState.FAILED, job.getState()); + assertJobState(job, JobStateInternal.FAILED); job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE)); + assertJobState(job, JobStateInternal.FAILED); + Assert.assertEquals(JobState.RUNNING, job.getState()); + when(mockContext.safeToReportTerminationToUser()).thenReturn(true); Assert.assertEquals(JobState.FAILED, job.getState()); dispatcher.stop(); @@ -750,6 +760,10 @@ public class TestJobImpl { Dispatcher dispatcher, int numSplits, AppContext appContext) { JobID jobID = JobID.forName("job_1234567890000_0001"); JobId jobId = TypeConverter.toYarn(jobID); + if (appContext == null) { + appContext = mock(AppContext.class); + when(appContext.safeToReportTerminationToUser()).thenReturn(true); + } StubbedJob job = new StubbedJob(jobId, ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0), conf,dispatcher.getEventHandler(), true, "somebody", numSplits, appContext); Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java Thu Sep 26 03:41:09 2013 @@ -18,8 +18,15 @@ package org.apache.hadoop.mapreduce.v2.jobhistory; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.net.NetUtils; /** * Stores Job History configuration keys that can be set by administrators of @@ -130,6 +137,13 @@ public class JHAdminConfig { public static final String DEFAULT_MR_HISTORY_WEBAPP_ADDRESS = "0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_PORT; + /**The https address the history server webapp is on.*/ + public static final String MR_HISTORY_WEBAPP_HTTPS_ADDRESS = + MR_HISTORY_PREFIX + "webapp.https.address"; + public static final int DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT = 19890; + public static final String DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS = + "0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT; + /**The kerberos principal to be used for spnego filter for history server*/ public static final String MR_WEBAPP_SPNEGO_USER_NAME_KEY = MR_HISTORY_PREFIX + "webapp.spnego-principal"; @@ -160,4 +174,36 @@ public class JHAdminConfig { */ public static boolean DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS = false; + public static String getResolvedMRHistoryWebAppURLWithoutScheme( + Configuration conf) { + InetSocketAddress address = null; + if (HttpConfig.isSecure()) { + address = + conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT); + } else { + address = + conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS, + JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT); } + address = NetUtils.getConnectAddress(address); + StringBuffer sb = new StringBuffer(); + InetAddress resolved = address.getAddress(); + if (resolved == null || resolved.isAnyLocalAddress() || + resolved.isLoopbackAddress()) { + String lh = address.getHostName(); + try { + lh = InetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + //Ignore and fallback. + } + sb.append(lh); + } else { + sb.append(address.getHostName()); + } + sb.append(":").append(address.getPort()); + return sb.toString(); + } + } Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1525759-1526365 Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm Thu Sep 26 03:41:09 2013 @@ -83,14 +83,14 @@ Hadoop MapReduce Next Generation - Plugg *--------------------------------------+---------------------+-----------------+ | <<Property>> | <<Default Value>> | <<Explanation>> | *--------------------------------------+---------------------+-----------------+ -| <<<yarn.nodemanager.aux-services>>> | <<<...,mapreduce.shuffle>>> | The auxiliary service name | +| <<<yarn.nodemanager.aux-services>>> | <<<...,mapreduce_shuffle>>> | The auxiliary service name | *--------------------------------------+---------------------+-----------------+ -| <<<yarn.nodemanager.aux-services.mapreduce.shuffle.class>>> | <<<org.apache.hadoop.mapred.ShuffleHandler>>> | The auxiliary service class to use | +| <<<yarn.nodemanager.aux-services.mapreduce_shuffle.class>>> | <<<org.apache.hadoop.mapred.ShuffleHandler>>> | The auxiliary service class to use | *--------------------------------------+---------------------+-----------------+ <<IMPORTANT:>> If setting an auxiliary service in addition the default - <<<mapreduce.shuffle>>> service, then a new service key should be added to the + <<<mapreduce_shuffle>>> service, then a new service key should be added to the <<<yarn.nodemanager.aux-services>>> property, for example <<<mapred.shufflex>>>. Then the property defining the corresponding class must be - <<<yarn.nodemanager.aux-services.mapreduce.shufflex.class>>>. + <<<yarn.nodemanager.aux-services.mapreduce_shufflex.class>>>. \ No newline at end of file Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java Thu Sep 26 03:41:09 2013 @@ -387,4 +387,11 @@ public class JobHistory extends Abstract // bogus - Not Required return false; } + + @Override + public boolean safeToReportTerminationToUser() { + // bogus - Not Required + return true; + } + } Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobClient.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobClient.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobClient.java Thu Sep 26 03:41:09 2013 @@ -79,7 +79,7 @@ public class TestJobClient extends TestM Configuration conf = createJobConf(); String jobId = runJob(); testGetCounter(jobId, conf); - testJobList(jobId, conf); + testAllJobList(jobId, conf); testChangingJobPriority(jobId, conf); } Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java Thu Sep 26 03:41:09 2013 @@ -29,6 +29,8 @@ import java.io.PipedInputStream; import java.io.PipedOutputStream; import java.io.PrintStream; +import junit.framework.Assert; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -60,6 +62,22 @@ public class TestMRJobClient extends Clu return job; } + private Job runJobInBackGround(Configuration conf) throws Exception { + String input = "hello1\nhello2\nhello3\n"; + + Job job = MapReduceTestUtil.createJob(conf, getInputDir(), getOutputDir(), + 1, 1, input); + job.setJobName("mr"); + job.setPriority(JobPriority.NORMAL); + job.submit(); + int i = 0; + while (i++ < 200 && job.getJobID() == null) { + LOG.info("waiting for jobId..."); + Thread.sleep(100); + } + return job; + } + public static int runTool(Configuration conf, Tool tool, String[] args, OutputStream out) throws Exception { PrintStream oldOut = System.out; @@ -108,8 +126,10 @@ public class TestMRJobClient extends Clu Job job = runJob(conf); String jobId = job.getJobID().toString(); - // test jobs list - testJobList(jobId, conf); + // test all jobs list + testAllJobList(jobId, conf); + // test only submitted jobs list + testSubmittedJobList(conf); // test job counter testGetCounter(jobId, conf); // status @@ -131,38 +151,37 @@ public class TestMRJobClient extends Clu // submit job from file testSubmit(conf); // kill a task - testKillTask(job, conf); + testKillTask(conf); // fail a task - testfailTask(job, conf); + testfailTask(conf); // kill job - testKillJob(jobId, conf); - + testKillJob(conf); } /** * test fail task */ - private void testfailTask(Job job, Configuration conf) throws Exception { + private void testfailTask(Configuration conf) throws Exception { + Job job = runJobInBackGround(conf); CLI jc = createJobClient(); TaskID tid = new TaskID(job.getJobID(), TaskType.MAP, 0); TaskAttemptID taid = new TaskAttemptID(tid, 1); ByteArrayOutputStream out = new ByteArrayOutputStream(); - // TaskAttemptId is not set + // TaskAttemptId is not set int exitCode = runTool(conf, jc, new String[] { "-fail-task" }, out); assertEquals("Exit code", -1, exitCode); - try { - runTool(conf, jc, new String[] { "-fail-task", taid.toString() }, out); - fail(" this task should field"); - } catch (IOException e) { - // task completed ! - assertTrue(e.getMessage().contains("_0001_m_000000_1")); - } + runTool(conf, jc, new String[] { "-fail-task", taid.toString() }, out); + String answer = new String(out.toByteArray(), "UTF-8"); + Assert + .assertTrue(answer.contains("Killed task " + taid + " by failing it")); } + /** * test a kill task */ - private void testKillTask(Job job, Configuration conf) throws Exception { + private void testKillTask(Configuration conf) throws Exception { + Job job = runJobInBackGround(conf); CLI jc = createJobClient(); TaskID tid = new TaskID(job.getJobID(), TaskType.MAP, 0); TaskAttemptID taid = new TaskAttemptID(tid, 1); @@ -171,20 +190,17 @@ public class TestMRJobClient extends Clu int exitCode = runTool(conf, jc, new String[] { "-kill-task" }, out); assertEquals("Exit code", -1, exitCode); - try { - runTool(conf, jc, new String[] { "-kill-task", taid.toString() }, out); - fail(" this task should be killed"); - } catch (IOException e) { - System.out.println(e); - // task completed - assertTrue(e.getMessage().contains("_0001_m_000000_1")); - } + runTool(conf, jc, new String[] { "-kill-task", taid.toString() }, out); + String answer = new String(out.toByteArray(), "UTF-8"); + Assert.assertTrue(answer.contains("Killed task " + taid)); } /** * test a kill job */ - private void testKillJob(String jobId, Configuration conf) throws Exception { + private void testKillJob(Configuration conf) throws Exception { + Job job = runJobInBackGround(conf); + String jobId = job.getJobID().toString(); CLI jc = createJobClient(); ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -435,7 +451,8 @@ public class TestMRJobClient extends Clu /** * print a job list */ - protected void testJobList(String jobId, Configuration conf) throws Exception { + protected void testAllJobList(String jobId, Configuration conf) + throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); // bad options @@ -458,23 +475,31 @@ public class TestMRJobClient extends Clu } assertEquals(1, counter); out.reset(); - // only submitted - exitCode = runTool(conf, createJobClient(), new String[] { "-list" }, out); - assertEquals("Exit code", 0, exitCode); - br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream( - out.toByteArray()))); + } + + protected void testSubmittedJobList(Configuration conf) throws Exception { + Job job = runJobInBackGround(conf); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + String line; + int counter = 0; + // only submitted + int exitCode = + runTool(conf, createJobClient(), new String[] { "-list" }, out); + assertEquals("Exit code", 0, exitCode); + BufferedReader br = + new BufferedReader(new InputStreamReader(new ByteArrayInputStream( + out.toByteArray()))); counter = 0; while ((line = br.readLine()) != null) { LOG.info("line = " + line); - if (line.contains(jobId)) { + if (line.contains(job.getJobID().toString())) { counter++; } } // all jobs submitted! no current assertEquals(1, counter); - } - + protected void verifyJobPriority(String jobId, String priority, Configuration conf, CLI jc) throws Exception { PipedInputStream pis = new PipedInputStream(); Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Thu Sep 26 03:41:09 2013 @@ -28,10 +28,12 @@ import org.apache.hadoop.fs.CommonConfig import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.mapred.LocalContainerLauncher; import org.apache.hadoop.mapred.ShuffleHandler; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.v2.app.webapp.WebAppUtil; import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; @@ -43,6 +45,8 @@ import org.apache.hadoop.yarn.exceptions import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.hamcrest.core.IsEqual; /** * Configures and starts the MR-specific components in the YARN cluster. @@ -155,8 +159,7 @@ public class MiniMRYarnCluster extends M // pick free random ports. getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, hostname + ":0"); - getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, - hostname + ":0"); + WebAppUtil.setJHSWebAppURLWithoutScheme(getConfig(), hostname + ":0"); getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS, hostname + ":0"); } @@ -182,17 +185,17 @@ public class MiniMRYarnCluster extends M //need to do this because historyServer.init creates a new Configuration getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); - getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, - historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS)); + WebAppUtil.setJHSWebAppURLWithoutScheme(getConfig(), + WebAppUtil.getJHSWebAppURLWithoutScheme(historyServer.getConfig())); LOG.info("MiniMRYARN ResourceManager address: " + getConfig().get(YarnConfiguration.RM_ADDRESS)); LOG.info("MiniMRYARN ResourceManager web address: " + - getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS)); + WebAppUtils.getRMWebAppURLWithoutScheme(getConfig())); LOG.info("MiniMRYARN HistoryServer address: " + getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); LOG.info("MiniMRYARN HistoryServer web address: " + - getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS)); + JHAdminConfig.getResolvedMRHistoryWebAppURLWithoutScheme(getConfig())); } @Override Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java?rev=1526366&r1=1526365&r2=1526366&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java Thu Sep 26 03:41:09 2013 @@ -146,7 +146,7 @@ public class ShuffleHandler extends Auxi private ReadaheadPool readaheadPool = ReadaheadPool.getInstance(); public static final String MAPREDUCE_SHUFFLE_SERVICEID = - "mapreduce.shuffle"; + "mapreduce_shuffle"; private static final Map<String,String> userRsrc = new ConcurrentHashMap<String,String>();