Author: szetszwo
Date: Tue May 7 00:17:50 2013
New Revision: 1479733
URL: http://svn.apache.org/r1479733
Log:
Merge r1477868 through r1479732 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/resources/hdfs-site.xml
- copied unchanged from r1479732,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/resources/hdfs-site.xml
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
(contents, props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/ (props
changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(contents, props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1477868-1479732
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt Tue
May 7 00:17:50 2013
@@ -215,6 +215,12 @@ Release 2.0.5-beta - UNRELEASED
MAPREDUCE-5069. add concrete common implementations of
CombineFileInputFormat (Sangjin Lee via bobby)
+ MAPREDUCE-5145. Changed default max-attempts to be more than one for MR
jobs
+ inline with YARN. (Zhijie Shen via vinodkv)
+
+ MAPREDUCE-5036. Default shuffle handler port should not be 8080.
+ (Sandy Ryza via tomwhite)
+
OPTIMIZATIONS
MAPREDUCE-4974. Optimising the LineRecordReader initialize() method
@@ -366,6 +372,14 @@ Release 2.0.5-beta - UNRELEASED
versionInfo has parantheses like when running on a git checkout. (Hitesh
Shah
via vinodkv)
+ MAPREDUCE-5193. A few MR tests use block sizes which are smaller than the
+ default minimum block size. (Andrew Wang via atm)
+
+ MAPREDUCE-5205. Fixed MR App to load tokens correctly. (vinodkv)
+
+ MAPREDUCE-5204. Handling YarnRemoteException separately from IOException in
+ MR app after YARN-629. (Xuan Gong via vinodkv)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1477868-1479732
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1477868-1479732
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
Tue May 7 00:17:50 2013
@@ -1360,8 +1360,13 @@ public class MRAppMaster extends Composi
final YarnConfiguration conf, String jobUserName) throws IOException,
InterruptedException {
UserGroupInformation.setConfiguration(conf);
+ // Security framework already loaded the tokens into current UGI, just use
+ // them
+ Credentials credentials =
+ UserGroupInformation.getCurrentUser().getCredentials();
UserGroupInformation appMasterUgi = UserGroupInformation
.createRemoteUser(jobUserName);
+ appMasterUgi.addCredentials(credentials);
appMasterUgi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
Tue May 7 00:17:50 2013
@@ -17,18 +17,29 @@
*/
package org.apache.hadoop.mapreduce.v2.app;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.TypeConverter;
@@ -41,13 +52,22 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -55,13 +75,20 @@ import org.junit.Test;
public class TestMRAppMaster {
private static final Log LOG = LogFactory.getLog(TestMRAppMaster.class);
static String stagingDir = "staging/";
+ private static FileContext localFS = null;
+ private static final File testDir = new File("target",
+ TestMRAppMaster.class.getName() + "-tmpDir").getAbsoluteFile();
@BeforeClass
- public static void setup() {
+ public static void setup() throws AccessControlException,
+ FileNotFoundException, IllegalArgumentException, IOException {
//Do not error out if metrics are inited multiple times
DefaultMetricsSystem.setMiniClusterMode(true);
File dir = new File(stagingDir);
stagingDir = dir.getAbsolutePath();
+ localFS = FileContext.getLocalFSFileContext();
+ localFS.delete(new Path(testDir.getAbsolutePath()), true);
+ testDir.mkdir();
}
@Before
@@ -269,6 +296,100 @@ public class TestMRAppMaster {
}
}
+ // A dirty hack to modify the env of the current JVM itself - Dirty, but
+ // should be okay for testing.
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private static void setNewEnvironmentHack(Map<String, String> newenv)
+ throws Exception {
+ try {
+ Class<?> cl = Class.forName("java.lang.ProcessEnvironment");
+ Field field = cl.getDeclaredField("theEnvironment");
+ field.setAccessible(true);
+ Map<String, String> env = (Map<String, String>) field.get(null);
+ env.clear();
+ env.putAll(newenv);
+ Field ciField = cl.getDeclaredField("theCaseInsensitiveEnvironment");
+ ciField.setAccessible(true);
+ Map<String, String> cienv = (Map<String, String>) ciField.get(null);
+ cienv.clear();
+ cienv.putAll(newenv);
+ } catch (NoSuchFieldException e) {
+ Class[] classes = Collections.class.getDeclaredClasses();
+ Map<String, String> env = System.getenv();
+ for (Class cl : classes) {
+ if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) {
+ Field field = cl.getDeclaredField("m");
+ field.setAccessible(true);
+ Object obj = field.get(env);
+ Map<String, String> map = (Map<String, String>) obj;
+ map.clear();
+ map.putAll(newenv);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testMRAppMasterCredentials() throws Exception {
+
+ Logger rootLogger = LogManager.getRootLogger();
+ rootLogger.setLevel(Level.DEBUG);
+
+ // Simulate credentials passed to AM via client->RM->NM
+ Credentials credentials = new Credentials();
+ byte[] identifier = "MyIdentifier".getBytes();
+ byte[] password = "MyPassword".getBytes();
+ Text kind = new Text("MyTokenKind");
+ Text service = new Text("host:port");
+ Token<? extends TokenIdentifier> myToken =
+ new Token<TokenIdentifier>(identifier, password, kind, service);
+ Text tokenAlias = new Text("myToken");
+ credentials.addToken(tokenAlias, myToken);
+ Token<? extends TokenIdentifier> storedToken =
+ credentials.getToken(tokenAlias);
+
+ YarnConfiguration conf = new YarnConfiguration();
+
+ Path tokenFilePath = new Path(testDir.getAbsolutePath(), "tokens-file");
+ Map<String, String> newEnv = new HashMap<String, String>();
+ newEnv.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, tokenFilePath
+ .toUri().getPath());
+ setNewEnvironmentHack(newEnv);
+ credentials.writeTokenStorageFile(tokenFilePath, conf);
+
+ ApplicationId appId = BuilderUtils.newApplicationId(12345, 56);
+ ApplicationAttemptId applicationAttemptId =
+ BuilderUtils.newApplicationAttemptId(appId, 1);
+ ContainerId containerId =
+ BuilderUtils.newContainerId(applicationAttemptId, 546);
+ String userName = UserGroupInformation.getCurrentUser().getShortUserName();
+
+ // Create staging dir, so MRAppMaster doesn't barf.
+ File stagingDir =
+ new File(MRApps.getStagingAreaDir(conf, userName).toString());
+ stagingDir.mkdirs();
+
+ // Set login-user to null as that is how real world MRApp starts with.
+ // This is null is the reason why token-file is read by UGI.
+ UserGroupInformation.setLoginUser(null);
+
+ MRAppMasterTest appMaster =
+ new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
+ System.currentTimeMillis(), 1, false, true);
+ MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
+
+ // Now validate the credentials
+ Credentials appMasterCreds = appMaster.credentials;
+ Assert.assertNotNull(appMasterCreds);
+ Token<? extends TokenIdentifier> usedToken =
+ appMasterCreds.getToken(tokenAlias);
+ Assert.assertNotNull(usedToken);
+ Assert
+ .assertEquals("MyIdentifier", new String(storedToken.getIdentifier()));
+ Assert.assertEquals("MyPassword", new String(storedToken.getPassword()));
+ Assert.assertEquals("MyTokenKind", storedToken.getKind().toString());
+ Assert.assertEquals("host:port", storedToken.getService().toString());
+ }
}
class MRAppMasterTest extends MRAppMaster {
@@ -280,6 +401,7 @@ class MRAppMasterTest extends MRAppMaste
ContainerAllocator mockContainerAllocator;
CommitterEventHandler mockCommitterEventHandler;
RMHeartbeatHandler mockRMHeartbeatHandler;
+ Credentials credentials;
public MRAppMasterTest(ApplicationAttemptId applicationAttemptId,
ContainerId containerId, String host, int port, int httpPort,
@@ -338,7 +460,9 @@ class MRAppMasterTest extends MRAppMaste
public void start() {
if (overrideStart) {
try {
- String user = UserGroupInformation.getCurrentUser().getShortUserName();
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String user = ugi.getShortUserName();
+ this.credentials = ugi.getCredentials();
stagingDirPath = MRApps.getStagingAreaDir(conf, user);
} catch (Exception e) {
fail(e.getMessage());
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
Tue May 7 00:17:50 2013
@@ -90,6 +90,7 @@ import org.junit.Test;
JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
+ Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
JobStateInternal.RUNNING, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
appMaster.init(conf);
@@ -116,8 +117,9 @@ import org.junit.Test;
appId.setId(0);
attemptId.setApplicationId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
+ Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
- JobStateInternal.REBOOT, 4);
+ JobStateInternal.REBOOT, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
appMaster.init(conf);
appMaster.start();
//shutdown the job, not the lastRetry
@@ -144,7 +146,7 @@ import org.junit.Test;
attemptId.setApplicationId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
- JobStateInternal.REBOOT, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
+ JobStateInternal.REBOOT, 1); //no retry
appMaster.init(conf);
appMaster.start();
//shutdown the job, is lastRetry
@@ -201,8 +203,7 @@ import org.junit.Test;
JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
- MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
- MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
+ MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, 1); //no retry
appMaster.init(conf);
//simulate the process being killed
MRAppMaster.MRAppMasterShutdownHook hook =
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java
Tue May 7 00:17:50 2013
@@ -98,7 +98,7 @@ import org.apache.hadoop.security.proto.
import
org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
import
org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
import com.google.protobuf.ServiceException;
@@ -133,7 +133,7 @@ public class MRClientProtocolPBClientImp
try {
return new GetJobReportResponsePBImpl(proxy.getJobReport(null,
requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -144,7 +144,7 @@ public class MRClientProtocolPBClientImp
try {
return new GetTaskReportResponsePBImpl(proxy.getTaskReport(null,
requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -155,7 +155,7 @@ public class MRClientProtocolPBClientImp
try {
return new
GetTaskAttemptReportResponsePBImpl(proxy.getTaskAttemptReport(null,
requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -166,7 +166,7 @@ public class MRClientProtocolPBClientImp
try {
return new GetCountersResponsePBImpl(proxy.getCounters(null,
requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -177,7 +177,7 @@ public class MRClientProtocolPBClientImp
try {
return new
GetTaskAttemptCompletionEventsResponsePBImpl(proxy.getTaskAttemptCompletionEvents(null,
requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -188,7 +188,7 @@ public class MRClientProtocolPBClientImp
try {
return new GetTaskReportsResponsePBImpl(proxy.getTaskReports(null,
requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -199,7 +199,7 @@ public class MRClientProtocolPBClientImp
try {
return new GetDiagnosticsResponsePBImpl(proxy.getDiagnostics(null,
requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -212,7 +212,7 @@ public class MRClientProtocolPBClientImp
return new GetDelegationTokenResponsePBImpl(proxy.getDelegationToken(
null, requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -223,7 +223,7 @@ public class MRClientProtocolPBClientImp
try {
return new KillJobResponsePBImpl(proxy.killJob(null, requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -234,7 +234,7 @@ public class MRClientProtocolPBClientImp
try {
return new KillTaskResponsePBImpl(proxy.killTask(null, requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -245,7 +245,7 @@ public class MRClientProtocolPBClientImp
try {
return new KillTaskAttemptResponsePBImpl(proxy.killTaskAttempt(null,
requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -256,7 +256,7 @@ public class MRClientProtocolPBClientImp
try {
return new FailTaskAttemptResponsePBImpl(proxy.failTaskAttempt(null,
requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -269,7 +269,7 @@ public class MRClientProtocolPBClientImp
return new RenewDelegationTokenResponsePBImpl(proxy.renewDelegationToken(
null, requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
@@ -283,7 +283,7 @@ public class MRClientProtocolPBClientImp
proxy.cancelDelegationToken(null, requestProto));
} catch (ServiceException e) {
- throw YarnRemoteExceptionPBImpl.unwrapAndThrowException(e);
+ throw RPCUtil.unwrapAndThrowException(e);
}
}
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/MRDelegationTokenRenewer.java
Tue May 7 00:17:50 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenRenewer;
import org.apache.hadoop.yarn.api.records.DelegationToken;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
@@ -67,6 +68,8 @@ public class MRDelegationTokenRenewer ex
.newRecord(RenewDelegationTokenRequest.class);
request.setDelegationToken(dToken);
return histProxy.renewDelegationToken(request).getNextExpirationTime();
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
} finally {
stopHistoryProxy(histProxy);
}
@@ -88,6 +91,8 @@ public class MRDelegationTokenRenewer ex
.newRecord(CancelDelegationTokenRequest.class);
request.setDelegationToken(dToken);
histProxy.cancelDelegationToken(request);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
} finally {
stopHistoryProxy(histProxy);
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
Tue May 7 00:17:50 2013
@@ -677,6 +677,6 @@ public interface MRJobConfig {
*/
public static final String MR_AM_MAX_ATTEMPTS = "mapreduce.am.max-attempts";
- public static final int DEFAULT_MR_AM_MAX_ATTEMPTS = 1;
+ public static final int DEFAULT_MR_AM_MAX_ATTEMPTS = 2;
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Tue May 7 00:17:50 2013
@@ -773,7 +773,7 @@
<property>
<name>mapreduce.shuffle.port</name>
- <value>8080</value>
+ <value>11000</value>
<description>Default port that the ShuffleHandler will run on.
ShuffleHandler
is a service run at the NodeManager to facilitate transfers of intermediate
Map outputs to requesting Reducers.
@@ -816,10 +816,11 @@
<property>
<name>mapreduce.am.max-attempts</name>
- <value>1</value>
+ <value>2</value>
<description>The maximum number of application attempts. It is a
application-specific setting. It should not be larger than the global number
- set by resourcemanager. Otherwise, it will be override.</description>
+ set by resourcemanager. Otherwise, it will be override. The default number is
+ set to 2, to allow at least one retry for AM.</description>
</property>
<!-- Job Notification Configuration -->
Propchange:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1477868-1479732
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
Tue May 7 00:17:50 2013
@@ -302,13 +302,13 @@ public class ClientServiceDelegate {
return methodOb.invoke(getProxy(), args);
} catch (YarnRemoteException yre) {
LOG.warn("Exception thrown by remote end.", yre);
- throw yre;
+ throw new IOException(yre);
} catch (InvocationTargetException e) {
if (e.getTargetException() instanceof YarnRemoteException) {
LOG.warn("Error from remote end: " + e
.getTargetException().getLocalizedMessage());
LOG.debug("Tracing remote error ", e.getTargetException());
- throw (YarnRemoteException) e.getTargetException();
+ throw new IOException(e.getTargetException());
}
LOG.debug("Failed to contact AM/History for job " + jobId +
" retrying..", e.getTargetException());
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
Tue May 7 00:17:50 2013
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.YarnClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.util.ProtoUtils;
public class ResourceMgrDelegate extends YarnClientImpl {
@@ -65,11 +66,19 @@ public class ResourceMgrDelegate extends
public TaskTrackerInfo[] getActiveTrackers() throws IOException,
InterruptedException {
- return TypeConverter.fromYarnNodes(super.getNodeReports());
+ try {
+ return TypeConverter.fromYarnNodes(super.getNodeReports());
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
public JobStatus[] getAllJobs() throws IOException, InterruptedException {
- return TypeConverter.fromYarnApps(super.getApplicationList(), this.conf);
+ try {
+ return TypeConverter.fromYarnApps(super.getApplicationList(), this.conf);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
public TaskTrackerInfo[] getBlacklistedTrackers() throws IOException,
@@ -81,11 +90,17 @@ public class ResourceMgrDelegate extends
public ClusterMetrics getClusterMetrics() throws IOException,
InterruptedException {
- YarnClusterMetrics metrics = super.getYarnClusterMetrics();
- ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1,
- metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1,
- metrics.getNumNodeManagers(), 0, 0);
- return oldMetrics;
+ try {
+ YarnClusterMetrics metrics = super.getYarnClusterMetrics();
+ ClusterMetrics oldMetrics =
+ new ClusterMetrics(1, 1, 1, 1, 1, 1,
+ metrics.getNumNodeManagers() * 10,
+ metrics.getNumNodeManagers() * 2, 1,
+ metrics.getNumNodeManagers(), 0, 0);
+ return oldMetrics;
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
InetSocketAddress getConnectAddress() {
@@ -95,8 +110,12 @@ public class ResourceMgrDelegate extends
@SuppressWarnings("rawtypes")
public Token getDelegationToken(Text renewer) throws IOException,
InterruptedException {
- return ProtoUtils.convertFromProtoFormat(
- super.getRMDelegationToken(renewer), rmAddress);
+ try {
+ return ProtoUtils.convertFromProtoFormat(
+ super.getRMDelegationToken(renewer), rmAddress);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
public String getFilesystemName() throws IOException, InterruptedException {
@@ -104,36 +123,62 @@ public class ResourceMgrDelegate extends
}
public JobID getNewJobID() throws IOException, InterruptedException {
- this.application = super.getNewApplication();
- this.applicationId = this.application.getApplicationId();
- return TypeConverter.fromYarn(applicationId);
+ try {
+ this.application = super.getNewApplication();
+ this.applicationId = this.application.getApplicationId();
+ return TypeConverter.fromYarn(applicationId);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
public QueueInfo getQueue(String queueName) throws IOException,
InterruptedException {
- org.apache.hadoop.yarn.api.records.QueueInfo queueInfo =
- super.getQueueInfo(queueName);
- return (queueInfo == null) ? null : TypeConverter.fromYarn(queueInfo,
conf);
+ try {
+ org.apache.hadoop.yarn.api.records.QueueInfo queueInfo =
+ super.getQueueInfo(queueName);
+ return (queueInfo == null) ? null : TypeConverter.fromYarn(queueInfo,
+ conf);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException,
InterruptedException {
- return TypeConverter.fromYarnQueueUserAclsInfo(super
- .getQueueAclsInfo());
+ try {
+ return TypeConverter.fromYarnQueueUserAclsInfo(super
+ .getQueueAclsInfo());
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
public QueueInfo[] getQueues() throws IOException, InterruptedException {
- return TypeConverter.fromYarnQueueInfo(super.getAllQueues(), this.conf);
+ try {
+ return TypeConverter.fromYarnQueueInfo(super.getAllQueues(), this.conf);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
public QueueInfo[] getRootQueues() throws IOException, InterruptedException {
- return TypeConverter.fromYarnQueueInfo(super.getRootQueueInfos(),
this.conf);
+ try {
+ return TypeConverter.fromYarnQueueInfo(super.getRootQueueInfos(),
+ this.conf);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
public QueueInfo[] getChildQueues(String parent) throws IOException,
InterruptedException {
- return TypeConverter.fromYarnQueueInfo(super.getChildQueueInfos(parent),
- this.conf);
+ try {
+ return TypeConverter.fromYarnQueueInfo(super.getChildQueueInfos(parent),
+ this.conf);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
public String getStagingAreaDir() throws IOException, InterruptedException {
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Tue May 7 00:17:50 2013
@@ -80,6 +80,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.client.RMTokenSelector;
@@ -207,10 +208,15 @@ public class YARNRunner implements Clien
GetDelegationTokenRequest request = recordFactory
.newRecordInstance(GetDelegationTokenRequest.class);
request.setRenewer(Master.getMasterPrincipal(conf));
- DelegationToken mrDelegationToken = hsProxy.getDelegationToken(request)
- .getDelegationToken();
- return ProtoUtils.convertFromProtoFormat(mrDelegationToken,
- hsProxy.getConnectAddress());
+ DelegationToken mrDelegationToken;
+ try {
+ mrDelegationToken = hsProxy.getDelegationToken(request)
+ .getDelegationToken();
+ return ProtoUtils.convertFromProtoFormat(mrDelegationToken,
+ hsProxy.getConnectAddress());
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
@Override
@@ -295,19 +301,25 @@ public class YARNRunner implements Clien
createApplicationSubmissionContext(conf, jobSubmitDir, ts);
// Submit to ResourceManager
- ApplicationId applicationId = resMgrDelegate.submitApplication(appContext);
+ try {
+ ApplicationId applicationId =
+ resMgrDelegate.submitApplication(appContext);
- ApplicationReport appMaster = resMgrDelegate
- .getApplicationReport(applicationId);
- String diagnostics =
- (appMaster == null ?
- "application report is null" : appMaster.getDiagnostics());
- if (appMaster == null || appMaster.getYarnApplicationState() ==
YarnApplicationState.FAILED
- || appMaster.getYarnApplicationState() == YarnApplicationState.KILLED)
{
- throw new IOException("Failed to run job : " +
- diagnostics);
+ ApplicationReport appMaster = resMgrDelegate
+ .getApplicationReport(applicationId);
+ String diagnostics =
+ (appMaster == null ?
+ "application report is null" : appMaster.getDiagnostics());
+ if (appMaster == null
+ || appMaster.getYarnApplicationState() == YarnApplicationState.FAILED
+ || appMaster.getYarnApplicationState() ==
YarnApplicationState.KILLED) {
+ throw new IOException("Failed to run job : " +
+ diagnostics);
+ }
+ return clientCache.getClient(jobId).getJobStatus(jobId);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
}
- return clientCache.getClient(jobId).getJobStatus(jobId);
}
private LocalResource createApplicationResource(FileContext fs, Path p,
LocalResourceType type)
@@ -552,7 +564,11 @@ public class YARNRunner implements Clien
/* check if the status is not running, if not send kill to RM */
JobStatus status = clientCache.getClient(arg0).getJobStatus(arg0);
if (status.getState() != JobStatus.State.RUNNING) {
- resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
+ try {
+ resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
return;
}
@@ -576,7 +592,11 @@ public class YARNRunner implements Clien
LOG.debug("Error when checking for application status", io);
}
if (status.getState() != JobStatus.State.KILLED) {
- resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
+ try {
+ resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
}
@@ -607,7 +627,11 @@ public class YARNRunner implements Clien
@Override
public LogParams getLogFileParams(JobID jobID, TaskAttemptID taskAttemptID)
throws IOException {
- return clientCache.getClient(jobID).getLogFilePath(jobID, taskAttemptID);
+ try {
+ return clientCache.getClient(jobID).getLogFilePath(jobID, taskAttemptID);
+ } catch (YarnRemoteException e) {
+ throw new IOException(e);
+ }
}
private static void warnForJavaLibPath(String opts, String component,
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
Tue May 7 00:17:50 2013
@@ -115,8 +115,9 @@ public class TestClientServiceDelegate {
try {
clientServiceDelegate.getJobStatus(oldJobId);
Assert.fail("Invoke should throw exception after retries.");
- } catch (YarnRemoteException e) {
- Assert.assertEquals("Job ID doesnot Exist", e.getMessage());
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Job ID doesnot Exist"));
}
}
@@ -198,7 +199,8 @@ public class TestClientServiceDelegate {
}
@Test
- public void testReconnectOnAMRestart() throws IOException {
+ public void testReconnectOnAMRestart() throws IOException,
+ YarnRemoteException {
//test not applicable when AM not reachable
//as instantiateAMProxy is not called at all
if(!isAMReachableFromClient) {
@@ -265,7 +267,7 @@ public class TestClientServiceDelegate {
}
@Test
- public void testAMAccessDisabled() throws IOException {
+ public void testAMAccessDisabled() throws IOException, YarnRemoteException {
//test only applicable when AM not reachable
if(isAMReachableFromClient) {
return;
@@ -317,7 +319,8 @@ public class TestClientServiceDelegate {
}
@Test
- public void testRMDownForJobStatusBeforeGetAMReport() throws IOException {
+ public void testRMDownForJobStatusBeforeGetAMReport() throws IOException,
+ YarnRemoteException {
Configuration conf = new YarnConfiguration();
testRMDownForJobStatusBeforeGetAMReport(conf,
MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES);
@@ -325,7 +328,7 @@ public class TestClientServiceDelegate {
@Test
public void testRMDownForJobStatusBeforeGetAMReportWithRetryTimes()
- throws IOException {
+ throws IOException, YarnRemoteException {
Configuration conf = new YarnConfiguration();
conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 2);
testRMDownForJobStatusBeforeGetAMReport(conf, conf.getInt(
@@ -335,7 +338,7 @@ public class TestClientServiceDelegate {
@Test
public void testRMDownRestoreForJobStatusBeforeGetAMReport()
- throws IOException {
+ throws IOException, YarnRemoteException {
Configuration conf = new YarnConfiguration();
conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
Tue May 7 00:17:50 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
@@ -47,9 +48,12 @@ public class TestResourceMgrDelegate {
/**
* Tests that getRootQueues makes a request for the (recursive) child queues
+ * @throws YarnRemoteException
+ * @throws IOException
*/
@Test
- public void testGetRootQueues() throws IOException, InterruptedException {
+ public void testGetRootQueues() throws IOException, InterruptedException,
+ YarnRemoteException {
final ClientRMProtocol applicationsManager =
Mockito.mock(ClientRMProtocol.class);
GetQueueInfoResponse response = Mockito.mock(GetQueueInfoResponse.class);
org.apache.hadoop.yarn.api.records.QueueInfo queueInfo =
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
Tue May 7 00:17:50 2013
@@ -40,7 +40,6 @@ import org.apache.hadoop.mapreduce.tools
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
/**
test CLI class. CLI class implemented the Tool interface.
@@ -155,7 +154,7 @@ public class TestMRJobClient extends Clu
try {
runTool(conf, jc, new String[] { "-fail-task", taid.toString() }, out);
fail(" this task should field");
- } catch (YarnRemoteException e) {
+ } catch (IOException e) {
// task completed !
assertTrue(e.getMessage().contains("_0001_m_000000_1"));
}
@@ -175,7 +174,7 @@ public class TestMRJobClient extends Clu
try {
runTool(conf, jc, new String[] { "-kill-task", taid.toString() }, out);
fail(" this task should be killed");
- } catch (YarnRemoteException e) {
+ } catch (IOException e) {
// task completed
assertTrue(e.getMessage().contains("_0001_m_000000_1"));
}
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
Tue May 7 00:17:50 2013
@@ -62,7 +62,8 @@ public class TestJHSSecurity {
private static final Log LOG = LogFactory.getLog(TestJHSSecurity.class);
@Test
- public void testDelegationToken() throws IOException, InterruptedException {
+ public void testDelegationToken() throws IOException, InterruptedException,
+ YarnRemoteException {
Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG);
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
Tue May 7 00:17:50 2013
@@ -46,6 +46,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.util.BuilderUtils;
@@ -114,7 +115,7 @@ public class TestMRJobsWithHistoryServic
@Test (timeout = 30000)
public void testJobHistoryData() throws IOException, InterruptedException,
- AvroRemoteException, ClassNotFoundException {
+ AvroRemoteException, ClassNotFoundException, YarnRemoteException {
if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
+ " not found. Not running test.");
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java?rev=1479733&r1=1479732&r2=1479733&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
(original)
+++
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
Tue May 7 00:17:50 2013
@@ -153,7 +153,7 @@ public class ShuffleHandler extends Abst
new JobTokenSecretManager();
public static final String SHUFFLE_PORT_CONFIG_KEY =
"mapreduce.shuffle.port";
- public static final int DEFAULT_SHUFFLE_PORT = 8080;
+ public static final int DEFAULT_SHUFFLE_PORT = 11000;
public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY =
"mapreduce.shuffle.ssl.file.buffer.size";
