Author: vinodkv
Date: Sat Oct 5 01:43:29 2013
New Revision: 1529376
URL: http://svn.apache.org/r1529376
Log:
YARN-1167. Fixed Distributed Shell to not incorrectly show empty hostname on RM
UI. Contributed by Xuan Gong.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1529376&r1=1529375&r2=1529376&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Sat Oct 5 01:43:29 2013
@@ -168,6 +168,9 @@ Release 2.1.2 - UNRELEASED
YARN-1251. TestDistributedShell#TestDSShell failed with timeout. (Xuan Gong
via hitesh)
+ YARN-1167. Fixed Distributed Shell to not incorrectly show empty hostname
+ on RM UI. (Xuan Gong via vinodkv)
+
Release 2.1.1-beta - 2013-09-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java?rev=1529376&r1=1529375&r2=1529376&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
Sat Oct 5 01:43:29 2013
@@ -48,6 +48,7 @@ public abstract class RegisterApplicatio
* <li>port: -1</li>
* <li>trackingUrl: null</li>
* </ul>
+ * The port is allowed to be any integer larger than or equal to -1.
* @return the new instance of <code>RegisterApplicationMasterRequest</code>
*/
@Public
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1529376&r1=1529375&r2=1529376&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
Sat Oct 5 01:43:29 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -99,7 +100,8 @@ import org.apache.hadoop.yarn.util.Recor
* within the <code>ResourceManager</code> regarding what host:port the
* ApplicationMaster is listening on to provide any form of functionality to a
* client as well as a tracking url that a client can use to keep track of
- * status/job history if needed.
+ * status/job history if needed. However, in the distributedshell, trackingurl
+ * and appMasterHost:appMasterRpcPort are not supported.
* </p>
*
* <p>
@@ -168,7 +170,7 @@ public class ApplicationMaster {
// Hostname of the container
private String appMasterHostname = "";
// Port on which the app master listens for status updates from clients
- private int appMasterRpcPort = 0;
+ private int appMasterRpcPort = -1;
// Tracking url to which app master publishes info for clients to monitor
private String appMasterTrackingUrl = "";
@@ -481,6 +483,7 @@ public class ApplicationMaster {
// Register self with ResourceManager
// This will start heartbeating to the RM
+ appMasterHostname = NetUtils.getHostname();
RegisterApplicationMasterResponse response = amRMClient
.registerApplicationMaster(appMasterHostname, appMasterRpcPort,
appMasterTrackingUrl);
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java?rev=1529376&r1=1529375&r2=1529376&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
Sat Oct 5 01:43:29 2013
@@ -24,14 +24,20 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URL;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@@ -117,14 +123,46 @@ public class TestDistributedShell {
};
LOG.info("Initializing DS Client");
- Client client = new Client(new Configuration(yarnCluster.getConfig()));
+ final Client client = new Client(new
Configuration(yarnCluster.getConfig()));
boolean initSuccess = client.init(args);
Assert.assertTrue(initSuccess);
LOG.info("Running DS Client");
- boolean result = client.run();
+ final AtomicBoolean result = new AtomicBoolean(false);
+ Thread t = new Thread() {
+ public void run() {
+ try {
+ result.set(client.run());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ };
+ };
+ t.start();
+ YarnClient yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(new Configuration(yarnCluster.getConfig()));
+ yarnClient.start();
+ String hostName = NetUtils.getHostname();
+ boolean verified = false;
+ while(!verified) {
+ List<ApplicationReport> apps = yarnClient.getApplications();
+ if (apps.size() == 0 ) {
+ Thread.sleep(10);
+ continue;
+ }
+ ApplicationReport appReport = apps.get(0);
+ if (appReport.getHost().startsWith(hostName)
+ && appReport.getRpcPort() == -1) {
+ verified = true;
+ }
+ if (appReport.getYarnApplicationState() ==
YarnApplicationState.FINISHED) {
+ break;
+ }
+ }
+ Assert.assertTrue(verified);
+ t.join();
LOG.info("Client run completed. Result=" + result);
- Assert.assertTrue(result);
+ Assert.assertTrue(result.get());
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java?rev=1529376&r1=1529375&r2=1529376&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
Sat Oct 5 01:43:29 2013
@@ -188,8 +188,8 @@ public class AMRMClientImpl<T extends Co
throws YarnException, IOException {
Preconditions.checkArgument(appHostName != null,
"The host name should not be null");
- Preconditions.checkArgument(appHostPort >= 0,
- "Port number of the host should not be negative");
+ Preconditions.checkArgument(appHostPort >= -1, "Port number of the host"
+ + " should be any integers larger than or equal to -1");
// do this only once ???
RegisterApplicationMasterRequest request =
RegisterApplicationMasterRequest.newInstance(appHostName, appHostPort,
