Repository: hadoop
Updated Branches:
  refs/heads/branch-2 d73359382 -> 254041b71


MAPREDUCE-6632. Master.getMasterAddress() should be updated to use YARN-4629 
(templedf via rkanter)

(cherry picked from commit 4fc632ae19a1d6b0ec09cc7ead789a3cab1c2f1c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/254041b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/254041b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/254041b7

Branch: refs/heads/branch-2
Commit: 254041b71a7f6655bf3fd29e8ec526c87563e3ee
Parents: d733593
Author: Robert Kanter <rkan...@apache.org>
Authored: Thu Sep 22 16:12:56 2016 -0700
Committer: Robert Kanter <rkan...@apache.org>
Committed: Thu Sep 22 16:13:24 2016 -0700

----------------------------------------------------------------------
 .../hadoop-mapreduce-client-core/pom.xml        |  6 +-
 .../java/org/apache/hadoop/mapred/Master.java   | 70 ++++++--------------
 .../org/apache/hadoop/mapred/TestMaster.java    | 56 +---------------
 3 files changed, 28 insertions(+), 104 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/254041b7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
----------------------------------------------------------------------
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index f64a097..04de271 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -37,7 +37,11 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-common</artifactId> 
+      <artifactId>hadoop-yarn-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-common</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/254041b7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
----------------------------------------------------------------------
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
index d84e395..ec00405 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
@@ -19,75 +19,45 @@
 package org.apache.hadoop.mapred;
 
 import java.io.IOException;
-import java.net.InetSocketAddress;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.yarn.conf.HAUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.client.util.YarnClientUtils;
 
 @Private
 @Unstable
 public class Master {
-
-  private static final Log LOG = LogFactory.getLog(Master.class);
-
   public enum State {
     INITIALIZING, RUNNING;
   }
 
-  public static String getMasterUserName(Configuration conf) {
-    String framework = conf.get(MRConfig.FRAMEWORK_NAME, 
MRConfig.YARN_FRAMEWORK_NAME);
-    if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {    
-      return conf.get(MRConfig.MASTER_USER_NAME);
-    } 
-    else {
-      return conf.get(YarnConfiguration.RM_PRINCIPAL);
-    }
+  public static String getMasterAddress(Configuration conf) {
+    String masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
+
+    return NetUtils.createSocketAddr(masterAddress, 8012,
+            MRConfig.MASTER_ADDRESS).getHostName();
   }
-  
-  public static InetSocketAddress getMasterAddress(Configuration conf) {
-    String masterAddress;
-    String framework = conf.get(MRConfig.FRAMEWORK_NAME, 
MRConfig.YARN_FRAMEWORK_NAME);
+
+  public static String getMasterPrincipal(Configuration conf)
+      throws IOException {
+    String masterPrincipal;
+    String framework = conf.get(MRConfig.FRAMEWORK_NAME,
+            MRConfig.YARN_FRAMEWORK_NAME);
+
     if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
-      masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
-      return NetUtils.createSocketAddr(masterAddress, 8012, 
MRConfig.MASTER_ADDRESS);
-    } else if (framework.equals(MRConfig.YARN_FRAMEWORK_NAME) &&
-        HAUtil.isHAEnabled(conf)) {
-      YarnConfiguration yarnConf = new YarnConfiguration(conf);
-      if (yarnConf.get(YarnConfiguration.RM_HA_ID) == null) {
-        String[] rmIds = yarnConf.getStrings(YarnConfiguration.RM_HA_IDS);
-        if (rmIds != null && rmIds.length > 0) {
-          // If RM_HA_ID is not configured, use the first one.
-          // Because any valid RM HA ID should work.
-          yarnConf.set(YarnConfiguration.RM_HA_ID, rmIds[0]);
-        } else {
-          LOG.warn("RM_HA_IDS is not configured when RM HA is enabled");
-        }
-      }
-      return yarnConf.getSocketAddr(
-          YarnConfiguration.RM_ADDRESS,
-          YarnConfiguration.DEFAULT_RM_ADDRESS,
-          YarnConfiguration.DEFAULT_RM_PORT);
+      String masterAddress = getMasterAddress(conf);
+      // get kerberos principal for use as delegation token renewer
+      masterPrincipal =
+          SecurityUtil.getServerPrincipal(conf.get(MRConfig.MASTER_USER_NAME),
+          masterAddress);
     } else {
-      return conf.getSocketAddr(
-          YarnConfiguration.RM_ADDRESS,
-          YarnConfiguration.DEFAULT_RM_ADDRESS,
-          YarnConfiguration.DEFAULT_RM_PORT);
+      masterPrincipal = YarnClientUtils.getRmPrincipal(conf);
     }
-  }
 
-  public static String getMasterPrincipal(Configuration conf) 
-  throws IOException {
-    String masterHostname = getMasterAddress(conf).getHostName();
-    // get kerberos principal for use as delegation token renewer
-    return SecurityUtil.getServerPrincipal(getMasterUserName(conf), 
masterHostname);
+    return masterPrincipal;
   }
-  
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/254041b7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
----------------------------------------------------------------------
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
index 498abbc..0f4ebee 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
@@ -18,13 +18,10 @@
 
 package org.apache.hadoop.mapred;
 
-import static org.junit.Assert.*;
-
-import java.net.InetSocketAddress;
-
 import org.apache.hadoop.mapreduce.MRConfig;
-import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 import org.junit.Test;
 
 public class TestMaster {
@@ -33,13 +30,6 @@ public class TestMaster {
   public void testGetMasterAddress() {
     YarnConfiguration conf = new YarnConfiguration();
 
-    // Default is yarn framework
-    String masterHostname = Master.getMasterAddress(conf).getHostName();
-    
-    // no address set so should default to default rm address
-    InetSocketAddress rmAddr = 
NetUtils.createSocketAddr(YarnConfiguration.DEFAULT_RM_ADDRESS);
-    assertEquals(masterHostname, rmAddr.getHostName());
-    
     // Trying invalid master address for classic 
     conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
     conf.set(MRConfig.MASTER_ADDRESS, "local:invalid");
@@ -55,47 +45,7 @@ public class TestMaster {
 
     // Change master address to a valid value
     conf.set(MRConfig.MASTER_ADDRESS, "bar.com:8042");    
-    masterHostname = Master.getMasterAddress(conf).getHostName();
+    String masterHostname = Master.getMasterAddress(conf);
     assertEquals(masterHostname, "bar.com");
-
-    // change framework to yarn
-    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
-    conf.set(YarnConfiguration.RM_ADDRESS, "foo1.com:8192");
-    masterHostname = Master.getMasterAddress(conf).getHostName();
-    assertEquals(masterHostname, "foo1.com");
-
-    // change framework to yarn and enable HA
-    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
-    conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
-    conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
-    conf.set(YarnConfiguration.RM_ADDRESS + ".rm1", "rm1.com:8192");
-    conf.set(YarnConfiguration.RM_ADDRESS + ".rm2", "rm2.com:8192");
-    masterHostname = Master.getMasterAddress(conf).getHostName();
-    // If RM_HA_ID is not configured, the first one in RM_HA_IDS will be used.
-    assertEquals(masterHostname, "rm1.com");
-    conf.set(YarnConfiguration.RM_HA_ID, "rm2");
-    masterHostname = Master.getMasterAddress(conf).getHostName();
-    // If RM_HA_ID is configured, use the given RM_HA_ID.
-    assertEquals(masterHostname, "rm2.com");
   }
-
-  @Test 
-  public void testGetMasterUser() {
-    YarnConfiguration conf = new YarnConfiguration();
-    conf.set(MRConfig.MASTER_USER_NAME, "foo");
-    conf.set(YarnConfiguration.RM_PRINCIPAL, "bar");
-
-    // default is yarn framework  
-    assertEquals(Master.getMasterUserName(conf), "bar");
-
-    // set framework name to classic
-    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
-    assertEquals(Master.getMasterUserName(conf), "foo");
-
-    // change framework to yarn
-    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
-    assertEquals(Master.getMasterUserName(conf), "bar");
-
-  }
-
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

Reply via email to