This is an automated email from the ASF dual-hosted git repository.

iwasakims pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 7e73cad  MAPREDUCE-7280. MiniMRYarnCluster has hard-coded timeout 
waiting to start history server, with no way to disable. (#2065)
7e73cad is described below

commit 7e73cad97419671b1020906bef36ad60aaa70ab9
Author: Masatake Iwasaki <iwasak...@apache.org>
AuthorDate: Tue Jun 30 10:52:25 2020 +0900

    MAPREDUCE-7280. MiniMRYarnCluster has hard-coded timeout waiting to start 
history server, with no way to disable. (#2065)
---
 .../apache/hadoop/mapred/TestMiniMRBringup.java    | 24 +++++++++++++--
 .../hadoop/mapreduce/v2/MiniMRYarnCluster.java     | 34 +++++++++++++++-------
 2 files changed, 45 insertions(+), 13 deletions(-)

diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRBringup.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRBringup.java
index b608d75..fc49fa5 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRBringup.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRBringup.java
@@ -18,9 +18,13 @@
 
 package org.apache.hadoop.mapred;
 
+import java.io.IOException;
+
+import org.junit.Assert;
 import org.junit.Test;
 
-import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
 
 /**
  * A Unit-test to test bringup and shutdown of Mini Map-Reduce Cluster.
@@ -36,5 +40,21 @@ public class TestMiniMRBringup {
       if (mr != null) { mr.shutdown(); }
     }
   }
-  
+
+  @Test
+  public void testMiniMRYarnClusterWithoutJHS() throws IOException {
+    MiniMRYarnCluster mr = null;
+    try {
+      final Configuration conf = new Configuration();
+      conf.setBoolean(MiniMRYarnCluster.MR_HISTORY_MINICLUSTER_ENABLED, false);
+      mr = new MiniMRYarnCluster("testMiniMRYarnClusterWithoutJHS");
+      mr.init(conf);
+      mr.start();
+      Assert.assertEquals(null, mr.getHistoryServer());
+    } finally {
+      if (mr != null) {
+        mr.stop();
+      }
+    }
+  }
 }
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
index 299383d..dbd87e2 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
@@ -69,6 +69,10 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
   private JobHistoryServer historyServer;
   private JobHistoryServerWrapper historyServerWrapper;
   private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
+  public static final String MR_HISTORY_MINICLUSTER_ENABLED =
+      JHAdminConfig.MR_HISTORY_PREFIX + "minicluster.enabled";
+  public static final String MR_HISTORY_MINICLUSTER_LAUNCH_TIMEOUT_MS =
+      JHAdminConfig.MR_HISTORY_PREFIX + "minicluster.launch.timeout.ms";
 
   public MiniMRYarnCluster(String testName) {
     this(testName, 1);
@@ -77,11 +81,10 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
   public MiniMRYarnCluster(String testName, int noOfNMs) {
     this(testName, noOfNMs, false);
   }
+
   @Deprecated
   public MiniMRYarnCluster(String testName, int noOfNMs, boolean enableAHS) {
     super(testName, 1, noOfNMs, 4, 4, enableAHS);
-    historyServerWrapper = new JobHistoryServerWrapper();
-    addService(historyServerWrapper);
   }
 
   public static String getResolvedMRHistoryWebAppURLWithoutScheme(
@@ -118,6 +121,11 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
 
   @Override
   public void serviceInit(Configuration conf) throws Exception {
+    if (conf.getBoolean(MR_HISTORY_MINICLUSTER_ENABLED, true)) {
+      historyServerWrapper = new JobHistoryServerWrapper();
+      addService(historyServerWrapper);
+    }
+
     conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
     String stagingDir = conf.get(MRJobConfig.MR_AM_STAGING_DIR);
     if (stagingDir == null ||
@@ -212,11 +220,13 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
   protected void serviceStart() throws Exception {
     super.serviceStart();
 
-    //need to do this because historyServer.init creates a new Configuration
-    getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
-                    
historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
-    MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(),
-        MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig()));
+    if (historyServer != null) {
+      //need to do this because historyServer.init creates a new Configuration
+      getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
+          historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
+      MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(),
+          
MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig()));
+    }
 
     LOG.info("MiniMRYARN ResourceManager address: " +
         getConfig().get(YarnConfiguration.RM_ADDRESS));
@@ -233,7 +243,6 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
     public JobHistoryServerWrapper() {
       super(JobHistoryServerWrapper.class.getName());
     }
-    private volatile boolean jhsStarted = false;
 
     @Override
     public synchronized void serviceStart() throws Exception {
@@ -255,12 +264,15 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
         new Thread() {
           public void run() {
             historyServer.start();
-            jhsStarted = true;
           };
         }.start();
 
-        GenericTestUtils.waitFor(() -> jhsStarted, 1500, 60_000);
-
+        final int launchTimeout = getConfig().getInt(
+            MR_HISTORY_MINICLUSTER_LAUNCH_TIMEOUT_MS, 60_000);
+        GenericTestUtils.waitFor(
+            () -> historyServer.getServiceState() == STATE.STARTED
+                || historyServer.getServiceState() == STATE.STOPPED,
+            100, launchTimeout);
         if (historyServer.getServiceState() != STATE.STARTED) {
           throw new IOException("HistoryServer failed to start");
         }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

Reply via email to