HBASE-20548 Master fails to startup on large clusters, refreshing block 
distribution

Signed-off-by: Andrew Purtell <apurt...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b892be74
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b892be74
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b892be74

Branch: refs/heads/branch-1
Commit: b892be7440afde8da8a2db8fe3053af3ba4daedf
Parents: 416f283
Author: Thiruvel Thirumoolan <thiru...@oath.com>
Authored: Tue May 22 18:28:31 2018 -0700
Committer: Andrew Purtell <apurt...@apache.org>
Committed: Thu May 24 12:27:19 2018 -0700

----------------------------------------------------------------------
 .../hbase/rsgroup/RSGroupBasedLoadBalancer.java |  5 ++++
 .../org/apache/hadoop/hbase/master/HMaster.java | 11 +++++++++
 .../hadoop/hbase/master/LoadBalancer.java       |  5 ++++
 .../hbase/master/balancer/BaseLoadBalancer.java | 26 +++++++++++++-------
 4 files changed, 38 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/b892be74/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
----------------------------------------------------------------------
diff --git 
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
 
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
index ebae030..77adf0d 100644
--- 
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
+++ 
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
@@ -455,4 +455,9 @@ public class RSGroupBasedLoadBalancer implements 
RSGroupableBalancer, LoadBalanc
   public boolean isStopped() {
     return false;
   }
+
+  @Override
+  public void postMasterStartupInitialize() {
+    this.internalBalancer.postMasterStartupInitialize();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b892be74/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 85d5e23..a2b43e9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -933,6 +933,17 @@ public class HMaster extends HRegionServer implements 
MasterServices, Server {
     }
 
     zombieDetector.interrupt();
+
+    /*
+     * After master has started up, lets do balancer post startup 
initialization. Since this runs
+     * in activeMasterManager thread, it should be fine.
+     */
+    long start = System.currentTimeMillis();
+    this.balancer.postMasterStartupInitialize();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Balancer post startup initialization complete, took " + (
+          (System.currentTimeMillis() - start) / 1000) + " seconds");
+    }
   }
 
   private void initQuotaManager() throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/b892be74/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
----------------------------------------------------------------------
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
index e387f59..8531ff7 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
@@ -152,4 +152,9 @@ public interface LoadBalancer extends Configurable, 
Stoppable, ConfigurationObse
    */
   @Override
   void onConfigurationChange(Configuration conf);
+
+  /**
+   * If balancer needs to do initialization after Master has started up, lets 
do that here.
+   */
+  void postMasterStartupInitialize();
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b892be74/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
----------------------------------------------------------------------
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index 41cbeaa..01df8a2 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -1267,6 +1267,19 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
     }
   }
 
+  @Override
+  public void postMasterStartupInitialize() {
+    if (services != null && regionFinder != null) {
+      try {
+        Set<HRegionInfo> regions =
+            
services.getAssignmentManager().getRegionStates().getRegionAssignments().keySet();
+        regionFinder.refreshAndWait(regions);
+      } catch (Exception e) {
+        LOG.warn("Refreshing region HDFS Block dist failed with exception, 
ignoring", e);
+      }
+    }
+  }
+
   public void setRackManager(RackManager rackManager) {
     this.rackManager = rackManager;
   }
@@ -1367,7 +1380,7 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
       return assignments;
     }
 
-    Cluster cluster = createCluster(servers, regions, false);
+    Cluster cluster = createCluster(servers, regions);
     List<HRegionInfo> unassignedRegions = new ArrayList<HRegionInfo>();
 
     roundRobinAssignment(cluster, regions, unassignedRegions,
@@ -1413,11 +1426,7 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
     return assignments;
   }
 
-  protected Cluster createCluster(List<ServerName> servers,
-      Collection<HRegionInfo> regions, boolean forceRefresh) {
-    if (forceRefresh && useRegionFinder) {
-      regionFinder.refreshAndWait(regions);
-    }
+  protected Cluster createCluster(List<ServerName> servers, 
Collection<HRegionInfo> regions) {
     // Get the snapshot of the current assignments for the regions in 
question, and then create
     // a cluster out of it. Note that we might have replicas already assigned 
to some servers
     // earlier. So we want to get the snapshot to see those assignments, but 
this will only contain
@@ -1491,7 +1500,7 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
     }
 
     List<HRegionInfo> regions = Lists.newArrayList(regionInfo);
-    Cluster cluster = createCluster(servers, regions, false);
+    Cluster cluster = createCluster(servers, regions);
     return randomAssignment(cluster, regionInfo, servers);
   }
 
@@ -1569,8 +1578,6 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
     int numRandomAssignments = 0;
     int numRetainedAssigments = 0;
 
-    Cluster cluster = createCluster(servers, regions.keySet(), true);
-
     for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) {
       HRegionInfo region = entry.getKey();
       ServerName oldServerName = entry.getValue();
@@ -1613,6 +1620,7 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 
     // If servers from prior assignment aren't present, then lets do 
randomAssignment on regions.
     if (randomAssignRegions.size() > 0) {
+      Cluster cluster = createCluster(servers, regions.keySet());
       for (Map.Entry<ServerName, List<HRegionInfo>> entry : 
assignments.entrySet()) {
         ServerName sn = entry.getKey();
         for (HRegionInfo region : entry.getValue()) {

Reply via email to