Author: liyin Date: Wed Apr 3 18:19:04 2013 New Revision: 1464136 URL: http://svn.apache.org/r1464136 Log: [HBASE-8245] Throttle region opens
Author: manukranthk Summary: By throttling the region reassignments when a region server come back up after his regions are spread out, we can reduce the spiked in get latencies caused due to multiple region opens on the same region server at round about the same time. Test Plan: Push onto Titan Shadow's master and cause a RegionServer failure and bring him up after a while. By going through the logs we can understand the working of the idea. Reviewers: aaiyer, rshroff, liyintang Reviewed By: rshroff CC: hbase-eng@ Differential Revision: https://phabricator.fb.com/D752483 Task ID: 2227053 Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1464136&r1=1464135&r2=1464136&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Wed Apr 3 18:19:04 2013 @@ -656,6 +656,12 @@ public final class HConstants { public static final String ENABLE_DELETE_COLUMN_BLOOM_FILTER_STRING = "enable_delete_column_bloom_filter"; + // Delaying the region server load balancing by the following amount for a + // load balancing where source is a favored region server. + public static final String HBASE_REGION_ASSIGNMENT_LOADBALANCER_WAITTIME_MS + = "hbase.master.assignment.load.balancer.waittime.ms"; + public static final int DEFAULT_HBASE_REGION_ASSIGNMENT_LOADBALANCER_WAITTIME_MS = 60000; + private HConstants() { // Can't be instantiated with this constructor. } Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=1464136&r1=1464135&r2=1464136&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java Wed Apr 3 18:19:04 2013 @@ -26,8 +26,10 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.NavigableMap; import java.util.Set; import java.util.SortedMap; @@ -192,6 +194,7 @@ public class RegionManager { private boolean stoppedScanners = false; private LegacyRootZNodeUpdater legacyRootZNodeUpdater; + RegionManager(HMaster master) throws IOException { Configuration conf = master.getConfiguration(); @@ -202,7 +205,7 @@ public class RegionManager { this.maxAssignInOneGo = conf.getInt("hbase.regions.percheckin", 10); if (master.shouldAssignRegionsWithFavoredNodes()) { - this.loadBalancer = new AssignmentLoadBalancer(); + this.loadBalancer = new AssignmentLoadBalancer(conf); } else { this.loadBalancer = new DefaultLoadBalancer(); } @@ -1899,8 +1902,15 @@ public class RegionManager { * consider secondary and tertiary preferred hosts if the primary is dead. */ class AssignmentLoadBalancer extends LoadBalancer { - AssignmentLoadBalancer() { + long timeDelta = + HConstants.DEFAULT_HBASE_REGION_ASSIGNMENT_LOADBALANCER_WAITTIME_MS; + Map<HServerAddress, Long> lastAssignedForTime; + + AssignmentLoadBalancer(Configuration conf) { super(); + timeDelta = conf.getLong(HConstants.HBASE_REGION_ASSIGNMENT_LOADBALANCER_WAITTIME_MS, + HConstants.DEFAULT_HBASE_REGION_ASSIGNMENT_LOADBALANCER_WAITTIME_MS); + lastAssignedForTime = new HashMap<HServerAddress, Long>(); } /** @@ -1954,6 +1964,10 @@ public class RegionManager { // Primary server is not alive, try next region. continue; } + if (canOffloadTo(preferences.get(0)) == false) { + // Primary server is not ready to accept a new region. + continue; + } // Primary server is alive, unassign this region. if (unassignRegion(info, region, returnMsgs)) { @@ -1963,6 +1977,7 @@ public class RegionManager { + " from the server " + info.getHostnamePort() + " because the primary server: " + preferences.get(0) + " is a live"); + unassignedFor(preferences.get(0)); return regionsUnassigned; } } @@ -2003,6 +2018,7 @@ public class RegionManager { for (int i = 1; i < preferences.size(); i++) { HServerLoad secondaryLoad = getLoadIfAlive(preferences.get(i)); if (secondaryLoad != null && + canOffloadTo(preferences.get(i)) && secondaryLoad.getNumberOfRegions() < leastLoad) { leastLoad = secondaryLoad.getNumberOfRegions(); leastLoadedSecondary = preferences.get(i); @@ -2014,6 +2030,7 @@ public class RegionManager { // for that region. if (unassignRegion(info, region, returnMsgs)) { regionsUnassigned++; + unassignedFor(leastLoadedSecondary); if (regionsUnassigned >= maxRegToClose && maxRegToClose > 0) { LOG.debug("Unassigned " + region.getRegionNameAsString() + " from the unfavoraed server " + info.getHostnamePort() @@ -2070,11 +2087,17 @@ public class RegionManager { continue; } + if (!canOffloadTo(preferences.get(i))) { + // Other server is not accepting new regions. + continue; + } + // Only move the region if the other server is under-loaded and the // current server is overloaded. if (serverLoad - regionsUnassigned > avgLoadPlusSlop && otherLoad.getNumberOfRegions() < avgLoadMinusSlop) { if (unassignRegion(info, region, returnMsgs)) { + unassignedFor(preferences.get(i)); // Need to override transient assignment that may have been added // for the region to its current server when unassigning. assignmentManager.removeTransientAssignment( @@ -2097,6 +2120,23 @@ public class RegionManager { return regionsUnassigned; } + private boolean canOffloadTo(HServerAddress hServerAddress) { + Map<HServerAddress, Long> map = lastAssignedForTime; + Long ts = map.get(hServerAddress); + long tscur = EnvironmentEdgeManager.currentTimeMillis(); + + if (ts == null || (tscur - ts) > timeDelta) { + return true; + } + return false; + } + + private void unassignedFor(HServerAddress hServerAddress) { + Map<HServerAddress, Long> map = this.lastAssignedForTime; + long currentTime = EnvironmentEdgeManager.currentTimeMillis(); + map.put(hServerAddress, currentTime); + } + /** * Get the load for the region server at the address {@code server} unless * that server does not exist, is dead, or is shutting down. In cases where
