Repository: hbase Updated Branches: refs/heads/branch-1 1cb82d911 -> d0bd4903c
HBASE-15463 Region normalizer should check whether split/merge is enabled Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d0bd4903 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d0bd4903 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d0bd4903 Branch: refs/heads/branch-1 Commit: d0bd4903c6c2cac8ede006fff9b00b01f4c726d1 Parents: 1cb82d9 Author: tedyu <[email protected]> Authored: Tue Mar 15 09:10:00 2016 -0700 Committer: tedyu <[email protected]> Committed: Tue Mar 15 09:10:00 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/master/HMaster.java | 1 + .../master/normalizer/RegionNormalizer.java | 8 +++ .../normalizer/SimpleRegionNormalizer.java | 59 ++++++++++++++------ .../normalizer/TestSimpleRegionNormalizer.java | 17 ++++++ 4 files changed, 69 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/d0bd4903/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 3fa0df3..05118f8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -576,6 +576,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { this.balancer = LoadBalancerFactory.getLoadBalancer(conf); this.normalizer = RegionNormalizerFactory.getRegionNormalizer(conf); this.normalizer.setMasterServices(this); + this.normalizer.setMasterRpcServices((MasterRpcServices)rpcServices); this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this); this.loadBalancerTracker.start(); http://git-wip-us.apache.org/repos/asf/hbase/blob/d0bd4903/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java index c5a1241..e5723db 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.master.MasterRpcServices; import org.apache.hadoop.hbase.master.MasterServices; /** @@ -45,6 +46,13 @@ public interface RegionNormalizer { void setMasterServices(MasterServices masterServices); /** + * Set the master RPC service. Must be called before first call to + * {@link #computePlanForTable(TableName)}. + * @param masterRpcServices master RPC services to use + */ + void setMasterRpcServices(MasterRpcServices masterRpcServices); + + /** * Computes next optimal normalization plan. * @param table table to normalize * @return normalization actions to perform. Null if no action to take http://git-wip-us.apache.org/repos/asf/hbase/blob/d0bd4903/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index 56032d6..67aaee1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -18,6 +18,11 @@ */ package org.apache.hadoop.hbase.master.normalizer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HBaseIOException; @@ -26,13 +31,12 @@ import org.apache.hadoop.hbase.RegionLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.Admin.MasterSwitchType; +import org.apache.hadoop.hbase.master.MasterRpcServices; import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; +import org.apache.hadoop.hbase.protobuf.RequestConverter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; +import com.google.protobuf.ServiceException; /** * Simple implementation of region normalizer. @@ -59,6 +63,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer { private static final Log LOG = LogFactory.getLog(SimpleRegionNormalizer.class); private static final int MIN_REGION_COUNT = 3; private MasterServices masterServices; + private MasterRpcServices masterRpcServices; /** * Set the master service. @@ -84,6 +89,11 @@ public class SimpleRegionNormalizer implements RegionNormalizer { } }; + @Override + public void setMasterRpcServices(MasterRpcServices masterRpcServices) { + this.masterRpcServices = masterRpcServices; + } + /** * Computes next most "urgent" normalization action on the table. * Action may be either a split, or a merge, or no action. @@ -127,27 +137,44 @@ public class SimpleRegionNormalizer implements RegionNormalizer { LOG.debug("Table " + table + ", average region size: " + avgRegionSize); int candidateIdx = 0; + boolean splitEnabled = true, mergeEnabled = true; + try { + splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null, + RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled(); + } catch (ServiceException se) { + LOG.debug("Unable to determine whether split is enabled", se); + } + try { + mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null, + RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled(); + } catch (ServiceException se) { + LOG.debug("Unable to determine whether split is enabled", se); + } while (candidateIdx < tableRegions.size()) { HRegionInfo hri = tableRegions.get(candidateIdx); long regionSize = getRegionSize(hri); // if the region is > 2 times larger than average, we split it, split // is more high priority normalization action than merge. if (regionSize > 2 * avgRegionSize) { - LOG.info("Table " + table + ", large region " + hri.getRegionNameAsString() + " has size " - + regionSize + ", more than twice avg size, splitting"); - plans.add(new SplitNormalizationPlan(hri, null)); + if (splitEnabled) { + LOG.info("Table " + table + ", large region " + hri.getRegionNameAsString() + " has size " + + regionSize + ", more than twice avg size, splitting"); + plans.add(new SplitNormalizationPlan(hri, null)); + } } else { if (candidateIdx == tableRegions.size()-1) { break; } - HRegionInfo hri2 = tableRegions.get(candidateIdx+1); - long regionSize2 = getRegionSize(hri2); - if (regionSize + regionSize2 < avgRegionSize) { - LOG.info("Table " + table + ", small region size: " + regionSize - + " plus its neighbor size: " + regionSize2 - + ", less than the avg size " + avgRegionSize + ", merging them"); - plans.add(new MergeNormalizationPlan(hri, hri2)); - candidateIdx++; + if (mergeEnabled) { + HRegionInfo hri2 = tableRegions.get(candidateIdx+1); + long regionSize2 = getRegionSize(hri2); + if (regionSize + regionSize2 < avgRegionSize) { + LOG.info("Table " + table + ", small region size: " + regionSize + + " plus its neighbor size: " + regionSize2 + + ", less than the avg size " + avgRegionSize + ", merging them"); + plans.add(new MergeNormalizationPlan(hri, hri2)); + candidateIdx++; + } } } candidateIdx++; http://git-wip-us.apache.org/repos/asf/hbase/blob/d0bd4903/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java index 2feadbf..fad2d33 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java @@ -25,7 +25,11 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.RegionLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.MasterRpcServices; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.protobuf.RequestConverter; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledResponse; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.BeforeClass; @@ -33,6 +37,9 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -55,6 +62,7 @@ public class TestSimpleRegionNormalizer { // mocks private static MasterServices masterServices; + private static MasterRpcServices masterRpcServices; @BeforeClass public static void beforeAllTests() throws Exception { @@ -258,6 +266,7 @@ public class TestSimpleRegionNormalizer { protected void setupMocksForNormalizer(Map<byte[], Integer> regionSizes, List<HRegionInfo> hris) { masterServices = Mockito.mock(MasterServices.class, RETURNS_DEEP_STUBS); + masterRpcServices = Mockito.mock(MasterRpcServices.class, RETURNS_DEEP_STUBS); // for simplicity all regions are assumed to be on one server; doesn't matter to us ServerName sn = ServerName.valueOf("localhost", -1, 1L); @@ -274,7 +283,15 @@ public class TestSimpleRegionNormalizer { when(masterServices.getServerManager().getLoad(sn). getRegionsLoad().get(region.getKey())).thenReturn(regionLoad); } + try { + when(masterRpcServices.isSplitOrMergeEnabled(any(RpcController.class), + any(IsSplitOrMergeEnabledRequest.class))).thenReturn( + IsSplitOrMergeEnabledResponse.newBuilder().setEnabled(true).build()); + } catch (ServiceException se) { + LOG.debug("error setting isSplitOrMergeEnabled switch", se); + } normalizer.setMasterServices(masterServices); + normalizer.setMasterRpcServices(masterRpcServices); } }
