Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java?rev=1581479&r1=1581478&r2=1581479&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java Tue Mar 25 19:34:52 2014 @@ -150,6 +150,7 @@ public class StochasticLoadBalancer exte costFunctions = new CostFunction[]{ new RegionCountSkewCostFunction(conf), + new RegionOnMasterCostFunction(conf), new MoveCostFunction(conf), localityCost, new TableSkewCostFunction(conf), @@ -191,14 +192,18 @@ public class StochasticLoadBalancer exte */ @Override public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) { - if (!needsBalance(new ClusterLoadState(clusterState))) { + List<RegionPlan> plans = balanceMasterRegions(clusterState); + if (plans != null) { + return plans; + } + if (!needsBalance(new ClusterLoadState(masterServerName, clusterState))) { return null; } long startTime = EnvironmentEdgeManager.currentTimeMillis(); // Keep track of servers to iterate through them. - Cluster cluster = new Cluster(clusterState, loads, regionFinder); + Cluster cluster = new Cluster(masterServerName, clusterState, loads, regionFinder); double currentCost = computeCost(cluster, Double.MAX_VALUE); double initCost = currentCost; @@ -257,7 +262,7 @@ public class StochasticLoadBalancer exte metricsBalancer.balanceCluster(endTime - startTime); if (initCost > currentCost) { - List<RegionPlan> plans = createRegionPlans(cluster); + plans = createRegionPlans(cluster); if (LOG.isDebugEnabled()) { LOG.debug("Finished computing new load balance plan. Computation took " + (endTime - startTime) + "ms to try " + step @@ -563,8 +568,8 @@ public class StochasticLoadBalancer exte */ public abstract static class CostFunction { - private float multiplier = 0; - private Configuration conf; + protected float multiplier = 0; + protected Configuration conf; CostFunction(Configuration c) { this.conf = c; @@ -742,6 +747,29 @@ public class StochasticLoadBalancer exte } } + /** + * Compute the cost of a potential cluster configuration based upon if putting + * user regions on the master regionserver. + */ + public static class RegionOnMasterCostFunction extends CostFunction { + + private static final String REGION_ON_MASTER_COST_KEY = + "hbase.master.balancer.stochastic.regionOnMasterCost"; + private static final float DEFAULT_REGION_ON_MASTER__COST = 1000; + + RegionOnMasterCostFunction(Configuration conf) { + super(conf); + this.setMultiplier(conf.getFloat( + REGION_ON_MASTER_COST_KEY, DEFAULT_REGION_ON_MASTER__COST)); + } + + @Override + double cost(Cluster cluster) { + double max = cluster.numRegions; + double value = cluster.numUserRegionsOnMaster; + return scale(0, max, value); + } + } /** * Compute a cost of a potential cluster configuration based upon where
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java Tue Mar 25 19:34:52 2014 @@ -148,7 +148,7 @@ public class CreateTableHandler extends LOG.info("Create table " + tableName); try { - MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost(); + MasterCoprocessorHost cpHost = ((HMaster) this.server).getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preCreateTableHandler(this.hTableDescriptor, this.newRegions); } Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java Tue Mar 25 19:34:52 2014 @@ -61,7 +61,7 @@ public class DeleteTableHandler extends protected void handleTableOperation(List<HRegionInfo> regions) throws IOException, KeeperException { MasterCoprocessorHost cpHost = ((HMaster) this.server) - .getCoprocessorHost(); + .getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preDeleteTableHandler(this.tableName); } Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java Tue Mar 25 19:34:52 2014 @@ -129,7 +129,7 @@ public class DisableTableHandler extends try { LOG.info("Attempting to disable table " + this.tableName); MasterCoprocessorHost cpHost = ((HMaster) this.server) - .getCoprocessorHost(); + .getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preDisableTableHandler(this.tableName); } Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java Tue Mar 25 19:34:52 2014 @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutorService; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -48,7 +47,6 @@ import org.apache.hadoop.hbase.master.Ta import org.apache.hadoop.hbase.master.TableLockManager.TableLock; import org.apache.hadoop.hbase.util.Pair; import org.apache.zookeeper.KeeperException; -import org.htrace.Trace; /** * Handler to run enable of a table. @@ -139,7 +137,7 @@ public class EnableTableHandler extends try { LOG.info("Attempting to enable the table " + this.tableName); MasterCoprocessorHost cpHost = ((HMaster) this.server) - .getCoprocessorHost(); + .getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preEnableTableHandler(this.tableName); } Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java Tue Mar 25 19:34:52 2014 @@ -60,7 +60,7 @@ public class ModifyTableHandler extends @Override protected void handleTableOperation(List<HRegionInfo> hris) throws IOException { - MasterCoprocessorHost cpHost = ((HMaster) this.server).getCoprocessorHost(); + MasterCoprocessorHost cpHost = ((HMaster) this.server).getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preModifyTableHandler(this.tableName, this.htd); } Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java Tue Mar 25 19:34:52 2014 @@ -61,7 +61,7 @@ public class TableAddFamilyHandler exten protected void handleTableOperation(List<HRegionInfo> hris) throws IOException { MasterCoprocessorHost cpHost = ((HMaster) this.server) - .getCoprocessorHost(); + .getMasterCoprocessorHost(); if(cpHost != null){ cpHost.preAddColumnHandler(this.tableName, this.familyDesc); } Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java Tue Mar 25 19:34:52 2014 @@ -57,7 +57,7 @@ public class TableDeleteFamilyHandler ex @Override protected void handleTableOperation(List<HRegionInfo> hris) throws IOException { MasterCoprocessorHost cpHost = ((HMaster) this.server) - .getCoprocessorHost(); + .getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preDeleteColumnHandler(this.tableName, this.familyName); } Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java?rev=1581479&r1=1581478&r2=1581479&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java Tue Mar 25 19:34:52 2014 @@ -56,7 +56,7 @@ public class TableModifyFamilyHandler ex @Override protected void handleTableOperation(List<HRegionInfo> regions) throws IOException { MasterCoprocessorHost cpHost = ((HMaster) this.server) - .getCoprocessorHost(); + .getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preModifyColumnHandler(this.tableName, this.familyDesc); } Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java?rev=1581479&r1=1581478&r2=1581479&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java Tue Mar 25 19:34:52 2014 @@ -137,7 +137,6 @@ public class SnapshotManager extends Mas private boolean stopped; private MasterServices master; // Needed by TableEventHandlers - private MetricsMaster metricsMaster; private ProcedureCoordinator coordinator; // Is snapshot feature enabled? @@ -172,7 +171,6 @@ public class SnapshotManager extends Mas ProcedureCoordinator coordinator, ExecutorService pool) throws IOException, UnsupportedOperationException { this.master = master; - this.metricsMaster = metricsMaster; this.rootDir = master.getMasterFileSystem().getRootDir(); checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem()); @@ -260,7 +258,7 @@ public class SnapshotManager extends Mas public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException { // call coproc pre hook - MasterCoprocessorHost cpHost = master.getCoprocessorHost(); + MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preDeleteSnapshot(snapshot); } @@ -547,7 +545,7 @@ public class SnapshotManager extends Mas .build(); // call pre coproc hook - MasterCoprocessorHost cpHost = master.getCoprocessorHost(); + MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preSnapshot(snapshot, desc); } @@ -668,7 +666,7 @@ public class SnapshotManager extends Mas public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException { FileSystem fs = master.getMasterFileSystem().getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir); - MasterCoprocessorHost cpHost = master.getCoprocessorHost(); + MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost(); // check if the snapshot exists if (!fs.exists(snapshotDir)) { @@ -1005,7 +1003,6 @@ public class SnapshotManager extends Mas public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException, IOException, UnsupportedOperationException { this.master = master; - this.metricsMaster = metricsMaster; this.rootDir = master.getMasterFileSystem().getRootDir(); checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem()); Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java?rev=1581479&r1=1581478&r2=1581479&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java Tue Mar 25 19:34:52 2014 @@ -26,6 +26,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorStatus; import org.apache.hadoop.hbase.util.VersionInfo; @@ -45,6 +46,10 @@ public abstract class StateDumpServlet e " on " + org.apache.hadoop.util.VersionInfo.getDate()); } + protected boolean isShowQueueDump(Configuration conf){ + return conf.getBoolean("hbase.regionserver.servlet.show.queuedump", true); + } + protected long getTailKbParam(HttpServletRequest request) { String param = request.getParameter("tailkb"); if (param == null) { Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java?rev=1581479&r1=1581478&r2=1581479&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java Tue Mar 25 19:34:52 2014 @@ -24,7 +24,6 @@ import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.ipc.PriorityFunction; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest; @@ -38,7 +37,7 @@ import org.apache.hadoop.hbase.protobuf. import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier; import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader; -import org.apache.hadoop.hbase.regionserver.HRegionServer.QosPriority; +import org.apache.hadoop.hbase.regionserver.RSRpcServices.QosPriority; import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.Message; @@ -72,7 +71,7 @@ class AnnotationReadingPriorityFunction private final Map<String, Integer> annotatedQos; //We need to mock the regionserver instance for some unit tests (set via //setRegionServer method. - private HRegionServer hRegionServer; + private RSRpcServices rpcServices; @SuppressWarnings("unchecked") private final Class<? extends Message>[] knownArgumentClasses = new Class[]{ GetRegionInfoRequest.class, @@ -92,10 +91,9 @@ class AnnotationReadingPriorityFunction private final Map<String, Map<Class<? extends Message>, Method>> methodMap = new HashMap<String, Map<Class<? extends Message>, Method>>(); - AnnotationReadingPriorityFunction(final HRegionServer hrs) { - this.hRegionServer = hrs; + AnnotationReadingPriorityFunction(final RSRpcServices rpcServices) { Map<String, Integer> qosMap = new HashMap<String, Integer>(); - for (Method m : HRegionServer.class.getMethods()) { + for (Method m : RSRpcServices.class.getMethods()) { QosPriority p = m.getAnnotation(QosPriority.class); if (p != null) { // Since we protobuf'd, and then subsequently, when we went with pb style, method names @@ -107,6 +105,7 @@ class AnnotationReadingPriorityFunction qosMap.put(capitalizedMethodName, p.priority()); } } + this.rpcServices = rpcServices; this.annotatedQos = qosMap; if (methodMap.get("getRegion") == null) { methodMap.put("hasRegion", new HashMap<Class<? extends Message>, Method>()); @@ -129,16 +128,6 @@ class AnnotationReadingPriorityFunction return strBuilder.toString(); } - public boolean isMetaRegion(byte[] regionName) { - HRegion region; - try { - region = hRegionServer.getRegion(regionName); - } catch (NotServingRegionException ignored) { - return false; - } - return region.getRegionInfo().isMetaTable(); - } - @Override public int getPriority(RequestHeader header, Message param) { String methodName = header.getMethodName(); @@ -167,7 +156,7 @@ class AnnotationReadingPriorityFunction if (hasRegion != null && (Boolean)hasRegion.invoke(param, (Object[])null)) { Method getRegion = methodMap.get("getRegion").get(rpcArgClass); regionSpecifier = (RegionSpecifier)getRegion.invoke(param, (Object[])null); - HRegion region = hRegionServer.getRegion(regionSpecifier); + HRegion region = rpcServices.getRegion(regionSpecifier); if (region.getRegionInfo().isMetaTable()) { if (LOG.isTraceEnabled()) { LOG.trace("High priority because region=" + region.getRegionNameAsString()); @@ -187,7 +176,7 @@ class AnnotationReadingPriorityFunction if (!request.hasScannerId()) { return HConstants.NORMAL_QOS; } - RegionScanner scanner = hRegionServer.getScanner(request.getScannerId()); + RegionScanner scanner = rpcServices.getScanner(request.getScannerId()); if (scanner != null && scanner.getRegionInfo().isMetaRegion()) { if (LOG.isTraceEnabled()) { // Scanner requests are small in size so TextFormat version should not overwhelm log. @@ -201,6 +190,6 @@ class AnnotationReadingPriorityFunction @VisibleForTesting void setRegionServer(final HRegionServer hrs) { - this.hRegionServer = hrs; + this.rpcServices = hrs.getRSRpcServices(); } }
