Repository: hbase
Updated Branches:
  refs/heads/branch-2 40cfb2aa8 -> 7a0057179


HBASE-18492 [AMv2] Embed code for selecting highest versioned region server for 
system table regions in AssignmentManager.processAssignQueue()

* Modified AssignmentManager.processAssignQueue() method to consider only 
highest versioned region servers for system table regions when
  destination server is not specified for them. Destination server is retained, 
if specified.
* Modified MoveRegionProcedure to allow null value for destination server i.e. 
moving a region from specific source server to non-specific/ unknown
  destination server (picked by load-balancer) is supported now.
* Removed destination server selection from 
HMaster.checkIfShouldMoveSystemRegionAsync(), as destination server will be 
picked by load balancer

Signed-off-by: Michael Stack <st...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7a005717
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7a005717
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7a005717

Branch: refs/heads/branch-2
Commit: 7a00571795a4e23eaed274f3f8bc542b9e2340dc
Parents: 40cfb2a
Author: Umesh Agashe <uaga...@cloudera.com>
Authored: Sat Jul 15 22:51:05 2017 -0700
Committer: Michael Stack <st...@apache.org>
Committed: Tue Aug 8 14:02:36 2017 -0700

----------------------------------------------------------------------
 .../src/main/protobuf/MasterProcedure.proto     |  3 +-
 .../master/assignment/AssignmentManager.java    | 44 +++++++++++++++-----
 .../master/assignment/MoveRegionProcedure.java  | 15 ++++---
 3 files changed, 46 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/7a005717/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto 
b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index 74ae16d..70753c6 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -387,7 +387,8 @@ enum MoveRegionState {
 message MoveRegionStateData {
   optional RegionInfo region_info = 1;
   required ServerName source_server = 2;
-  required ServerName destination_server = 3;
+  // if destination server not specified, its selected with load balancer
+  optional ServerName destination_server = 3;
 }
 
 enum GCRegionState {

http://git-wip-us.apache.org/repos/asf/hbase/blob/7a005717/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
----------------------------------------------------------------------
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index 255ea5e..54cb1ca 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -478,15 +478,13 @@ public class AssignmentManager implements ServerListener {
     new Thread(() -> {
       try {
         synchronized (checkIfShouldMoveSystemRegionLock) {
-          List<ServerName> serverList = master.getServerManager()
-              
.createDestinationServersList(getExcludedServersForSystemTable());
           List<RegionPlan> plans = new ArrayList<>();
           for (ServerName server : getExcludedServersForSystemTable()) {
             List<HRegionInfo> regionsShouldMove = 
getCarryingSystemTables(server);
             if (!regionsShouldMove.isEmpty()) {
               for (HRegionInfo regionInfo : regionsShouldMove) {
-                RegionPlan plan = new RegionPlan(regionInfo, server,
-                    getBalancer().randomAssignment(regionInfo, serverList));
+                // null value for dest forces destination server to be 
selected by balancer
+                RegionPlan plan = new RegionPlan(regionInfo, server, null);
                 if (regionInfo.isMetaRegion()) {
                   // Must move meta region first.
                   moveAsync(plan);
@@ -1648,9 +1646,14 @@ public class AssignmentManager implements ServerListener 
{
     }
 
     // TODO: Optimize balancer. pass a RegionPlan?
-    final HashMap<HRegionInfo, ServerName> retainMap = new 
HashMap<HRegionInfo, ServerName>();
-    final List<HRegionInfo> rrList = new ArrayList<HRegionInfo>();
-    for (RegionStateNode regionNode: regions.values()) {
+    final HashMap<HRegionInfo, ServerName> retainMap = new HashMap<>();
+    final List<HRegionInfo> userRRList = new ArrayList<>();
+    // regions for system tables requiring reassignment
+    final List<HRegionInfo> sysRRList = new ArrayList<>();
+    for (RegionStateNode regionNode : regions.values()) {
+      boolean sysTable = regionNode.isSystemTable();
+      final List<HRegionInfo> rrList = sysTable ? sysRRList : userRRList;
+
       if (regionNode.getRegionLocation() != null) {
         retainMap.put(regionNode.getRegionInfo(), 
regionNode.getRegionLocation());
       } else {
@@ -1659,7 +1662,6 @@ public class AssignmentManager implements ServerListener {
     }
 
     // TODO: connect with the listener to invalidate the cache
-    final LoadBalancer balancer = getBalancer();
 
     // TODO use events
     List<ServerName> servers = 
master.getServerManager().createDestinationServersList();
@@ -1679,13 +1681,35 @@ public class AssignmentManager implements 
ServerListener {
       servers = master.getServerManager().createDestinationServersList();
     }
 
-    final boolean isTraceEnabled = LOG.isTraceEnabled();
+    if (!sysRRList.isEmpty()) {
+      // system table regions requiring reassignment are present, get region 
servers
+      // not available for system table regions
+      final List<ServerName> excludeServers = 
getExcludedServersForSystemTable();
+      List<ServerName> serversForSysTables = servers.stream()
+          .filter(s -> 
!excludeServers.contains(s)).collect(Collectors.toList());
+      if (serversForSysTables.isEmpty()) {
+        LOG.warn("No servers available for system table regions, considering 
all servers!");
+      }
+      LOG.debug("Processing assignment plans for System tables 
sysServersCount=" +
+          serversForSysTables.size() + ", allServersCount=" + servers.size());
+      processAssignmentPlans(regions, null, sysRRList,
+          serversForSysTables.isEmpty() ? servers : serversForSysTables);
+    }
+
+    processAssignmentPlans(regions, retainMap, userRRList, servers);
+  }
+
+  private void processAssignmentPlans(final HashMap<HRegionInfo, 
RegionStateNode> regions,
+      final HashMap<HRegionInfo, ServerName> retainMap, final 
List<HRegionInfo> rrList,
+      final List<ServerName> servers) {
+    boolean isTraceEnabled = LOG.isTraceEnabled();
     if (isTraceEnabled) {
       LOG.trace("available servers count=" + servers.size() + ": " + servers);
     }
 
+    final LoadBalancer balancer = getBalancer();
     // ask the balancer where to place regions
-    if (!retainMap.isEmpty()) {
+    if (retainMap != null && !retainMap.isEmpty()) {
       if (isTraceEnabled) {
         LOG.trace("retain assign regions=" + retainMap);
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/7a005717/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java
----------------------------------------------------------------------
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java
index d8c1b7d..1907e98 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MoveRegionProcedure.java
@@ -54,7 +54,6 @@ public class MoveRegionProcedure extends 
AbstractStateMachineRegionProcedure<Mov
 
   public MoveRegionProcedure(final MasterProcedureEnv env, final RegionPlan 
plan) {
     super(env, plan.getRegionInfo());
-    assert plan.getDestination() != null: plan.toString();
     this.plan = plan;
   }
 
@@ -70,7 +69,10 @@ public class MoveRegionProcedure extends 
AbstractStateMachineRegionProcedure<Mov
         setNextState(MoveRegionState.MOVE_REGION_ASSIGN);
         break;
       case MOVE_REGION_ASSIGN:
-        addChildProcedure(new AssignProcedure(plan.getRegionInfo(), 
plan.getDestination()));
+        AssignProcedure assignProcedure = plan.getDestination() == null ?
+            new AssignProcedure(plan.getRegionInfo(), true) :
+            new AssignProcedure(plan.getRegionInfo(), plan.getDestination());
+        addChildProcedure(assignProcedure);
         return Flow.NO_MORE_STATE;
       default:
         throw new UnsupportedOperationException("unhandled state=" + state);
@@ -127,8 +129,10 @@ public class MoveRegionProcedure extends 
AbstractStateMachineRegionProcedure<Mov
 
     final MoveRegionStateData.Builder state = MoveRegionStateData.newBuilder()
         // No need to serialize the HRegionInfo. The super class has the 
region.
-        .setSourceServer(ProtobufUtil.toServerName(plan.getSource()))
-        
.setDestinationServer(ProtobufUtil.toServerName(plan.getDestination()));
+        .setSourceServer(ProtobufUtil.toServerName(plan.getSource()));
+    if (plan.getDestination() != null) {
+      
state.setDestinationServer(ProtobufUtil.toServerName(plan.getDestination()));
+    }
     state.build().writeDelimitedTo(stream);
   }
 
@@ -139,7 +143,8 @@ public class MoveRegionProcedure extends 
AbstractStateMachineRegionProcedure<Mov
     final MoveRegionStateData state = 
MoveRegionStateData.parseDelimitedFrom(stream);
     final HRegionInfo regionInfo = getRegion(); // Get it from super class 
deserialization.
     final ServerName sourceServer = 
ProtobufUtil.toServerName(state.getSourceServer());
-    final ServerName destinationServer = 
ProtobufUtil.toServerName(state.getDestinationServer());
+    final ServerName destinationServer = state.hasDestinationServer() ?
+        ProtobufUtil.toServerName(state.getDestinationServer()) : null;
     this.plan = new RegionPlan(regionInfo, sourceServer, destinationServer);
   }
 }
\ No newline at end of file

Reply via email to