Repository: hbase Updated Branches: refs/heads/0.98 4d86e2249 -> 4e0d26ea2
HBASE-13635 Regions stuck in transition because master is incorrectly assumed dead Summary: Requests to tell master that meta is moved can be blocked behind other requests that are trying to mutate meta. This causes a dead lock and the master is assumed dead. However the master stays up the whole time. This patch adds prioritization onto reporting meta moves. It should allow meta to make progress. Test Plan: unit tests. Differential Revision: https://reviews.facebook.net/D38109 Amending-Author: Andrew Purtell <apurt...@apache.org> Conflicts: hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RWQueueRpcExecutor.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQosFunction.java Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4e0d26ea Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4e0d26ea Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4e0d26ea Branch: refs/heads/0.98 Commit: 4e0d26ea23f91f8e04ddfbaa027ad35fc26697fa Parents: 4d86e22 Author: Elliott Clark <ecl...@apache.org> Authored: Tue May 12 10:08:48 2015 -0700 Committer: Andrew Purtell <apurt...@apache.org> Committed: Tue May 12 10:08:48 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/ipc/RWQueueRpcExecutor.java | 3 ++ .../AnnotationReadingPriorityFunction.java | 23 +++++++- .../hbase/regionserver/TestQosFunction.java | 57 +++++++++++++++++++- 3 files changed, 81 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/4e0d26ea/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RWQueueRpcExecutor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RWQueueRpcExecutor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RWQueueRpcExecutor.java index 9a3d8cd..55889ea 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RWQueueRpcExecutor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RWQueueRpcExecutor.java @@ -138,6 +138,9 @@ public class RWQueueRpcExecutor extends RpcExecutor { } } } + if (methodName.equalsIgnoreCase("ReportRegionStateTransition")) { + return true; + } return false; } http://git-wip-us.apache.org/repos/asf/hbase/blob/4e0d26ea/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java index 009a244..3786af1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java @@ -23,10 +23,15 @@ import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.ipc.PriorityFunction; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest; +import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest; @@ -198,6 +203,22 @@ class AnnotationReadingPriorityFunction implements PriorityFunction { return HConstants.HIGH_QOS; } } + + // If meta is moving then all the rest of report the report state transitions will be + // blocked. We shouldn't be in the same queue. + if (methodName.equalsIgnoreCase("ReportRegionStateTransition")) { // Regions are moving + ReportRegionStateTransitionRequest tRequest = (ReportRegionStateTransitionRequest) param; + for (RegionStateTransition transition : tRequest.getTransitionList()) { + if (transition.getRegionInfoList() != null) { + for (HBaseProtos.RegionInfo info : transition.getRegionInfoList()) { + TableName tn = ProtobufUtil.toTableName(info.getTableName()); + if (tn.isSystemTable()) { + return HConstants.HIGH_QOS; + } + } + } + } + } return HConstants.NORMAL_QOS; } http://git-wip-us.apache.org/repos/asf/hbase/blob/4e0d26ea/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQosFunction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQosFunction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQosFunction.java index ca0dfed..d5f890c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQosFunction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQosFunction.java @@ -18,10 +18,21 @@ package org.apache.hadoop.hbase.regionserver; */ import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest; import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader; -import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; @@ -45,6 +56,50 @@ public class TestQosFunction { checkMethod("OpenRegion", HConstants.HIGH_QOS, qosFunction); // Check multi works. checkMethod("Multi", HConstants.NORMAL_QOS, qosFunction, MultiRequest.getDefaultInstance()); + + } + + @Test + public void testRegionInTransition() { + Configuration conf = HBaseConfiguration.create(); + HRegionServer rs = Mockito.mock(HRegionServer.class); + when(rs.getConfiguration()).thenReturn(conf); + + AnnotationReadingPriorityFunction qosFunction = new AnnotationReadingPriorityFunction(rs); + + // Check ReportRegionInTransition + HBaseProtos.RegionInfo meta_ri = HRegionInfo.convert(HRegionInfo.FIRST_META_REGIONINFO); + HBaseProtos.RegionInfo normal_ri = HRegionInfo.convert( + new HRegionInfo(TableName.valueOf("test:table"), + Bytes.toBytes("a"), Bytes.toBytes("b"), false)); + + + RegionServerStatusProtos.RegionStateTransition metaTransition = RegionServerStatusProtos + .RegionStateTransition.newBuilder() + .addRegionInfo(meta_ri) + .setTransitionCode(RegionServerStatusProtos.RegionStateTransition.TransitionCode.CLOSED) + .build(); + + RegionServerStatusProtos.RegionStateTransition normalTransition = RegionServerStatusProtos + .RegionStateTransition.newBuilder() + .addRegionInfo(normal_ri) + .setTransitionCode(RegionServerStatusProtos.RegionStateTransition.TransitionCode.CLOSED) + .build(); + + RegionServerStatusProtos.ReportRegionStateTransitionRequest metaTransitionRequest = + RegionServerStatusProtos.ReportRegionStateTransitionRequest.newBuilder() + .setServer(ProtobufUtil.toServerName(ServerName.valueOf("locahost:60020", 100))) + .addTransition(normalTransition) + .addTransition(metaTransition).build(); + + RegionServerStatusProtos.ReportRegionStateTransitionRequest normalTransitionRequest = + RegionServerStatusProtos.ReportRegionStateTransitionRequest.newBuilder() + .setServer(ProtobufUtil.toServerName(ServerName.valueOf("locahost:60020", 100))) + .addTransition(normalTransition).build(); + + final String reportFuncName = "ReportRegionStateTransition"; + checkMethod(reportFuncName, HConstants.HIGH_QOS, qosFunction, metaTransitionRequest); + checkMethod(reportFuncName, HConstants.NORMAL_QOS, qosFunction, normalTransitionRequest); } private void checkMethod(final String methodName, final int expected,
