Ted Yu created HBASE-8687:
-----------------------------
Summary: When moving region with region_mover.rb, there is long
stack trace for RegionMovedException
Key: HBASE-8687
URL: https://issues.apache.org/jira/browse/HBASE-8687
Project: HBase
Issue Type: Bug
Reporter: Ted Yu
Priority: Minor
When gracefully rolling restart region servers, I saw the following in output:
{code}
2013-06-04 20:44:40,135 DEBUG [main] client.ClientScanner: Scan
table=usertable, startRow=user8129671889902366092
2013-06-04 20:44:40,141 DEBUG [main] client.ClientScanner: Scan table=.META.,
startRow=usertable,user8129671889902366092,00000000000000
2013-06-04 20:44:40,158 INFO [main] region_mover: Moving region
13168d8b86f1ace9472f60555207a707 (2 of 2) to
server=hor8n09.gq1.ygridcore.net,60020,1370378675859
2013-06-04 20:44:40,405 DEBUG [main] client.ClientScanner: Scan
table=usertable, startRow=user8129671889902366092
2013-06-04 20:44:40,407 WARN [main] client.ServerCallable: Call exception,
tries=0, numRetries=100
org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to:
hostname=hor8n09.gq1.ygridcore.net port=60020 startCode=1370378675859. As of
locationSeqNum=194375.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at
org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
at
org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95)
at
org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:230)
at
org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:299)
at
org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:147)
at
org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:55)
at
org.apache.hadoop.hbase.client.ServerCallable.withRetries(ServerCallable.java:174)
at
org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:215)
at
org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:130)
at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:585)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.jruby.javasupport.JavaMethod.invokeDirectWithExceptionHandling(JavaMethod.java:450)
at org.jruby.javasupport.JavaMethod.invokeDirect(JavaMethod.java:311)
at
org.jruby.java.invokers.InstanceMethodInvoker.call(InstanceMethodInvoker.java:59)
at
org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:167)
at
homes.hortonzy.hbase_minus_0_dot_95_dot_1.bin.region_mover.method__6$RUBY$isSuccessfulScan(/homes/hortonzy/hbase-0.95.1/bin/region_mover.rb:121)
at
homes$hortonzy$hbase_minus_0_dot_95_dot_1$bin$region_mover$method__6$RUBY$isSuccessfulScan.call(homes$hortonzy$hbase_minus_0_dot_95_dot_1$bin$region_mover$method__6$RUBY$isSuccessfulScan:65535)
at
homes$hortonzy$hbase_minus_0_dot_95_dot_1$bin$region_mover$method__6$RUBY$isSuccessfulScan.call(homes$hortonzy$hbase_minus_0_dot_95_dot_1$bin$region_mover$method__6$RUBY$isSuccessfulScan:65535)
at
org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:201)
at
homes.hortonzy.hbase_minus_0_dot_95_dot_1.bin.region_mover.method__8$RUBY$move(/homes/hortonzy/hbase-0.95.1/bin/region_mover.rb:164)
at
homes$hortonzy$hbase_minus_0_dot_95_dot_1$bin$region_mover$method__8$RUBY$move.call(homes$hortonzy$hbase_minus_0_dot_95_dot_1$bin$region_mover$method__8$RUBY$move:65535)
at
org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:181)
at
org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:69)
at
homes.hortonzy.hbase_minus_0_dot_95_dot_1.bin.region_mover.block_6$RUBY$__for__(/homes/hortonzy/hbase-0.95.1/bin/region_mover.rb:381)
at
homes$hortonzy$hbase_minus_0_dot_95_dot_1$bin$region_mover$block_6$RUBY$__for__.call(homes$hortonzy$hbase_minus_0_dot_95_dot_1$bin$region_mover$block_6$RUBY$__for__:65535)
at org.jruby.runtime.CompiledBlock.yield(CompiledBlock.java:112)
at org.jruby.runtime.CompiledBlock.yield(CompiledBlock.java:95)
at org.jruby.runtime.Block.yield(Block.java:130)
at org.jruby.ast.YieldNode.interpret(YieldNode.java:119)
at org.jruby.ast.NewlineNode.interpret(NewlineNode.java:104)
at org.jruby.ast.WhileNode.interpret(WhileNode.java:131)
at org.jruby.ast.NewlineNode.interpret(NewlineNode.java:104)
at org.jruby.ast.BlockNode.interpret(BlockNode.java:71)
at
org.jruby.evaluator.ASTInterpreter.INTERPRET_METHOD(ASTInterpreter.java:74)
at
org.jruby.internal.runtime.methods.InterpretedMethod.call(InterpretedMethod.java:169)
at
org.jruby.internal.runtime.methods.DefaultMethod.call(DefaultMethod.java:191)
at
org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:302)
at
org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:144)
at
org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:153)
at
homes.hortonzy.hbase_minus_0_dot_95_dot_1.bin.region_mover.method__27$RUBY$loadRegions(/homes/hortonzy/hbase-0.95.1/bin/region_mover.rb:363)
at
homes$hortonzy$hbase_minus_0_dot_95_dot_1$bin$region_mover$method__27$RUBY$loadRegions.call(homes$hortonzy$hbase_minus_0_dot_95_dot_1$bin$region_mover$method__27$RUBY$loadRegions:65535)
at
homes$hortonzy$hbase_minus_0_dot_95_dot_1$bin$region_mover$method__27$RUBY$loadRegions.call(homes$hortonzy$hbase_minus_0_dot_95_dot_1$bin$region_mover$method__27$RUBY$loadRegions:65535)
at
org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:332)
at
org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:203)
at
homes.hortonzy.hbase_minus_0_dot_95_dot_1.bin.region_mover.__file__(/homes/hortonzy/hbase-0.95.1/bin/region_mover.rb:454)
at
homes.hortonzy.hbase_minus_0_dot_95_dot_1.bin.region_mover.load(/homes/hortonzy/hbase-0.95.1/bin/region_mover.rb)
at org.jruby.Ruby.runScript(Ruby.java:697)
at org.jruby.Ruby.runScript(Ruby.java:690)
at org.jruby.Ruby.runNormally(Ruby.java:597)
at org.jruby.Ruby.runFromMain(Ruby.java:446)
at org.jruby.Main.doRunFromMain(Main.java:369)
at org.jruby.Main.internalRun(Main.java:258)
at org.jruby.Main.run(Main.java:224)
at org.jruby.Main.run(Main.java:208)
at org.jruby.Main.main(Main.java:188)
Caused by:
org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.RegionMovedException):
org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to:
hostname=hor8n09.gq1.ygridcore.net port=60020 startCode=1370378675859. As of
locationSeqNum=194375.
at
org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2550)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3844)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.scan(HRegionServer.java:2997)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:20932)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2122)
at
org.apache.hadoop.hbase.ipc.RpcServer$Handler.run(RpcServer.java:1829)
at org.apache.hadoop.hbase.ipc.RpcClient.call(RpcClient.java:1336)
at
org.apache.hadoop.hbase.ipc.RpcClient.callBlockingMethod(RpcClient.java:1540)
at
org.apache.hadoop.hbase.ipc.RpcClient$BlockingRpcChannelImplementation.callBlockingMethod(RpcClient.java:1597)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:21367)
at
org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:290)
... 54 more
2013-06-04 20:44:40,912 DEBUG [main]
client.HConnectionManager$HConnectionImplementation: Removed
hor8n02.gq1.ygridcore.net:60020 as a location of
usertable,user8129671889902366092,1370373946456.13168d8b86f1ace9472f60555207a707.
for tableName=usertable from cache
Reloaded hor8n09.gq1.ygridcore.net region(s)
{code}
The script tried to move region to hor8n09.
RegionMovedException involving hor8n09 is misleading because hor8n09 is the
recipient of the region movement.
Looks like such exception should be suppressed.
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
