zhengsicheng created HBASE-27664:
------------------------------------
Summary: RS crash on ipc request to big
Key: HBASE-27664
URL: https://issues.apache.org/jira/browse/HBASE-27664
Project: HBase
Issue Type: Bug
Reporter: zhengsicheng
Assignee: zhengsicheng
2023-02-17 16:44:09,601 WARN [RS-EventLoopGroup-1-46] ipc.NettyRpcServer: RPC
data length of 825701220 received from client_ip1 is greater than max allowed
268435456. Set "hba se.ipc.max.request.size" on server to override this
limit (not recommended)
9423 2023-02-17 16:44:12,668 ERROR
[RpcServer.default.RWQ.Fifo.write.handler=62,queue=62,port=16020]
ipc.RpcServer: Unexpected throwable object
9424 java.lang.RuntimeException: Unknown code 98
9425 at org.apache.hadoop.hbase.KeyValue$Type.codeToType(KeyValue.java:276)
9426 at
org.apache.hadoop.hbase.CellUtil.getCellKeyAsString(CellUtil.java:1340)
9427 at
org.apache.hadoop.hbase.CellUtil.getCellKeyAsString(CellUtil.java:1318)
9428 at org.apache.hadoop.hbase.CellUtil.toString(CellUtil.java:1512)
9429 at
org.apache.hadoop.hbase.ByteBufferKeyValue.toString(ByteBufferKeyValue.java:301)
9430 at org.apache.hadoop.hbase.client.Mutation.add(Mutation.java:786)
9431 at org.apache.hadoop.hbase.client.Put.add(Put.java:282)
9432 at
org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.toPut(ProtobufUtil.java:656)
9433 at
org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1033)
9434 at
org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:987)
9435 at
org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:950)
9436 at
org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2949)
9437 at
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265)
9438 at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:394)
9439 at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133)
9440 at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338)
9441 at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318)
9442 2023-02-17 16:44:19,430 WARN [RS-EventLoopGroup-1-60] ipc.RpcServer:
Invalid request header: , should have param set in it
9443 2023-02-17 16:44:19,431 WARN [RS-EventLoopGroup-1-60] ipc.RpcServer:
/hostname_rs1:16020 is unable to read call parameter from client client_ip1
9444 org.apache.hadoop.hbase.DoNotRetryIOException: Invalid request header: ,
should have param set in it
9445 at
org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:654)
9446 at
org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:448)
9447 at
org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89)
9448 at
org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:63)
9449 at
org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead(NettyRpcServerRequestDecoder.java:62)
9450 at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
9451 at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
9452 at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
9453 at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:327)
9454 at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:314)
9455 at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:435)
9456 at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279)
9457 at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
9458 at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
9459 at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
9460 at
org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
9461 at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
9462 at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
9463 at
org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
9464 at
org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:795)
9465 at
org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:480)
9466 at
org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:378)
9467 at
org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
9468 at
org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
9469 at
org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
9470 at java.base/java.lang.Thread.run(Thread.java:832)
9471 2023-02-17 16:44:19,431 WARN [RS-EventLoopGroup-1-60]
ipc.NettyRpcServer: RPC data length of 4294967044 received from client_ip1 is
greater than max allowed 268435456. Set "hb ase.ipc.max.request.size" on
server to override this limit (not recommended)
9472 2023-02-17 16:44:24,215 INFO [qtp1820051628-4977731]
requests.regionserver: 127.0.0.1 - - [17/2月/2023:08:44:24 +0000] "GET
//127.0.0.1:16030/jmx HTTP/1.1" 200 1397322
9473 2023-02-17 16:44:24,703 ERROR
[RpcServer.default.RWQ.Fifo.write.handler=532,queue=132,port=16020]
ipc.RpcServer: Unexpected throwable object
9474 java.lang.NegativeArraySizeException: -620684796
9475 at org.apache.hadoop.hbase.CellUtil.cloneValue(CellUtil.java:126)
9476 at
org.apache.hadoop.hbase.ByteBufferKeyValue.getValueArray(ByteBufferKeyValue.java:168)
9477 at org.apache.hadoop.hbase.CellUtil.toString(CellUtil.java:1522)
9478 at
org.apache.hadoop.hbase.ByteBufferKeyValue.toString(ByteBufferKeyValue.java:301)
9479 at org.apache.hadoop.hbase.client.Mutation.add(Mutation.java:786)
9480 at org.apache.hadoop.hbase.client.Put.add(Put.java:282)
9481 at
org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.toPut(ProtobufUtil.java:656)
9482 at
org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1033)
9483 at
org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:987)
9484 at
org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:950)
9485 at
org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2949)
9486 at
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265)
9487 at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:394)
9488 at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133)
9489 at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338)
9490 at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318)
9491 2023-02-17 16:44:25,356 WARN [RS-EventLoopGroup-1-13]
ipc.NettyRpcServer: RPC data length of 929117235 received from client_ip1 is
greater than max allowed 268435456. Set "hbase.ipc.max.request.size" on server
to override this limit (not recommended)
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
