[
https://issues.apache.org/jira/browse/HBASE-27097?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17555240#comment-17555240
]
Viraj Jasani edited comment on HBASE-27097 at 6/16/22 6:33 PM:
---------------------------------------------------------------
When we use BlockingRpcClient or NettyRpcClient against SimpleRpcServer in
hbase2 (with SaslAuth, Negotiated QoP: auth-conf), hbase shell commands throw
checksum failed errors:
{code:java}
hbase:010:0> rit
ERROR: Checksum failed
{code}
Similarly, scanning of SYSTEM.CATALOG in Phoenix also fails:
{code:java}
Caused by: java.net.SocketTimeoutException: callTimeout=60000,
callDuration=68788: Call to address=regionserver-1:60020 failed on local
exception: javax.security.sasl.SaslException: Problems unwrapping SASL buffer
[Caused by GSSException: Failure unspecified at GSS-API level (Mechanism level:
Could not use AES128 Cipher - Checksum failed)] row '' on table
'SYSTEM.CATALOG' at
region=SYSTEM.CATALOG,,1651182322114.9706a466ac24135ce93769671b601652.,
hostname=regionserver-1,60020,1655397068266, seqNum=256
at
org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:156)
at
org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74)
... 3 more
Caused by: javax.security.sasl.SaslException: Call to
address=regionserver-1:60020 failed on local exception:
javax.security.sasl.SaslException: Problems unwrapping SASL buffer [Caused by
GSSException: Failure unspecified at GSS-API level (Mechanism level: Could not
use AES128 Cipher - Checksum failed)] [Caused by
javax.security.sasl.SaslException: Problems unwrapping SASL buffer [Caused by
GSSException: Failure unspecified at GSS-API level (Mechanism level: Could not
use AES128 Cipher - Checksum failed)]]
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:240)
at
org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:385)
at
org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:89)
at
org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:417)
at
org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:413)
at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:115)
at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:130)
at
org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:203)
at
org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.exceptionCaught(NettyRpcDuplexHandler.java:220)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:302)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:281)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:273)
at
org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:302)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:381)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:327)
at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:314)
at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:435)
at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at
org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at
org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at
org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:658)
at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:584)
at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
at
org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
at
org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at
org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
... 1 more
Caused by: javax.security.sasl.SaslException: Problems unwrapping SASL buffer
[Caused by GSSException: Failure unspecified at GSS-API level (Mechanism level:
Could not use AES128 Cipher - Checksum failed)]
at com.sun.security.sasl.gsskerb.GssKrb5Base.unwrap(GssKrb5Base.java:90)
at
org.apache.hadoop.hbase.security.SaslUnwrapHandler.channelRead0(SaslUnwrapHandler.java:51)
at
org.apache.hadoop.hbase.security.SaslUnwrapHandler.channelRead0(SaslUnwrapHandler.java:32)
at
org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
... 22 more
Caused by: GSSException: Failure unspecified at GSS-API level (Mechanism level:
Could not use AES128 Cipher - Checksum failed)
at sun.security.jgss.krb5.CipherHelper.aes256Decrypt(CipherHelper.java:1399)
at sun.security.jgss.krb5.CipherHelper.decryptData(CipherHelper.java:574)
at sun.security.jgss.krb5.WrapToken_v2.getData(WrapToken_v2.java:130)
at sun.security.jgss.krb5.WrapToken_v2.getData(WrapToken_v2.java:105)
at sun.security.jgss.krb5.Krb5Context.unwrap(Krb5Context.java:1062)
at sun.security.jgss.GSSContextImpl.unwrap(GSSContextImpl.java:403)
at com.sun.security.sasl.gsskerb.GssKrb5Base.unwrap(GssKrb5Base.java:77)
... 26 more
Caused by: java.security.GeneralSecurityException: Checksum failed
at
sun.security.krb5.internal.crypto.dk.AesDkCrypto.decryptCTS(AesDkCrypto.java:451)
at
sun.security.krb5.internal.crypto.dk.AesDkCrypto.decryptRaw(AesDkCrypto.java:291)
at sun.security.krb5.internal.crypto.Aes256.decryptRaw(Aes256.java:86)
at sun.security.jgss.krb5.CipherHelper.aes256Decrypt(CipherHelper.java:1395)
... 32 more
{code}
was (Author: vjasani):
When we use BlockingRpcClient or NettyRpcClient against SimpleRpcServer in
hbase2 (with SaslAuth, Negotiated QoP: auth-conf), hbase shell commands throw
checksum failed errors:
{code:java}
hbase:010:0> rit
ERROR: Checksum failed
{code}
> SimpleRpcServer is broken
> -------------------------
>
> Key: HBASE-27097
> URL: https://issues.apache.org/jira/browse/HBASE-27097
> Project: HBase
> Issue Type: Bug
> Components: rpc
> Affects Versions: 2.5.0
> Reporter: Andrew Kyle Purtell
> Assignee: Andrew Kyle Purtell
> Priority: Blocker
> Fix For: 2.5.0, 3.0.0-alpha-3, 2.4.13
>
> Attachments: MultiByteBuff.patch
>
>
> Concerns about SimpleRpcServer are not new, and not new to 2.5. @chenxu
> noticed a problem on HBASE-23917 back in 2020. After some simple evaluations
> it seems quite broken.
> When I run an async version of ITLCC against a 2.5.0 cluster configured with
> hbase.rpc.server.impl=SimpleRpcServer, the client almost immediately stalls
> because there are too many in flight requests. The logic to pause with too
> many in flight requests is my own. That's not important. Looking at the
> server logs it is apparent that SimpleRpcServer is quite broken. Handlers
> suffer frequent protobuf parse errors and do not properly return responses to
> the client. This is what stalls my test client. Rather quickly all available
> request slots are full of requests that will have to time out on the client
> side.
> Exceptions have three patterns but they all have in common
> SimpleServerRpcConnection#process. It seems likely the root cause is
> mismatched expectations or bugs in connection buffer handling in
> SimpleRpcServer/SimpleServerRpcConnection versus downstream classes that
> process and parse the buffers. It also seems likely that changes were made to
> downstream classes like ServerRpcConnection expecting NettyRpcServer's
> particulars without updating SimpleServerRpcConnection and/or
> SimpleRpcServer. That said, this is just a superficial analysis.
> 1) "Protocol message end-group tag did not match expected tag"
> {noformat}
> 2022-06-07T16:44:04,625 WARN
> [Reader=5,bindAddress=buildbox.localdomain,port=8120] ipc.RpcServer:
> /127.0.1.1:8120 is unable to read call parameter from client 127.0.0.1
> org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException:
> Protocol message end-group tag did not match expected tag.
> at
> org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException.invalidEndTag(InvalidProtocolBufferException.java:129)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream$ByteInputDecoder.checkLastTagWas(CodedInputStream.java:4034)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream$ByteInputDecoder.readMessage(CodedInputStream.java:4275)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue.<init>(ClientProtos.java:10520)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue.<init>(ClientProtos.java:10464)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$1.parsePartialFrom(ClientProtos.java:12251)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$1.parsePartialFrom(ClientProtos.java:12245)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream$ByteInputDecoder.readMessage(CodedInputStream.java:4274)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto.<init>(ClientProtos.java:9981)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto.<init>(ClientProtos.java:9910)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$1.parsePartialFrom(ClientProtos.java:14097)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$1.parsePartialFrom(ClientProtos.java:14091)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream$ByteInputDecoder.readMessage(CodedInputStream.java:4274)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest.<init>(ClientProtos.java:14251)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest.<init>(ClientProtos.java:14190)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest$1.parsePartialFrom(ClientProtos.java:15304)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest$1.parsePartialFrom(ClientProtos.java:15298)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest$Builder.mergeFrom(ClientProtos.java:14860)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest$Builder.mergeFrom(ClientProtos.java:14651)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage$Builder.mergeFrom(AbstractMessage.java:420)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage$Builder.mergeFrom(AbstractMessage.java:317)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.mergeFrom(ProtobufUtil.java:2638)
> ~[hbase-client-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:644)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:444)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleServerRpcConnection.process(SimpleServerRpcConnection.java:285)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleServerRpcConnection.readAndProcess(SimpleServerRpcConnection.java:251)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.doRead(SimpleRpcServer.java:318)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:180)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:153)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> ~[?:?]
> {noformat}
> 2) "Protocol message tag had invalid wire type."
> {noformat}
> 2022-06-07T16:44:04,705 WARN
> [Reader=6,bindAddress=buildbox.localdomain,port=8120] ipc.RpcServer:
> /127.0.1.1:8120 is unable to read call parameter from client 127.0.0.1
> org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException$InvalidWireTypeException:
> Protocol message tag had invalid wire type.
> at
> org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException.invalidWireType(InvalidProtocolBufferException.java:134)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.UnknownFieldSet$Builder.mergeFieldFrom(UnknownFieldSet.java:527)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessageV3.parseUnknownField(GeneratedMessageV3.java:320)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$QualifierValue.<init>(ClientProtos.java:10700)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$QualifierValue.<init>(ClientProtos.java:10620)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$QualifierValue$1.parsePartialFrom(ClientProtos.java:11481)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$QualifierValue$1.parsePartialFrom(ClientProtos.java:11475)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream$ByteInputDecoder.readMessage(CodedInputStream.java:4274)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue.<init>(ClientProtos.java:10520)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue.<init>(ClientProtos.java:10464)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$1.parsePartialFrom(ClientProtos.java:12251)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$1.parsePartialFrom(ClientProtos.java:12245)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream$ByteInputDecoder.readMessage(CodedInputStream.java:4274)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto.<init>(ClientProtos.java:9981)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto.<init>(ClientProtos.java:9910)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$1.parsePartialFrom(ClientProtos.java:14097)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$1.parsePartialFrom(ClientProtos.java:14091)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream$ByteInputDecoder.readMessage(CodedInputStream.java:4274)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest.<init>(ClientProtos.java:14251)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest.<init>(ClientProtos.java:14190)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest$1.parsePartialFrom(ClientProtos.java:15304)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest$1.parsePartialFrom(ClientProtos.java:15298)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest$Builder.mergeFrom(ClientProtos.java:14860)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest$Builder.mergeFrom(ClientProtos.java:14651)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage$Builder.mergeFrom(AbstractMessage.java:420)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage$Builder.mergeFrom(AbstractMessage.java:317)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.mergeFrom(ProtobufUtil.java:2638)
> ~[hbase-client-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:644)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:444)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleServerRpcConnection.process(SimpleServerRpcConnection.java:285)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleServerRpcConnection.readAndProcess(SimpleServerRpcConnection.java:251)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.doRead(SimpleRpcServer.java:318)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:180)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:153)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> ~[?:?]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> ~[?:?]
> at java.lang.Thread.run(Thread.java:829) ~[?:?]
> {noformat}
> 3) "While parsing a protocol message, the input ended unexpectedly in the
> middle of a field."
> {noformat}
> 2022-06-07T16:44:04,885 WARN
> [Reader=9,bindAddress=buildbox.localdomain,port=8120] ipc.RpcServer:
> /127.0.1.1:8120 is unable to read call parameter from client 127.0.0.1
> org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException:
> While parsing a protocol message, the input ended unexpectedly in the middle
> of a field. This could mean either that the input has been truncated or that
> an embedded message misreported its own length.
> at
> org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException.truncatedMessage(InvalidProtocolBufferException.java:107)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream$ByteInputDecoder.readRawLittleEndian64(CodedInputStream.java:4478)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream$ByteInputDecoder.readFixed64(CodedInputStream.java:4167)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.UnknownFieldSet$Builder.mergeFieldFrom(UnknownFieldSet.java:511)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.GeneratedMessageV3.parseUnknownField(GeneratedMessageV3.java:320)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$QualifierValue.<init>(ClientProtos.java:10700)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$QualifierValue.<init>(ClientProtos.java:10620)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$QualifierValue$1.parsePartialFrom(ClientProtos.java:11481)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$QualifierValue$1.parsePartialFrom(ClientProtos.java:11475)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream$ByteInputDecoder.readMessage(CodedInputStream.java:4274)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue.<init>(ClientProtos.java:10520)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue.<init>(ClientProtos.java:10464)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$1.parsePartialFrom(ClientProtos.java:12251)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$ColumnValue$1.parsePartialFrom(ClientProtos.java:12245)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream$ByteInputDecoder.readMessage(CodedInputStream.java:4274)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto.<init>(ClientProtos.java:9981)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto.<init>(ClientProtos.java:9910)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$1.parsePartialFrom(ClientProtos.java:14097)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutationProto$1.parsePartialFrom(ClientProtos.java:14091)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream$ByteInputDecoder.readMessage(CodedInputStream.java:4274)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest.<init>(ClientProtos.java:14251)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest.<init>(ClientProtos.java:14190)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest$1.parsePartialFrom(ClientProtos.java:15304)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest$1.parsePartialFrom(ClientProtos.java:15298)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest$Builder.mergeFrom(ClientProtos.java:14860)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest$Builder.mergeFrom(ClientProtos.java:14651)
> ~[hbase-protocol-shaded-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage$Builder.mergeFrom(AbstractMessage.java:420)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hbase.thirdparty.com.google.protobuf.AbstractMessage$Builder.mergeFrom(AbstractMessage.java:317)
> ~[hbase-shaded-protobuf-4.1.0.jar:4.1.0]
> at
> org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.mergeFrom(ProtobufUtil.java:2638)
> ~[hbase-client-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:644)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:444)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleServerRpcConnection.process(SimpleServerRpcConnection.java:285)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleServerRpcConnection.readAndProcess(SimpleServerRpcConnection.java:251)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.doRead(SimpleRpcServer.java:318)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:180)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:153)
> ~[hbase-server-2.5.1-SNAPSHOT.jar:2.5.1-SNAPSHOT]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> ~[?:?]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> ~[?:?]
> at java.lang.Thread.run(Thread.java:829) ~[?:?]
> {noformat}
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
