Pratyush Bhatt created HDDS-8200:
------------------------------------
Summary: [Hbase-Ozone] "RegionTooBusyException: Failed to obtain
lock" while running IntegrationTestIngestWithTags
Key: HDDS-8200
URL: https://issues.apache.org/jira/browse/HDDS-8200
Project: Apache Ozone
Issue Type: Bug
Components: Ozone Filesystem
Affects Versions: 1.3.0
Reporter: Pratyush Bhatt
After running:
{noformat}
hbase org.apache.hadoop.hbase.IntegrationTestsDriver -regex
IntegrationTestIngestWithTags{noformat}
Hregion lock exeptions/RegionTooBusyException were continously observed:
{noformat}
org.apache.hadoop.hbase.RegionTooBusyException:
org.apache.hadoop.hbase.RegionTooBusyException: Failed to obtain lock;
regionName=IntegrationTestIngestWithTags,3a83a838,1678808439339.7b62f84a7a9e158b17ba15d2140d738a.,
server=hb2-cf4-5.hb2-cf4.root.hwx.site,22101,1678808050486
2023-03-14 16:45:49,901|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:8876)
2023-03-14 16:45:49,901|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4518)
2023-03-14 16:45:49,902|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4468)
2023-03-14 16:45:49,902|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4398)
2023-03-14 16:45:49,903|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:4874)
2023-03-14 16:45:49,903|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:4868)
2023-03-14 16:45:49,903|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:4864)
2023-03-14 16:45:49,904|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3174)
2023-03-14 16:45:49,904|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3014)
2023-03-14 16:45:49,905|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2977)
2023-03-14 16:45:49,905|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45817)
2023-03-14 16:45:49,905|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:384)
2023-03-14 16:45:49,906|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:138)
2023-03-14 16:45:49,906|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:371)
2023-03-14 16:45:49,906|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:351)
2023-03-14 16:45:49,907|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|
2023-03-14 16:45:49,907|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|
2023-03-14 16:45:49,907|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142)
2023-03-14 16:45:49,908|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.client.HTable.put(HTable.java:514)
2023-03-14 16:45:49,908|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.util.MultiThreadedWriter$HBaseWriterThread.insert(MultiThreadedWriter.java:144)
2023-03-14 16:45:49,909|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.util.MultiThreadedWriter$HBaseWriterThread.run(MultiThreadedWriter.java:108)
2023-03-14 16:45:49,909|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|Caused by:
org.apache.hadoop.hbase.RegionTooBusyException:
org.apache.hadoop.hbase.RegionTooBusyException: Failed to obtain lock;
regionName=IntegrationTestIngestWithTags,3a83a838,1678808439339.7b62f84a7a9e158b17ba15d2140d738a.,
server=hb2-cf4-5.hb2-cf4.root.hwx.site,22101,1678808050486
2023-03-14 16:45:49,910|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:8876)
2023-03-14 16:45:49,910|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4518)
2023-03-14 16:45:49,910|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4468)
2023-03-14 16:45:49,911|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4398)
2023-03-14 16:45:49,911|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:4874)
2023-03-14 16:45:49,911|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:4868)
2023-03-14 16:45:49,912|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:4864)
2023-03-14 16:45:49,912|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3174)
2023-03-14 16:45:49,913|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3014)
2023-03-14 16:45:49,913|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2977)
2023-03-14 16:45:49,913|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45817)
2023-03-14 16:45:49,914|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:384)
2023-03-14 16:45:49,914|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:138)
2023-03-14 16:45:49,915|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:371)
2023-03-14 16:45:49,915|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:351)
2023-03-14 16:45:49,915|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|
2023-03-14 16:45:49,916|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
2023-03-14 16:45:49,916|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
2023-03-14 16:45:49,916|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
2023-03-14 16:45:49,917|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
java.lang.reflect.Constructor.newInstance(Constructor.java:423)
2023-03-14 16:45:49,917|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:97)
2023-03-14 16:45:49,918|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:87)
2023-03-14 16:45:49,918|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:278)
2023-03-14 16:45:49,918|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:263)
2023-03-14 16:45:49,919|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:126)
2023-03-14 16:45:49,919|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104)
2023-03-14 16:45:49,919|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|... 3 more
2023-03-14 16:45:49,920|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|Caused by:
org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException):
org.apache.hadoop.hbase.RegionTooBusyException: Failed to obtain lock;
regionName=IntegrationTestIngestWithTags,3a83a838,1678808439339.7b62f84a7a9e158b17ba15d2140d738a.,
server=hb2-cf4-5.hb2-cf4.root.hwx.site,22101,1678808050486
2023-03-14 16:45:49,920|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:8876)
2023-03-14 16:45:49,921|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutate(HRegion.java:4518)
2023-03-14 16:45:49,921|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4468)
2023-03-14 16:45:49,922|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4398)
2023-03-14 16:45:49,922|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:4874)
2023-03-14 16:45:49,922|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:4868)
2023-03-14 16:45:49,923|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:4864)
2023-03-14 16:45:49,923|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3174)
2023-03-14 16:45:49,923|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3014)
2023-03-14 16:45:49,924|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2977)
2023-03-14 16:45:49,924|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45817)
2023-03-14 16:45:49,924|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:384)
2023-03-14 16:45:49,925|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:138)
2023-03-14 16:45:49,925|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:371)
2023-03-14 16:45:49,926|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:351)
2023-03-14 16:45:49,926|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|
2023-03-14 16:45:49,926|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:381)
2023-03-14 16:45:49,927|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:88)
2023-03-14 16:45:49,927|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:415)
2023-03-14 16:45:49,927|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:411)
2023-03-14 16:45:49,928|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:114)
2023-03-14 16:45:49,928|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.Call.setException(Call.java:129)
2023-03-14 16:45:49,928|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:162)
2023-03-14 16:45:49,929|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:192)
2023-03-14 16:45:49,929|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
2023-03-14 16:45:49,930|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
2023-03-14 16:45:49,930|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
2023-03-14 16:45:49,930|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:327)
2023-03-14 16:45:49,931|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:299)
2023-03-14 16:45:49,931|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
2023-03-14 16:45:49,932|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
2023-03-14 16:45:49,932|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
2023-03-14 16:45:49,932|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
2023-03-14 16:45:49,933|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
2023-03-14 16:45:49,933|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
2023-03-14 16:45:49,934|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
2023-03-14 16:45:49,934|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
2023-03-14 16:45:49,934|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
2023-03-14 16:45:49,935|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
2023-03-14 16:45:49,935|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
2023-03-14 16:45:49,936|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
2023-03-14 16:45:49,936|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722)
2023-03-14 16:45:49,936|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:658)
2023-03-14 16:45:49,937|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:584)
2023-03-14 16:45:49,937|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)
2023-03-14 16:45:49,937|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:995)
2023-03-14 16:45:49,938|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
2023-03-14 16:45:49,938|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
2023-03-14 16:45:49,938|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|at
java.lang.Thread.run(Thread.java:750)
2023-03-14 16:45:49,939|INFO|MainThread|machine.py:203 -
run()||GUID=4ab065b8-f676-4cbc-b2b6-0814342c920f|{noformat}
And after sometime all the Ozone and Hbase role went down.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
