[
https://issues.apache.org/jira/browse/PHOENIX-4915?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
jaanai updated PHOENIX-4915:
----------------------------
Priority: Trivial (was: Blocker)
> The client gets stuck when using same rows concurrently writing data table
> --------------------------------------------------------------------------
>
> Key: PHOENIX-4915
> URL: https://issues.apache.org/jira/browse/PHOENIX-4915
> Project: Phoenix
> Issue Type: Bug
> Affects Versions: 4.12.0, 4.14.0
> Reporter: jaanai
> Assignee: jaanai
> Priority: Trivial
> Attachments: image-2018-09-21-19-30-12-989.png, test.java, test.sql
>
>
> The client has got stuck when using the multi-thread writes the same rows
> data into a data table which has a global index.
> I find that rows lock of the data table will not be released under highly
> writing load and throwing " ERROR 2008 (INT10): ERROR 2008 (INT10): Unable to
> find cached index metadata." exception information. Most of the threads will
> be waiting for getting the row lock in Jstack information.
> The following are exceptions on the server side:
> {code:java}
> [B.defaultRpcServer.handler=37,queue=1,port=16020]
> regionserver.RSRpcServices(103): Failed doing multi operation, current call
> is : callId: 3455 service: ClientService meth
> odName: Multi size: 23.1 K connection: 192.168.199.7:52050 param:
> actionCount=44#regionCount=8#LOCK,\x02,1537434393195.ee6d441a04ee6a59b24262f22f618d88.#####
> org.apache.hadoop.hbase.DoNotRetryIOException: ERROR 2008 (INT10): ERROR 2008
> (INT10): Unable to find cached index metadata. key=-727998515684050837
> region=LOCK,\x0E,1537434393195.f4de29d4b36775589a49f
> 1c7a20c73a2.host=hb-bp1v2q830426r6763-004.hbase.rds.aliyuncs.com,16020,1537434304031
> Index update failed
> at
> org.apache.phoenix.util.ServerUtil.createIOException(ServerUtil.java:88)
> at
> org.apache.phoenix.util.ServerUtil.throwIOException(ServerUtil.java:62)
> at
> org.apache.phoenix.index.PhoenixIndexMetaData.getIndexMetaData(PhoenixIndexMetaData.java:87)
> at
> org.apache.phoenix.index.PhoenixIndexMetaData.<init>(PhoenixIndexMetaData.java:103)
> at
> org.apache.phoenix.index.PhoenixIndexBuilder.getIndexMetaData(PhoenixIndexBuilder.java:95)
> at
> org.apache.phoenix.hbase.index.builder.IndexBuildManager.getIndexUpdate(IndexBuildManager.java:80)
> at
> org.apache.phoenix.hbase.index.Indexer.preBatchMutateWithExceptions(Indexer.java:528)
> at
> org.apache.phoenix.hbase.index.Indexer.preBatchMutate(Indexer.java:374)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$37.call(RegionCoprocessorHost.java:1032)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1714)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1789)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1746)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preBatchMutate(RegionCoprocessorHost.java:1028)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.asyncBatchMutate(HRegion.java:3236)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.doAsyncBatchOp(RSRpcServices.java:2147)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchMutationCrossRegions(RSRpcServices.java:2308)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2578)
> at
> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32303)
> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2394)
> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:174)
> at
> org.apache.hadoop.hbase.ipc.RpcExecutor$CallHandler.run(RpcExecutor.java:178)
> Caused by: java.sql.SQLException: ERROR 2008 (INT10): Unable to find cached
> index metadata. key=-727998515684050837
> region=LOCK,\x0E,1537434393195.f4de29d4b36775589a49f1c7a20c73a2.host=hb-bp1v2q830426r
> 6763-004.hbase.rds.aliyuncs.com,16020,1537434304031
> at
> org.apache.phoenix.exception.SQLExceptionCode$Factory$1.newException(SQLExceptionCode.java:493)
> at
> org.apache.phoenix.exception.SQLExceptionInfo.buildException(SQLExceptionInfo.java:150)
> at
> org.apache.phoenix.index.PhoenixIndexMetaData.getIndexMetaData(PhoenixIndexMetaData.java:85)
> 2018-09-20 17:35:39,254 INFO
> [B.defaultRpcServer.handler=13,queue=1,port=16020]
> regionserver.RSRpcServices(103): Failed doing multi operation, current call
> is : callId: 3848 service: ClientService meth
> odName: Multi size: 27.2 K connection: 192.168.199.7:52042 param:
> actionCount=52#regionCount=8#LOCK,\x02,1537434393195.ee6d441a04ee6a59b24262f22f618d88.#####
> org.apache.hadoop.hbase.exceptions.TimeoutIOException: Timed out waiting for
> lock for row: 0e 30 64 32 65 34 35 63 37 2d 63 63 33 64 2d 34 36 61 35 2d 61
> 34 38 64 2d 31 38 61 62 36 31 61 31 30 63 30 39
> at
> org.apache.phoenix.hbase.index.LockManager.lockRow(LockManager.java:96)
> at
> org.apache.phoenix.hbase.index.Indexer.preBatchMutateWithExceptions(Indexer.java:425)
> at
> org.apache.phoenix.hbase.index.Indexer.preBatchMutate(Indexer.java:374)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$37.call(RegionCoprocessorHost.java:1032)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1714)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1789)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1746)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preBatchMutate(RegionCoprocessorHost.java:1028)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.asyncBatchMutate(HRegion.java:3236)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.doAsyncBatchOp(RSRpcServices.java:2147)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchMutationCrossRegions(RSRpcServices.java:2308)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2578)
> at
> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32303)
> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2394)
> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:174)
> at
> org.apache.hadoop.hbase.ipc.RpcExecutor$CallHandler.run(RpcExecutor.java:178)
> {code}
> {code:java}
> [B.defaultRpcServer.handler=37,queue=1,port=16020]
> regionserver.RSRpcServices(103): Failed doing multi operation, current call
> is : callId: 3455 service: ClientService meth
> odName: Multi size: 23.1 K connection: 192.168.199.7:52050 param:
> actionCount=44#regionCount=8#LOCK,\x02,1537434393195.ee6d441a04ee6a59b24262f22f618d88.#####
> org.apache.hadoop.hbase.DoNotRetryIOException: ERROR 2008 (INT10): ERROR 2008
> (INT10): Unable to find cached index metadata. key=-727998515684050837
> region=LOCK,\x0E,1537434393195.f4de29d4b36775589a49f
> 1c7a20c73a2.host=hb-bp1v2q830426r6763-004.hbase.rds.aliyuncs.com,16020,1537434304031
> Index update failed
> at
> org.apache.phoenix.util.ServerUtil.createIOException(ServerUtil.java:88)
> at
> org.apache.phoenix.util.ServerUtil.throwIOException(ServerUtil.java:62)
> at
> org.apache.phoenix.index.PhoenixIndexMetaData.getIndexMetaData(PhoenixIndexMetaData.java:87)
> at
> org.apache.phoenix.index.PhoenixIndexMetaData.<init>(PhoenixIndexMetaData.java:103)
> at
> org.apache.phoenix.index.PhoenixIndexBuilder.getIndexMetaData(PhoenixIndexBuilder.java:95)
> at
> org.apache.phoenix.hbase.index.builder.IndexBuildManager.getIndexUpdate(IndexBuildManager.java:80)
> at
> org.apache.phoenix.hbase.index.Indexer.preBatchMutateWithExceptions(Indexer.java:528)
> at
> org.apache.phoenix.hbase.index.Indexer.preBatchMutate(Indexer.java:374)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$37.call(RegionCoprocessorHost.java:1032)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1714)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1789)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1746)
> at
> org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preBatchMutate(RegionCoprocessorHost.java:1028)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.asyncBatchMutate(HRegion.java:3236)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.doAsyncBatchOp(RSRpcServices.java:2147)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchMutationCrossRegions(RSRpcServices.java:2308)
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2578)
> at
> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32303)
> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2394)
> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:174)
> at
> org.apache.hadoop.hbase.ipc.RpcExecutor$CallHandler.run(RpcExecutor.java:178)
> Caused by: java.sql.SQLException: ERROR 2008 (INT10): Unable to find cached
> index metadata. key=-727998515684050837
> region=LOCK,\x0E,1537434393195.f4de29d4b36775589a49f1c7a20c73a2.host=hb-bp1v2q830426r
> 6763-004.hbase.rds.aliyuncs.com,16020,1537434304031
> at
> org.apache.phoenix.exception.SQLExceptionCode$Factory$1.newException(SQLExceptionCode.java:493)
> at
> org.apache.phoenix.exception.SQLExceptionInfo.buildException(SQLExceptionInfo.java:150)
> at
> org.apache.phoenix.index.PhoenixIndexMetaData.getIndexMetaData(PhoenixIndexMetaData.java:85)
> {code}
>
>
> The left shows the RPC handler queue was filled with requests which have
> total 120 handler size on all RS processes. The middle and right are normal
> after fixed this bug.
> !image-2018-09-21-19-30-12-989.png|width=434,height=225!
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
