[
https://issues.apache.org/jira/browse/IGNITE-28304?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Denis Chudov updated IGNITE-28304:
----------------------------------
Description:
Several threads may look like this
{{"%cac-dpd-cde-gg-aks-dev-4%partition-operations-2" #231 [425] prio=10
os_prio=0 cpu=179512.94ms elapsed=14666.19s tid=0x00007f835316b640 nid=425
waiting for monitor entry [0x00007f834e793000]
java.lang.Thread.State: BLOCKED (on object monitor)
at
org.apache.ignite.internal.tx.impl.HeapLockManager$LockState.tryAcquire(HeapLockManager.java:828)
- waiting to lock <0x000000062ede70b8> (a java.util.TreeMap)
at
org.apache.ignite.internal.tx.impl.HeapLockManager.acquire(HeapLockManager.java:168)
at
org.apache.ignite.internal.table.distributed.SortedIndexLocker.lambda$locksForInsert$4(SortedIndexLocker.java:179)
at
org.apache.ignite.internal.table.distributed.SortedIndexLocker$$Lambda/0x00000008011b3810.apply(Unknown
Source)
at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source)
at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source)
at
org.apache.ignite.internal.table.distributed.SortedIndexLocker.locksForInsert(SortedIndexLocker.java:176)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.takePutLockOnIndexes(PartitionReplicaListener.java:3850)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$takeLocksForInsert$190(PartitionReplicaListener.java:3835)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener$$Lambda/0x00000008011aeb58.apply(Unknown
Source)
at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source)
at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.takeLocksForInsert(PartitionReplicaListener.java:3835)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$processSingleEntryAction$150(PartitionReplicaListener.java:3497)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener$$Lambda/0x00000008011a4000.apply(Unknown
Source)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.continueResolvingByPk(PartitionReplicaListener.java:2036)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$resolveRowByPk$80(PartitionReplicaListener.java:2010)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener$$Lambda/0x000000080115b198.get(Unknown
Source)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.resolveRowByPk(PartitionReplicaListener.java:2024)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processSingleEntryAction(PartitionReplicaListener.java:3491)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$processOperationRequest$8(PartitionReplicaListener.java:649)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener$$Lambda/0x0000000801197be8.get(Unknown
Source)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.appendTxCommand(PartitionReplicaListener.java:2134)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processOperationRequest(PartitionReplicaListener.java:644)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processOperationRequestWithTxOperationManagementLogic(PartitionReplicaListener.java:4478)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processRequest(PartitionReplicaListener.java:534)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processRequestInContext(PartitionReplicaListener.java:497)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.process(PartitionReplicaListener.java:488)
at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.lambda$processTableAwareRequest$4(ZonePartitionReplicaListener.java:268)
at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener$$Lambda/0x0000000801072810.apply(Unknown
Source)
at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source)
at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source)
at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.processTableAwareRequest(ZonePartitionReplicaListener.java:252)
at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.processRequest(ZonePartitionReplicaListener.java:219)
at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.lambda$invoke$0(ZonePartitionReplicaListener.java:202)
at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener$$Lambda/0x0000000801058000.apply(Unknown
Source)
at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source)
at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source)
at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.invoke(ZonePartitionReplicaListener.java:202)
at
org.apache.ignite.internal.replicator.ZonePartitionReplicaImpl.processRequest(ZonePartitionReplicaImpl.java:67)
at
org.apache.ignite.internal.replicator.ReplicaManager.handleReplicaRequest(ReplicaManager.java:399)
at
org.apache.ignite.internal.replicator.ReplicaManager.onReplicaMessageReceived(ReplicaManager.java:332)
at
org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x0000000800486c98.onReceived(Unknown
Source)
at
org.apache.ignite.internal.network.DefaultMessagingService.sendToSelf(DefaultMessagingService.java:471)
at
org.apache.ignite.internal.network.DefaultMessagingService.invoke0(DefaultMessagingService.java:365)
at
org.apache.ignite.internal.network.DefaultMessagingService.invoke(DefaultMessagingService.java:278)
at
org.apache.ignite.internal.network.wrapper.JumpToExecutorByConsistentIdAfterSend.invoke(JumpToExecutorByConsistentIdAfterSend.java:100)
at
org.apache.ignite.internal.network.MessagingService.invoke(MessagingService.java:233)
at
org.apache.ignite.internal.replicator.ReplicaService.sendToReplicaRaw(ReplicaService.java:146)
at
org.apache.ignite.internal.replicator.ReplicaService.sendToReplica(ReplicaService.java:127)
at
org.apache.ignite.internal.replicator.ReplicaService.invoke(ReplicaService.java:310)
at
org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.trackingInvoke(InternalTableImpl.java:815)
at
org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.enlistInTx(InternalTableImpl.java:468)
at
org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.enlistInTx(InternalTableImpl.java:412)
at
org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.upsert(InternalTableImpl.java:1330)
at
org.apache.ignite.internal.table.RecordBinaryViewImpl.lambda$upsertAsync$7(RecordBinaryViewImpl.java:200)
at
org.apache.ignite.internal.table.RecordBinaryViewImpl$$Lambda/0x0000000801192da8.act(Unknown
Source)
at
org.apache.ignite.internal.table.AbstractTableView.lambda$withSchemaSync$1(AbstractTableView.java:139)
at
org.apache.ignite.internal.table.AbstractTableView$$Lambda/0x0000000801130678.apply(Unknown
Source)
at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source)
at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source)
at
org.apache.ignite.internal.table.AbstractTableView.withSchemaSync(AbstractTableView.java:139)
at
org.apache.ignite.internal.table.AbstractTableView.withSchemaSync(AbstractTableView.java:126)
at
org.apache.ignite.internal.table.AbstractTableView.doOperation(AbstractTableView.java:104)
at
org.apache.ignite.internal.table.RecordBinaryViewImpl.upsertAsync(RecordBinaryViewImpl.java:197)
at
org.apache.ignite.internal.table.RecordBinaryViewImpl.upsertAsync(RecordBinaryViewImpl.java:71)
at
org.apache.ignite.internal.table.CloseableRecordView.upsertAsync(CloseableRecordView.java:127)
at
org.gridgain.internal.table.SecuredRecordView.lambda$upsertAsync$4(SecuredRecordView.java:145)
at
org.gridgain.internal.table.SecuredRecordView$$Lambda/0x000000080118d820.get(Unknown
Source)
at
org.gridgain.internal.rbac.authorization.Authorizer.lambda$authorizeThenCompose$4(Authorizer.java:156)
at
org.gridgain.internal.rbac.authorization.Authorizer$$Lambda/0x00000008011926d0.apply(Unknown
Source)
at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source)
at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source)
at
org.gridgain.internal.rbac.authorization.Authorizer.authorizeThenCompose(Authorizer.java:156)
at
org.gridgain.internal.table.SecuredRecordView.upsertAsync(SecuredRecordView.java:145)
at
org.apache.ignite.client.handler.requests.table.ClientTupleUpsertRequest.lambda$process$2(ClientTupleUpsertRequest.java:56)
at
org.apache.ignite.client.handler.requests.table.ClientTupleUpsertRequest$$Lambda/0x000000080118cf00.apply(Unknown
Source)
at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source)
at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source)
at
org.apache.ignite.client.handler.requests.table.ClientTupleUpsertRequest.process(ClientTupleUpsertRequest.java:56)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler.processOperation(ClientInboundMessageHandler.java:928)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler.processOperationInternal(ClientInboundMessageHandler.java:1236)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler.lambda$processOperationInSecurityContext$11(ClientInboundMessageHandler.java:1221)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler$$Lambda/0x00000008010865c0.run(Unknown
Source)
at
org.gridgain.internal.security.context.SecuredRunnable.run(SecuredRunnable.java:34)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler.processOperationInSecurityContext(ClientInboundMessageHandler.java:1221)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler.lambda$processOperation$6(ClientInboundMessageHandler.java:884)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler$$Lambda/0x000000080110eda8.run(Unknown
Source)
at
java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/Unknown
Source)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/Unknown
Source)
at java.lang.Thread.runWith([email protected]/Unknown Source)
at java.lang.Thread.run([email protected]/Unknown Source)}}
The reason is waiting on synchronized block to access tree map which stores
waiters.
Approach with the tree map may be more effective by memory but probably, tree
map should be replaced with concurrent structure when its size grows.
was:
Several threads may look like this
{code:java}
"%cac-dpd-cde-gg-aks-dev-4%partition-operations-2" #231 [425] prio=10 os_prio=0
cpu=179512.94ms elapsed=14666.19s tid=0x00007f835316b640 nid=425 waiting for
monitor entry [0x00007f834e793000] java.lang.Thread.State: BLOCKED (on
object monitor) at
org.apache.ignite.internal.tx.impl.HeapLockManager$LockState.tryAcquire(HeapLockManager.java:828)
- waiting to lock <0x000000062ede70b8> (a java.util.TreeMap) at
org.apache.ignite.internal.tx.impl.HeapLockManager.acquire(HeapLockManager.java:168)
at
org.apache.ignite.internal.table.distributed.SortedIndexLocker.lambda$locksForInsert$4(SortedIndexLocker.java:179)
at
org.apache.ignite.internal.table.distributed.SortedIndexLocker$$Lambda/0x00000008011b3810.apply(Unknown
Source) at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source) at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source) at
org.apache.ignite.internal.table.distributed.SortedIndexLocker.locksForInsert(SortedIndexLocker.java:176)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.takePutLockOnIndexes(PartitionReplicaListener.java:3850)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$takeLocksForInsert$190(PartitionReplicaListener.java:3835)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener$$Lambda/0x00000008011aeb58.apply(Unknown
Source) at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source) at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source) at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.takeLocksForInsert(PartitionReplicaListener.java:3835)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$processSingleEntryAction$150(PartitionReplicaListener.java:3497)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener$$Lambda/0x00000008011a4000.apply(Unknown
Source) at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.continueResolvingByPk(PartitionReplicaListener.java:2036)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$resolveRowByPk$80(PartitionReplicaListener.java:2010)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener$$Lambda/0x000000080115b198.get(Unknown
Source) at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.resolveRowByPk(PartitionReplicaListener.java:2024)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processSingleEntryAction(PartitionReplicaListener.java:3491)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$processOperationRequest$8(PartitionReplicaListener.java:649)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener$$Lambda/0x0000000801197be8.get(Unknown
Source) at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.appendTxCommand(PartitionReplicaListener.java:2134)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processOperationRequest(PartitionReplicaListener.java:644)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processOperationRequestWithTxOperationManagementLogic(PartitionReplicaListener.java:4478)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processRequest(PartitionReplicaListener.java:534)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processRequestInContext(PartitionReplicaListener.java:497)
at
org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.process(PartitionReplicaListener.java:488)
at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.lambda$processTableAwareRequest$4(ZonePartitionReplicaListener.java:268)
at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener$$Lambda/0x0000000801072810.apply(Unknown
Source) at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source) at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source) at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.processTableAwareRequest(ZonePartitionReplicaListener.java:252)
at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.processRequest(ZonePartitionReplicaListener.java:219)
at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.lambda$invoke$0(ZonePartitionReplicaListener.java:202)
at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener$$Lambda/0x0000000801058000.apply(Unknown
Source) at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source) at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source) at
org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.invoke(ZonePartitionReplicaListener.java:202)
at
org.apache.ignite.internal.replicator.ZonePartitionReplicaImpl.processRequest(ZonePartitionReplicaImpl.java:67)
at
org.apache.ignite.internal.replicator.ReplicaManager.handleReplicaRequest(ReplicaManager.java:399)
at
org.apache.ignite.internal.replicator.ReplicaManager.onReplicaMessageReceived(ReplicaManager.java:332)
at
org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x0000000800486c98.onReceived(Unknown
Source) at
org.apache.ignite.internal.network.DefaultMessagingService.sendToSelf(DefaultMessagingService.java:471)
at
org.apache.ignite.internal.network.DefaultMessagingService.invoke0(DefaultMessagingService.java:365)
at
org.apache.ignite.internal.network.DefaultMessagingService.invoke(DefaultMessagingService.java:278)
at
org.apache.ignite.internal.network.wrapper.JumpToExecutorByConsistentIdAfterSend.invoke(JumpToExecutorByConsistentIdAfterSend.java:100)
at
org.apache.ignite.internal.network.MessagingService.invoke(MessagingService.java:233)
at
org.apache.ignite.internal.replicator.ReplicaService.sendToReplicaRaw(ReplicaService.java:146)
at
org.apache.ignite.internal.replicator.ReplicaService.sendToReplica(ReplicaService.java:127)
at
org.apache.ignite.internal.replicator.ReplicaService.invoke(ReplicaService.java:310)
at
org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.trackingInvoke(InternalTableImpl.java:815)
at
org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.enlistInTx(InternalTableImpl.java:468)
at
org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.enlistInTx(InternalTableImpl.java:412)
at
org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.upsert(InternalTableImpl.java:1330)
at
org.apache.ignite.internal.table.RecordBinaryViewImpl.lambda$upsertAsync$7(RecordBinaryViewImpl.java:200)
at
org.apache.ignite.internal.table.RecordBinaryViewImpl$$Lambda/0x0000000801192da8.act(Unknown
Source) at
org.apache.ignite.internal.table.AbstractTableView.lambda$withSchemaSync$1(AbstractTableView.java:139)
at
org.apache.ignite.internal.table.AbstractTableView$$Lambda/0x0000000801130678.apply(Unknown
Source) at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source) at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source) at
org.apache.ignite.internal.table.AbstractTableView.withSchemaSync(AbstractTableView.java:139)
at
org.apache.ignite.internal.table.AbstractTableView.withSchemaSync(AbstractTableView.java:126)
at
org.apache.ignite.internal.table.AbstractTableView.doOperation(AbstractTableView.java:104)
at
org.apache.ignite.internal.table.RecordBinaryViewImpl.upsertAsync(RecordBinaryViewImpl.java:197)
at
org.apache.ignite.internal.table.RecordBinaryViewImpl.upsertAsync(RecordBinaryViewImpl.java:71)
at
org.apache.ignite.internal.table.CloseableRecordView.upsertAsync(CloseableRecordView.java:127)
at
org.gridgain.internal.table.SecuredRecordView.lambda$upsertAsync$4(SecuredRecordView.java:145)
at
org.gridgain.internal.table.SecuredRecordView$$Lambda/0x000000080118d820.get(Unknown
Source) at
org.gridgain.internal.rbac.authorization.Authorizer.lambda$authorizeThenCompose$4(Authorizer.java:156)
at
org.gridgain.internal.rbac.authorization.Authorizer$$Lambda/0x00000008011926d0.apply(Unknown
Source) at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source) at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source) at
org.gridgain.internal.rbac.authorization.Authorizer.authorizeThenCompose(Authorizer.java:156)
at
org.gridgain.internal.table.SecuredRecordView.upsertAsync(SecuredRecordView.java:145)
at
org.apache.ignite.client.handler.requests.table.ClientTupleUpsertRequest.lambda$process$2(ClientTupleUpsertRequest.java:56)
at
org.apache.ignite.client.handler.requests.table.ClientTupleUpsertRequest$$Lambda/0x000000080118cf00.apply(Unknown
Source) at
java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
Source) at
java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
Source) at
org.apache.ignite.client.handler.requests.table.ClientTupleUpsertRequest.process(ClientTupleUpsertRequest.java:56)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler.processOperation(ClientInboundMessageHandler.java:928)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler.processOperationInternal(ClientInboundMessageHandler.java:1236)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler.lambda$processOperationInSecurityContext$11(ClientInboundMessageHandler.java:1221)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler$$Lambda/0x00000008010865c0.run(Unknown
Source) at
org.gridgain.internal.security.context.SecuredRunnable.run(SecuredRunnable.java:34)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler.processOperationInSecurityContext(ClientInboundMessageHandler.java:1221)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler.lambda$processOperation$6(ClientInboundMessageHandler.java:884)
at
org.apache.ignite.client.handler.ClientInboundMessageHandler$$Lambda/0x000000080110eda8.run(Unknown
Source) at
java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/Unknown
Source) at
java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/Unknown
Source) at java.lang.Thread.runWith([email protected]/Unknown Source) at
java.lang.Thread.run([email protected]/Unknown Source){code}
The reason is waiting on synchronized block to access tree map which stores
waiters.
Approach with the tree map may be more effective by memory but probably, tree
map should be replaced with concurrent structure when its size grows.
> Contention on waiters in lock manager when inserting rows with low selective
> indexes
> ------------------------------------------------------------------------------------
>
> Key: IGNITE-28304
> URL: https://issues.apache.org/jira/browse/IGNITE-28304
> Project: Ignite
> Issue Type: Bug
> Reporter: Denis Chudov
> Priority: Major
> Labels: ignite-3
>
> Several threads may look like this
>
> {{"%cac-dpd-cde-gg-aks-dev-4%partition-operations-2" #231 [425] prio=10
> os_prio=0 cpu=179512.94ms elapsed=14666.19s tid=0x00007f835316b640 nid=425
> waiting for monitor entry [0x00007f834e793000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.ignite.internal.tx.impl.HeapLockManager$LockState.tryAcquire(HeapLockManager.java:828)
> - waiting to lock <0x000000062ede70b8> (a java.util.TreeMap)
> at
> org.apache.ignite.internal.tx.impl.HeapLockManager.acquire(HeapLockManager.java:168)
> at
> org.apache.ignite.internal.table.distributed.SortedIndexLocker.lambda$locksForInsert$4(SortedIndexLocker.java:179)
> at
> org.apache.ignite.internal.table.distributed.SortedIndexLocker$$Lambda/0x00000008011b3810.apply(Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
> Source)
> at
> org.apache.ignite.internal.table.distributed.SortedIndexLocker.locksForInsert(SortedIndexLocker.java:176)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.takePutLockOnIndexes(PartitionReplicaListener.java:3850)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$takeLocksForInsert$190(PartitionReplicaListener.java:3835)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener$$Lambda/0x00000008011aeb58.apply(Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
> Source)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.takeLocksForInsert(PartitionReplicaListener.java:3835)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$processSingleEntryAction$150(PartitionReplicaListener.java:3497)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener$$Lambda/0x00000008011a4000.apply(Unknown
> Source)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.continueResolvingByPk(PartitionReplicaListener.java:2036)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$resolveRowByPk$80(PartitionReplicaListener.java:2010)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener$$Lambda/0x000000080115b198.get(Unknown
> Source)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.resolveRowByPk(PartitionReplicaListener.java:2024)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processSingleEntryAction(PartitionReplicaListener.java:3491)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$processOperationRequest$8(PartitionReplicaListener.java:649)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener$$Lambda/0x0000000801197be8.get(Unknown
> Source)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.appendTxCommand(PartitionReplicaListener.java:2134)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processOperationRequest(PartitionReplicaListener.java:644)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processOperationRequestWithTxOperationManagementLogic(PartitionReplicaListener.java:4478)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processRequest(PartitionReplicaListener.java:534)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.processRequestInContext(PartitionReplicaListener.java:497)
> at
> org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.process(PartitionReplicaListener.java:488)
> at
> org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.lambda$processTableAwareRequest$4(ZonePartitionReplicaListener.java:268)
> at
> org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener$$Lambda/0x0000000801072810.apply(Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
> Source)
> at
> org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.processTableAwareRequest(ZonePartitionReplicaListener.java:252)
> at
> org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.processRequest(ZonePartitionReplicaListener.java:219)
> at
> org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.lambda$invoke$0(ZonePartitionReplicaListener.java:202)
> at
> org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener$$Lambda/0x0000000801058000.apply(Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
> Source)
> at
> org.apache.ignite.internal.partition.replicator.ZonePartitionReplicaListener.invoke(ZonePartitionReplicaListener.java:202)
> at
> org.apache.ignite.internal.replicator.ZonePartitionReplicaImpl.processRequest(ZonePartitionReplicaImpl.java:67)
> at
> org.apache.ignite.internal.replicator.ReplicaManager.handleReplicaRequest(ReplicaManager.java:399)
> at
> org.apache.ignite.internal.replicator.ReplicaManager.onReplicaMessageReceived(ReplicaManager.java:332)
> at
> org.apache.ignite.internal.replicator.ReplicaManager$$Lambda/0x0000000800486c98.onReceived(Unknown
> Source)
> at
> org.apache.ignite.internal.network.DefaultMessagingService.sendToSelf(DefaultMessagingService.java:471)
> at
> org.apache.ignite.internal.network.DefaultMessagingService.invoke0(DefaultMessagingService.java:365)
> at
> org.apache.ignite.internal.network.DefaultMessagingService.invoke(DefaultMessagingService.java:278)
> at
> org.apache.ignite.internal.network.wrapper.JumpToExecutorByConsistentIdAfterSend.invoke(JumpToExecutorByConsistentIdAfterSend.java:100)
> at
> org.apache.ignite.internal.network.MessagingService.invoke(MessagingService.java:233)
> at
> org.apache.ignite.internal.replicator.ReplicaService.sendToReplicaRaw(ReplicaService.java:146)
> at
> org.apache.ignite.internal.replicator.ReplicaService.sendToReplica(ReplicaService.java:127)
> at
> org.apache.ignite.internal.replicator.ReplicaService.invoke(ReplicaService.java:310)
> at
> org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.trackingInvoke(InternalTableImpl.java:815)
> at
> org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.enlistInTx(InternalTableImpl.java:468)
> at
> org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.enlistInTx(InternalTableImpl.java:412)
> at
> org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.upsert(InternalTableImpl.java:1330)
> at
> org.apache.ignite.internal.table.RecordBinaryViewImpl.lambda$upsertAsync$7(RecordBinaryViewImpl.java:200)
> at
> org.apache.ignite.internal.table.RecordBinaryViewImpl$$Lambda/0x0000000801192da8.act(Unknown
> Source)
> at
> org.apache.ignite.internal.table.AbstractTableView.lambda$withSchemaSync$1(AbstractTableView.java:139)
> at
> org.apache.ignite.internal.table.AbstractTableView$$Lambda/0x0000000801130678.apply(Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
> Source)
> at
> org.apache.ignite.internal.table.AbstractTableView.withSchemaSync(AbstractTableView.java:139)
> at
> org.apache.ignite.internal.table.AbstractTableView.withSchemaSync(AbstractTableView.java:126)
> at
> org.apache.ignite.internal.table.AbstractTableView.doOperation(AbstractTableView.java:104)
> at
> org.apache.ignite.internal.table.RecordBinaryViewImpl.upsertAsync(RecordBinaryViewImpl.java:197)
> at
> org.apache.ignite.internal.table.RecordBinaryViewImpl.upsertAsync(RecordBinaryViewImpl.java:71)
> at
> org.apache.ignite.internal.table.CloseableRecordView.upsertAsync(CloseableRecordView.java:127)
> at
> org.gridgain.internal.table.SecuredRecordView.lambda$upsertAsync$4(SecuredRecordView.java:145)
> at
> org.gridgain.internal.table.SecuredRecordView$$Lambda/0x000000080118d820.get(Unknown
> Source)
> at
> org.gridgain.internal.rbac.authorization.Authorizer.lambda$authorizeThenCompose$4(Authorizer.java:156)
> at
> org.gridgain.internal.rbac.authorization.Authorizer$$Lambda/0x00000008011926d0.apply(Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
> Source)
> at
> org.gridgain.internal.rbac.authorization.Authorizer.authorizeThenCompose(Authorizer.java:156)
> at
> org.gridgain.internal.table.SecuredRecordView.upsertAsync(SecuredRecordView.java:145)
> at
> org.apache.ignite.client.handler.requests.table.ClientTupleUpsertRequest.lambda$process$2(ClientTupleUpsertRequest.java:56)
> at
> org.apache.ignite.client.handler.requests.table.ClientTupleUpsertRequest$$Lambda/0x000000080118cf00.apply(Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.uniComposeStage([email protected]/Unknown
> Source)
> at
> java.util.concurrent.CompletableFuture.thenCompose([email protected]/Unknown
> Source)
> at
> org.apache.ignite.client.handler.requests.table.ClientTupleUpsertRequest.process(ClientTupleUpsertRequest.java:56)
> at
> org.apache.ignite.client.handler.ClientInboundMessageHandler.processOperation(ClientInboundMessageHandler.java:928)
> at
> org.apache.ignite.client.handler.ClientInboundMessageHandler.processOperationInternal(ClientInboundMessageHandler.java:1236)
> at
> org.apache.ignite.client.handler.ClientInboundMessageHandler.lambda$processOperationInSecurityContext$11(ClientInboundMessageHandler.java:1221)
> at
> org.apache.ignite.client.handler.ClientInboundMessageHandler$$Lambda/0x00000008010865c0.run(Unknown
> Source)
> at
> org.gridgain.internal.security.context.SecuredRunnable.run(SecuredRunnable.java:34)
> at
> org.apache.ignite.client.handler.ClientInboundMessageHandler.processOperationInSecurityContext(ClientInboundMessageHandler.java:1221)
> at
> org.apache.ignite.client.handler.ClientInboundMessageHandler.lambda$processOperation$6(ClientInboundMessageHandler.java:884)
> at
> org.apache.ignite.client.handler.ClientInboundMessageHandler$$Lambda/0x000000080110eda8.run(Unknown
> Source)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/Unknown
> Source)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/Unknown
> Source)
> at java.lang.Thread.runWith([email protected]/Unknown Source)
> at java.lang.Thread.run([email protected]/Unknown Source)}}
>
> The reason is waiting on synchronized block to access tree map which stores
> waiters.
> Approach with the tree map may be more effective by memory but probably, tree
> map should be replaced with concurrent structure when its size grows.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
