[
https://issues.apache.org/jira/browse/IGNITE-11683?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ivan Bessonov updated IGNITE-11683:
-----------------------------------
Description:
The problem occurs right after this line:
{code:java}
assertTrue(GridTestUtils.waitForCondition(() ->
metastorage(1).getUpdatesCount() == expUpdatesCnt, 15_000));
{code}
Client node might not be fully reconnected yet. Adding following line resolves
the problem in the particular test:
{code:java}
grid(1).cluster().clientReconnectFuture().get();
{code}
I don't consider this a proper fix. Stopping the client that hasn't finished
its reconnect shouldn't result in deadlock. Client node should be stopped
successfully.
Stack traces of hanging client node:
{code:java}
Thread
[name="test-runner-#673%metastorage.DistributedMetaStoragePersistentTest%",
id=823, state=WAITING, blockCnt=60, waitCnt=200]
Lock [object=java.lang.Object@2a2d45ba, ownerName=null, ownerId=-1]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at o.a.i.i.util.worker.GridWorker.join(GridWorker.java:243)
at o.a.i.i.util.IgniteUtils.join(IgniteUtils.java:4831)
at
o.a.i.i.processors.cache.GridCachePartitionExchangeManager.onKernalStop0(GridCachePartitionExchangeManager.java:815)
at
o.a.i.i.processors.cache.GridCacheSharedManagerAdapter.onKernalStop(GridCacheSharedManagerAdapter.java:120)
at
o.a.i.i.processors.cache.GridCacheProcessor.onKernalStop(GridCacheProcessor.java:1183)
at o.a.i.i.IgniteKernal.stop0(IgniteKernal.java:2321)
at o.a.i.i.IgniteKernal.stop(IgniteKernal.java:2269)
at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2574)
- locked o.a.i.i.IgnitionEx$IgniteNamedInstance@69d9c55
at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2537)
at o.a.i.i.IgnitionEx.stop(IgnitionEx.java:330)
at o.a.i.Ignition.stop(Ignition.java:223)
at
o.a.i.testframework.junits.GridAbstractTest.stopGrid(GridAbstractTest.java:1153)
at
o.a.i.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1193)
at
o.a.i.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1174)
at
o.a.i.i.processors.metastorage.DistributedMetaStorageTest.after(DistributedMetaStorageTest.java:85)
at
o.a.i.i.processors.metastorage.DistributedMetaStoragePersistentTest.after(DistributedMetaStoragePersistentTest.java:67)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
at o.a.i.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2044)
at java.lang.Thread.run(Thread.java:748)
{code}
{code:java}
Thread
[name="exchange-worker-#893%metastorage.DistributedMetaStoragePersistentTest1%",
id=1088, state=BLOCKED, blockCnt=2, waitCnt=2] Lock
[object=o.a.i.i.IgnitionEx$IgniteNamedInstance@69d9c55,
ownerName=test-runner-#673%metastorage.DistributedMetaStoragePersistentTest%,
ownerId=823] at
o.a.i.i.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2545) at
o.a.i.i.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2537) at
o.a.i.i.IgnitionEx.stop(IgnitionEx.java:330) at
o.a.i.Ignition.stop(Ignition.java:223) at
o.a.i.i.IgniteKernal.close(IgniteKernal.java:3626) at
o.a.i.i.IgniteKernal$5.apply(IgniteKernal.java:4086) at
o.a.i.i.IgniteKernal$5.apply(IgniteKernal.java:4066) at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at
o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:125) at
o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45) at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at
o.a.i.i.processors.cache.GridCachePartitionExchangeManager$7.apply(GridCachePartitionExchangeManager.java:723)
at
o.a.i.i.processors.cache.GridCachePartitionExchangeManager$7.apply(GridCachePartitionExchangeManager.java:709)
at
o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
at o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511) at
o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490) at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2263)
at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:145)
at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478) at
o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.init(GridDhtPartitionsExchangeFuture.java:903)
at
o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(GridCachePartitionExchangeManager.java:2958)
at
o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(GridCachePartitionExchangeManager.java:2807)
at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at
java.lang.Thread.run(Thread.java:748)
{code}
was:
The problem occurs right after this line:
{code:java}
assertTrue(GridTestUtils.waitForCondition(() ->
metastorage(1).getUpdatesCount() == expUpdatesCnt, 15_000));
{code}
Client node might not be fully reconnected yet. Adding following line resolves
the problem in the particular test:
{code:java}
grid(1).cluster().clientReconnectFuture().get();
{code}
I don't consider this a proper fix. Stopping the client that hasn't finished
its reconnect shouldn't result in infinite waiting (or deadlock). Client node
should be stopped successfully.
> DistributedMetaStoragePersistentTest#testClientReconnect hangs sometimes.
> -------------------------------------------------------------------------
>
> Key: IGNITE-11683
> URL: https://issues.apache.org/jira/browse/IGNITE-11683
> Project: Ignite
> Issue Type: Test
> Reporter: Ivan Bessonov
> Assignee: Ivan Bessonov
> Priority: Major
> Labels: MakeTeamcityGreenAgain
>
> The problem occurs right after this line:
> {code:java}
> assertTrue(GridTestUtils.waitForCondition(() ->
> metastorage(1).getUpdatesCount() == expUpdatesCnt, 15_000));
> {code}
> Client node might not be fully reconnected yet. Adding following line
> resolves the problem in the particular test:
> {code:java}
> grid(1).cluster().clientReconnectFuture().get();
> {code}
> I don't consider this a proper fix. Stopping the client that hasn't finished
> its reconnect shouldn't result in deadlock. Client node should be stopped
> successfully.
> Stack traces of hanging client node:
>
> {code:java}
> Thread
> [name="test-runner-#673%metastorage.DistributedMetaStoragePersistentTest%",
> id=823, state=WAITING, blockCnt=60, waitCnt=200]
> Lock [object=java.lang.Object@2a2d45ba, ownerName=null, ownerId=-1]
> at java.lang.Object.wait(Native Method)
> at java.lang.Object.wait(Object.java:502)
> at o.a.i.i.util.worker.GridWorker.join(GridWorker.java:243)
> at o.a.i.i.util.IgniteUtils.join(IgniteUtils.java:4831)
> at
> o.a.i.i.processors.cache.GridCachePartitionExchangeManager.onKernalStop0(GridCachePartitionExchangeManager.java:815)
> at
> o.a.i.i.processors.cache.GridCacheSharedManagerAdapter.onKernalStop(GridCacheSharedManagerAdapter.java:120)
> at
> o.a.i.i.processors.cache.GridCacheProcessor.onKernalStop(GridCacheProcessor.java:1183)
> at o.a.i.i.IgniteKernal.stop0(IgniteKernal.java:2321)
> at o.a.i.i.IgniteKernal.stop(IgniteKernal.java:2269)
> at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2574)
> - locked o.a.i.i.IgnitionEx$IgniteNamedInstance@69d9c55
> at o.a.i.i.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2537)
> at o.a.i.i.IgnitionEx.stop(IgnitionEx.java:330)
> at o.a.i.Ignition.stop(Ignition.java:223)
> at
> o.a.i.testframework.junits.GridAbstractTest.stopGrid(GridAbstractTest.java:1153)
> at
> o.a.i.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1193)
> at
> o.a.i.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1174)
> at
> o.a.i.i.processors.metastorage.DistributedMetaStorageTest.after(DistributedMetaStorageTest.java:85)
> at
> o.a.i.i.processors.metastorage.DistributedMetaStoragePersistentTest.after(DistributedMetaStoragePersistentTest.java:67)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
> at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
> at
> o.a.i.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2044)
> at java.lang.Thread.run(Thread.java:748)
> {code}
> {code:java}
>
> Thread
> [name="exchange-worker-#893%metastorage.DistributedMetaStoragePersistentTest1%",
> id=1088, state=BLOCKED, blockCnt=2, waitCnt=2] Lock
> [object=o.a.i.i.IgnitionEx$IgniteNamedInstance@69d9c55,
> ownerName=test-runner-#673%metastorage.DistributedMetaStoragePersistentTest%,
> ownerId=823] at
> o.a.i.i.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2545) at
> o.a.i.i.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2537) at
> o.a.i.i.IgnitionEx.stop(IgnitionEx.java:330) at
> o.a.i.Ignition.stop(Ignition.java:223) at
> o.a.i.i.IgniteKernal.close(IgniteKernal.java:3626) at
> o.a.i.i.IgniteKernal$5.apply(IgniteKernal.java:4086) at
> o.a.i.i.IgniteKernal$5.apply(IgniteKernal.java:4066) at
> o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
> at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
> at
> o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
> at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511)
> at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490)
> at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478)
> at o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:125)
> at o.a.i.i.util.future.GridCompoundFuture.apply(GridCompoundFuture.java:45)
> at
> o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
> at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
> at
> o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
> at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511)
> at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490)
> at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478)
> at
> o.a.i.i.processors.cache.GridCachePartitionExchangeManager$7.apply(GridCachePartitionExchangeManager.java:723)
> at
> o.a.i.i.processors.cache.GridCachePartitionExchangeManager$7.apply(GridCachePartitionExchangeManager.java:709)
> at
> o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:399)
> at o.a.i.i.util.future.GridFutureAdapter.unblock(GridFutureAdapter.java:347)
> at
> o.a.i.i.util.future.GridFutureAdapter.unblockAll(GridFutureAdapter.java:335)
> at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:511)
> at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:490)
> at
> o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2263)
> at
> o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:145)
> at o.a.i.i.util.future.GridFutureAdapter.onDone(GridFutureAdapter.java:478)
> at
> o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.init(GridDhtPartitionsExchangeFuture.java:903)
> at
> o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(GridCachePartitionExchangeManager.java:2958)
> at
> o.a.i.i.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(GridCachePartitionExchangeManager.java:2807)
> at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120) at
> java.lang.Thread.run(Thread.java:748)
> {code}
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
