[
https://issues.apache.org/jira/browse/IGNITE-29163?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Vladimir Steshin updated IGNITE-29163:
--------------------------------------
Attachment: _Apache_Ignite_2.x_Tests_Snapshots_1_19207.log.zip
> Flaky IgniteClusterSnapshotSelfTest.testClientHandlesSnapshotFailOnStartStage
> -----------------------------------------------------------------------------
>
> Key: IGNITE-29163
> URL: https://issues.apache.org/jira/browse/IGNITE-29163
> Project: Ignite
> Issue Type: Improvement
> Reporter: Vladimir Steshin
> Priority: Major
> Attachments: _Apache_Ignite_2.x_Tests_Snapshots_1_19207.log.zip
>
>
> Test `Flaky
> IgniteClusterSnapshotSelfTest.testClientHandlesSnapshotFailOnStartStage()`
> looks flaky. Fails on master sometimes. The visible failure is:
> {code:java}
> [10:50:42]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:42,945][ERROR][main][] Test failed
> [test=IgniteClusterSnapshotSelfTest#testClientHandlesSnapshotFailOnStartStage[encryption=false,
> onlyPrimay=true], duration=4216]
> [10:50:42]W: [org.apache.ignite:ignite-core]
> javax.cache.CacheException: class
> org.apache.ignite.cluster.ClusterTopologyException: Remote node has left
> topology: ee282fb0-8ec1-422f-b9d0-00f6f4f00001
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1164)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.query.GridCacheQueryFutureAdapter.next(GridCacheQueryFutureAdapter.java:169)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager$4.onHasNext(GridCacheDistributedQueryManager.java:618)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.util.GridCloseableIteratorAdapter.hasNextX(GridCloseableIteratorAdapter.java:56)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.util.lang.GridIteratorAdapter.hasNext(GridIteratorAdapter.java:45)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.QueryCursorImpl$LazyIterator.hasNext(QueryCursorImpl.java:252)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.AutoClosableCursorIterator.hasNext(AutoClosableCursorIterator.java:49)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> java.base/java.lang.Iterable.forEach(Iterable.java:74)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.assertSnapshotCacheKeys(AbstractSnapshotSelfTest.java:723)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotSelfTest.testClientHandlesSnapshotFailOnStartStage(IgniteClusterSnapshotSelfTest.java:1331)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> java.base/java.lang.reflect.Method.invoke(Method.java:569)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.testframework.junits.GridAbstractTest$6.run(GridAbstractTest.java:2464)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> java.base/java.lang.Thread.run(Thread.java:840)
> [10:50:42]W: [org.apache.ignite:ignite-core] Caused by: class
> org.apache.ignite.cluster.ClusterTopologyException: Remote node has left
> topology: ee282fb0-8ec1-422f-b9d0-00f6f4f00001
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.util.IgniteUtils$3.apply(IgniteUtils.java:574)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.util.IgniteUtils$3.apply(IgniteUtils.java:572)
> [10:50:42]W: [org.apache.ignite:ignite-core] ... 22 more
> [10:50:42]W: [org.apache.ignite:ignite-core] Caused by: class
> org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Remote
> node has left topology: ee282fb0-8ec1-422f-b9d0-00f6f4f00001
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryFuture.onNodeLeft(GridCacheDistributedQueryFuture.java:160)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager.sendRequest(GridCacheDistributedQueryManager.java:724)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryFuture.startQuery(GridCacheDistributedQueryFuture.java:258)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager.queryDistributed(GridCacheDistributedQueryManager.java:526)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager.queryDistributed(GridCacheDistributedQueryManager.java:489)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager.scanQueryDistributed(GridCacheDistributedQueryManager.java:572)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.query.CacheQuery.executeScanQuery(CacheQuery.java:778)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl$1.applyx(IgniteCacheProxyImpl.java:490)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl$1.applyx(IgniteCacheProxyImpl.java:488)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.util.lang.IgniteOutClosureX.apply(IgniteOutClosureX.java:36)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.query.GridQueryProcessor.executeQuery(GridQueryProcessor.java:3906)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.query(IgniteCacheProxyImpl.java:487)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.query(IgniteCacheProxyImpl.java:809)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.query(GatewayProtectedCacheProxy.java:400)
> [10:50:42]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.assertSnapshotCacheKeys(AbstractSnapshotSelfTest.java:722)
> [10:50:42]W: [org.apache.ignite:ignite-core] ... 13 more
> [10:50:42]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:42,960][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][G] Ignite work
> directory is not provided, automatically resolved to:
> /opt/teamcity/work/7bc1c54bc719b67c/work
> [10:50:42]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:42,962][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][IgniteClusterSnapshotSelfTest0]
> Peer class loading is enabled (disable it in production for performance and
> deployment consistency reasons)
> [10:50:42]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:42,972][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][TestRecordingCommunicationSpi]
> Message queue limit is set to 0 which may lead to potential OOMEs when
> running cache operations in FULL_ASYNC or PRIMARY_SYNC modes due to message
> queues growth on sender and receiver sides.
> [10:50:43]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:43,224][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][G] Ignite work
> directory is not provided, automatically resolved to:
> /opt/teamcity/work/7bc1c54bc719b67c/work
> [10:50:43]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:43,226][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][IgniteClusterSnapshotSelfTest1]
> Peer class loading is enabled (disable it in production for performance and
> deployment consistency reasons)
> [10:50:43]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:43,236][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][TestRecordingCommunicationSpi]
> Message queue limit is set to 0 which may lead to potential OOMEs when
> running cache operations in FULL_ASYNC or PRIMARY_SYNC modes due to message
> queues growth on sender and receiver sides.
> [10:50:43]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:43,436][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][G] Ignite work
> directory is not provided, automatically resolved to:
> /opt/teamcity/work/7bc1c54bc719b67c/work
> [10:50:43]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:43,438][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][IgniteClusterSnapshotSelfTest2]
> Peer class loading is enabled (disable it in production for performance and
> deployment consistency reasons)
> [10:50:43]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:43,446][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][TestRecordingCommunicationSpi]
> Message queue limit is set to 0 which may lead to potential OOMEs when
> running cache operations in FULL_ASYNC or PRIMARY_SYNC modes due to message
> queues growth on sender and receiver sides.
> [10:50:43]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:43,842][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][DataStreamerImpl]
> The Data Streamer loads data with 'allowOverwrite' set to false. It doesn't
> guarantee data consistency until successfully finishes. Streamer cancelation
> or topology change can cause data inconsistency. Concurrently created
> snapshot may contain inconsistent data and might not be restored entirely.
> [10:50:48]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:48,442][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][G] Ignite work
> directory is not provided, automatically resolved to:
> /opt/teamcity/work/7bc1c54bc719b67c/work
> [10:50:48]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:48,444][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][IgniteClusterSnapshotSelfTest3]
> Peer class loading is enabled (disable it in production for performance and
> deployment consistency reasons)
> [10:50:48]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:48,452][WARN
> ][test-runner-#3596%snapshot.IgniteClusterSnapshotSelfTest%][TestRecordingCommunicationSpi]
> Message queue limit is set to 0 which may lead to potential OOMEs when
> running cache operations in FULL_ASYNC or PRIMARY_SYNC modes due to message
> queues growth on sender and receiver sides.
> [10:50:48]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:48,518][WARN
> ][exchange-worker-#3653%snapshot.IgniteClusterSnapshotSelfTest0%][GridDhtPartitionsExchangeFuture]
> Stop waiting for partitions release latch: Got interrupted while waiting for
> future to complete.
> [10:50:48]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:48,531][ERROR][disco-event-worker-#3778%snapshot.IgniteClusterSnapshotSelfTest2%][SnapshotFutureTask]
> Snapshot task has accepted exception to stop
> [10:50:48]W: [org.apache.ignite:ignite-core] class
> org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Snapshot
> operation interrupted, because baseline node left the cluster:
> aeee4f40-d319-4633-b318-df33f0000000
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.lambda$start0$20(IgniteSnapshotManager.java:591)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager$DiscoveryListenerWrapper.onEvent(GridEventStorageManager.java:1471)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:912)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:897)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record0(GridEventStorageManager.java:355)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record(GridEventStorageManager.java:318)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryEventHandler.recordEvent(GridDiscoveryManager.java:3012)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryEventHandler.body0(GridDiscoveryManager.java:3191)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryEventHandler.body(GridDiscoveryManager.java:3032)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> java.base/java.lang.Thread.run(Thread.java:840)
> [10:50:48]W: [org.apache.ignite:ignite-core]
> [2026-10-19T10:50:48,532][ERROR][disco-event-worker-#3715%snapshot.IgniteClusterSnapshotSelfTest1%][SnapshotFutureTask]
> Snapshot task has accepted exception to stop
> [10:50:48]W: [org.apache.ignite:ignite-core] class
> org.apache.ignite.internal.cluster.ClusterTopologyCheckedException: Snapshot
> operation interrupted, because baseline node left the cluster:
> aeee4f40-d319-4633-b318-df33f0000000
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.lambda$start0$20(IgniteSnapshotManager.java:591)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager$DiscoveryListenerWrapper.onEvent(GridEventStorageManager.java:1471)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:912)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.notifyListeners(GridEventStorageManager.java:897)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record0(GridEventStorageManager.java:355)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager.record(GridEventStorageManager.java:318)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryEventHandler.recordEvent(GridDiscoveryManager.java:3012)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryEventHandler.body0(GridDiscoveryManager.java:3191)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$DiscoveryEventHandler.body(GridDiscoveryManager.java:3032)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:125)
> [10:50:48]W: [org.apache.ignite:ignite-core] at
> java.base/java.lang.Thread.run(Thread.java:840) {code}
> Run logs attached
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
