[
https://issues.apache.org/jira/browse/IGNITE-26989?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ivan Bessonov updated IGNITE-26989:
-----------------------------------
Description:
{noformat}
java.lang.AssertionError: There are replicas alive [replicas=[23_part_7,
23_part_4]]
at
java.base/java.lang.invoke.MethodHandle.invokeWithArguments(MethodHandle.java:732)
at
org.apache.ignite.internal.util.ExceptionUtils$8.copy(ExceptionUtils.java:1032)
at
org.apache.ignite.internal.util.ExceptionUtils$ExceptionFactory.createCopy(ExceptionUtils.java:878)
at
org.apache.ignite.internal.util.ExceptionUtils.copyExceptionWithCause(ExceptionUtils.java:680)
at
org.apache.ignite.internal.util.ExceptionUtils.copyExceptionWithCauseInternal(ExceptionUtils.java:813)
at
org.apache.ignite.internal.util.ExceptionUtils.copyExceptionWithCause(ExceptionUtils.java:658)
at
org.apache.ignite.internal.app.IgniteServerImpl.tryToCopyExceptionWithCause(IgniteServerImpl.java:518)
at
org.apache.ignite.internal.app.IgniteServerImpl.sync(IgniteServerImpl.java:510)
at
org.apache.ignite.internal.app.IgniteServerImpl.shutdown(IgniteServerImpl.java:336)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
at
java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:179)
at
java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1625)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
at
java.base/java.util.stream.ForEachOps$ForEachTask.compute(ForEachOps.java:290)
at
java.base/java.util.concurrent.CountedCompleter.exec(CountedCompleter.java:754)
at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:373)
at java.base/java.util.concurrent.ForkJoinTask.invoke(ForkJoinTask.java:686)
at
java.base/java.util.stream.ForEachOps$ForEachOp.evaluateParallel(ForEachOps.java:159)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateParallel(ForEachOps.java:173)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:233)
at
java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:596)
at org.apache.ignite.internal.Cluster.shutdown(Cluster.java:636)
at
org.apache.ignite.internal.ClusterPerClassIntegrationTest.stopCluster(ClusterPerClassIntegrationTest.java:191)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
at
java.base/java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
Caused by: java.util.concurrent.ExecutionException: java.lang.AssertionError:
There are replicas alive [replicas=[23_part_7, 23_part_4]]
at
java.base/java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396)
at
java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073)
at
org.apache.ignite.internal.app.IgniteServerImpl.sync(IgniteServerImpl.java:508)
... 19 more
Caused by: java.lang.AssertionError: There are replicas alive
[replicas=[23_part_7, 23_part_4]]
at
org.apache.ignite.internal.replicator.ReplicaManager.lambda$stopAsync$30(ReplicaManager.java:1005)
at
org.apache.ignite.internal.util.IgniteUtils.lambda$closeAllManually$1(IgniteUtils.java:627)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
at
java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:179)
at
java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:992)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
at
java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at
java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:596)
at
org.apache.ignite.internal.util.IgniteUtils.closeAllManually(IgniteUtils.java:625)
at
org.apache.ignite.internal.util.IgniteUtils.closeAllManually(IgniteUtils.java:659)
at
org.apache.ignite.internal.replicator.ReplicaManager.stopAsync(ReplicaManager.java:1002)
at
org.apache.ignite.internal.util.IgniteUtils.lambda$stopAsync$6(IgniteUtils.java:1338)
at
java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:197)
at
java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:179)
at
java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1625)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:575)
at
java.base/java.util.stream.AbstractPipeline.evaluateToArrayNode(AbstractPipeline.java:260)
at
java.base/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:616)
at
org.apache.ignite.internal.util.IgniteUtils.stopAsync(IgniteUtils.java:1344)
at
org.apache.ignite.internal.util.IgniteUtils.stopAsync(IgniteUtils.java:1386)
at
org.apache.ignite.internal.app.LifecycleManager.initiateAllComponentsStop(LifecycleManager.java:174)
at
org.apache.ignite.internal.app.LifecycleManager.stopNode(LifecycleManager.java:148)
at org.apache.ignite.internal.app.IgniteImpl.stopAsync(IgniteImpl.java:2312)
at
org.apache.ignite.internal.app.IgniteServerImpl.doShutdownAsync(IgniteServerImpl.java:320)
at
org.apache.ignite.internal.app.IgniteServerImpl.lambda$chainRestartOrShutdownAction$5(IgniteServerImpl.java:259)
at
java.base/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187)
at
java.base/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2309)
at
org.apache.ignite.internal.app.IgniteServerImpl.chainRestartOrShutdownAction(IgniteServerImpl.java:259)
at
org.apache.ignite.internal.app.IgniteServerImpl.shutdownAsync(IgniteServerImpl.java:296)
... 19 more{noformat}
h3. Root cause
We were not in fact guaranteeing that there would be no alive replicas when we
stop the node. Stopping the node does not involve zone locks, we also never
tracked replicas that are concurrently starting, only those that are already
started, thus having a data race. There was a missing busy lock too.
h3. Fix
We track completable futures for starting replication groups and wait for them
before we start stopping alive replication groups.
was:
{noformat}
java.lang.AssertionError: There are replicas alive [replicas=[23_part_7,
23_part_4]]
at
java.base/java.lang.invoke.MethodHandle.invokeWithArguments(MethodHandle.java:732)
at
org.apache.ignite.internal.util.ExceptionUtils$8.copy(ExceptionUtils.java:1032)
at
org.apache.ignite.internal.util.ExceptionUtils$ExceptionFactory.createCopy(ExceptionUtils.java:878)
at
org.apache.ignite.internal.util.ExceptionUtils.copyExceptionWithCause(ExceptionUtils.java:680)
at
org.apache.ignite.internal.util.ExceptionUtils.copyExceptionWithCauseInternal(ExceptionUtils.java:813)
at
org.apache.ignite.internal.util.ExceptionUtils.copyExceptionWithCause(ExceptionUtils.java:658)
at
org.apache.ignite.internal.app.IgniteServerImpl.tryToCopyExceptionWithCause(IgniteServerImpl.java:518)
at
org.apache.ignite.internal.app.IgniteServerImpl.sync(IgniteServerImpl.java:510)
at
org.apache.ignite.internal.app.IgniteServerImpl.shutdown(IgniteServerImpl.java:336)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
at
java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:179)
at
java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1625)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
at
java.base/java.util.stream.ForEachOps$ForEachTask.compute(ForEachOps.java:290)
at
java.base/java.util.concurrent.CountedCompleter.exec(CountedCompleter.java:754)
at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:373)
at java.base/java.util.concurrent.ForkJoinTask.invoke(ForkJoinTask.java:686)
at
java.base/java.util.stream.ForEachOps$ForEachOp.evaluateParallel(ForEachOps.java:159)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateParallel(ForEachOps.java:173)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:233)
at
java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:596)
at org.apache.ignite.internal.Cluster.shutdown(Cluster.java:636)
at
org.apache.ignite.internal.ClusterPerClassIntegrationTest.stopCluster(ClusterPerClassIntegrationTest.java:191)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
at
java.base/java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
Caused by: java.util.concurrent.ExecutionException: java.lang.AssertionError:
There are replicas alive [replicas=[23_part_7, 23_part_4]]
at
java.base/java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396)
at
java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073)
at
org.apache.ignite.internal.app.IgniteServerImpl.sync(IgniteServerImpl.java:508)
... 19 more
Caused by: java.lang.AssertionError: There are replicas alive
[replicas=[23_part_7, 23_part_4]]
at
org.apache.ignite.internal.replicator.ReplicaManager.lambda$stopAsync$30(ReplicaManager.java:1005)
at
org.apache.ignite.internal.util.IgniteUtils.lambda$closeAllManually$1(IgniteUtils.java:627)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
at
java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:179)
at
java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:992)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
at
java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
at
java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at
java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:596)
at
org.apache.ignite.internal.util.IgniteUtils.closeAllManually(IgniteUtils.java:625)
at
org.apache.ignite.internal.util.IgniteUtils.closeAllManually(IgniteUtils.java:659)
at
org.apache.ignite.internal.replicator.ReplicaManager.stopAsync(ReplicaManager.java:1002)
at
org.apache.ignite.internal.util.IgniteUtils.lambda$stopAsync$6(IgniteUtils.java:1338)
at
java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:197)
at
java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:179)
at
java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1625)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:575)
at
java.base/java.util.stream.AbstractPipeline.evaluateToArrayNode(AbstractPipeline.java:260)
at
java.base/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:616)
at
org.apache.ignite.internal.util.IgniteUtils.stopAsync(IgniteUtils.java:1344)
at
org.apache.ignite.internal.util.IgniteUtils.stopAsync(IgniteUtils.java:1386)
at
org.apache.ignite.internal.app.LifecycleManager.initiateAllComponentsStop(LifecycleManager.java:174)
at
org.apache.ignite.internal.app.LifecycleManager.stopNode(LifecycleManager.java:148)
at org.apache.ignite.internal.app.IgniteImpl.stopAsync(IgniteImpl.java:2312)
at
org.apache.ignite.internal.app.IgniteServerImpl.doShutdownAsync(IgniteServerImpl.java:320)
at
org.apache.ignite.internal.app.IgniteServerImpl.lambda$chainRestartOrShutdownAction$5(IgniteServerImpl.java:259)
at
java.base/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187)
at
java.base/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2309)
at
org.apache.ignite.internal.app.IgniteServerImpl.chainRestartOrShutdownAction(IgniteServerImpl.java:259)
at
org.apache.ignite.internal.app.IgniteServerImpl.shutdownAsync(IgniteServerImpl.java:296)
... 19 more{noformat}
> There are replicas alive in
> ItDisasterRecoveryControllerRestartPartitionsTest.executionError
> --------------------------------------------------------------------------------------------
>
> Key: IGNITE-26989
> URL: https://issues.apache.org/jira/browse/IGNITE-26989
> Project: Ignite
> Issue Type: Bug
> Components: replication ai3
> Reporter: Ivan Bessonov
> Assignee: Ivan Bessonov
> Priority: Major
> Labels: ignite-3
> Fix For: 3.2
>
> Time Spent: 10m
> Remaining Estimate: 0h
>
> {noformat}
> java.lang.AssertionError: There are replicas alive [replicas=[23_part_7,
> 23_part_4]]
> at
> java.base/java.lang.invoke.MethodHandle.invokeWithArguments(MethodHandle.java:732)
> at
> org.apache.ignite.internal.util.ExceptionUtils$8.copy(ExceptionUtils.java:1032)
> at
> org.apache.ignite.internal.util.ExceptionUtils$ExceptionFactory.createCopy(ExceptionUtils.java:878)
> at
> org.apache.ignite.internal.util.ExceptionUtils.copyExceptionWithCause(ExceptionUtils.java:680)
> at
> org.apache.ignite.internal.util.ExceptionUtils.copyExceptionWithCauseInternal(ExceptionUtils.java:813)
> at
> org.apache.ignite.internal.util.ExceptionUtils.copyExceptionWithCause(ExceptionUtils.java:658)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.tryToCopyExceptionWithCause(IgniteServerImpl.java:518)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.sync(IgniteServerImpl.java:510)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.shutdown(IgniteServerImpl.java:336)
> at
> java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
> at
> java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:179)
> at
> java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1625)
> at
> java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
> at
> java.base/java.util.stream.ForEachOps$ForEachTask.compute(ForEachOps.java:290)
> at
> java.base/java.util.concurrent.CountedCompleter.exec(CountedCompleter.java:754)
> at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:373)
> at java.base/java.util.concurrent.ForkJoinTask.invoke(ForkJoinTask.java:686)
> at
> java.base/java.util.stream.ForEachOps$ForEachOp.evaluateParallel(ForEachOps.java:159)
> at
> java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateParallel(ForEachOps.java:173)
> at
> java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:233)
> at
> java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:596)
> at org.apache.ignite.internal.Cluster.shutdown(Cluster.java:636)
> at
> org.apache.ignite.internal.ClusterPerClassIntegrationTest.stopCluster(ClusterPerClassIntegrationTest.java:191)
> at java.base/java.lang.reflect.Method.invoke(Method.java:568)
> at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
> at
> java.base/java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092)
> at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
> Caused by: java.util.concurrent.ExecutionException: java.lang.AssertionError:
> There are replicas alive [replicas=[23_part_7, 23_part_4]]
> at
> java.base/java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396)
> at
> java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.sync(IgniteServerImpl.java:508)
> ... 19 more
> Caused by: java.lang.AssertionError: There are replicas alive
> [replicas=[23_part_7, 23_part_4]]
> at
> org.apache.ignite.internal.replicator.ReplicaManager.lambda$stopAsync$30(ReplicaManager.java:1005)
> at
> org.apache.ignite.internal.util.IgniteUtils.lambda$closeAllManually$1(IgniteUtils.java:627)
> at
> java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
> at
> java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:179)
> at
> java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:992)
> at
> java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
> at
> java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
> at
> java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
> at
> java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
> at
> java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> at
> java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:596)
> at
> org.apache.ignite.internal.util.IgniteUtils.closeAllManually(IgniteUtils.java:625)
> at
> org.apache.ignite.internal.util.IgniteUtils.closeAllManually(IgniteUtils.java:659)
> at
> org.apache.ignite.internal.replicator.ReplicaManager.stopAsync(ReplicaManager.java:1002)
> at
> org.apache.ignite.internal.util.IgniteUtils.lambda$stopAsync$6(IgniteUtils.java:1338)
> at
> java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:197)
> at
> java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:179)
> at
> java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1625)
> at
> java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
> at
> java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
> at
> java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:575)
> at
> java.base/java.util.stream.AbstractPipeline.evaluateToArrayNode(AbstractPipeline.java:260)
> at
> java.base/java.util.stream.ReferencePipeline.toArray(ReferencePipeline.java:616)
> at
> org.apache.ignite.internal.util.IgniteUtils.stopAsync(IgniteUtils.java:1344)
> at
> org.apache.ignite.internal.util.IgniteUtils.stopAsync(IgniteUtils.java:1386)
> at
> org.apache.ignite.internal.app.LifecycleManager.initiateAllComponentsStop(LifecycleManager.java:174)
> at
> org.apache.ignite.internal.app.LifecycleManager.stopNode(LifecycleManager.java:148)
> at org.apache.ignite.internal.app.IgniteImpl.stopAsync(IgniteImpl.java:2312)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.doShutdownAsync(IgniteServerImpl.java:320)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.lambda$chainRestartOrShutdownAction$5(IgniteServerImpl.java:259)
> at
> java.base/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187)
> at
> java.base/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2309)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.chainRestartOrShutdownAction(IgniteServerImpl.java:259)
> at
> org.apache.ignite.internal.app.IgniteServerImpl.shutdownAsync(IgniteServerImpl.java:296)
> ... 19 more{noformat}
> h3. Root cause
> We were not in fact guaranteeing that there would be no alive replicas when
> we stop the node. Stopping the node does not involve zone locks, we also
> never tracked replicas that are concurrently starting, only those that are
> already started, thus having a data race. There was a missing busy lock too.
> h3. Fix
> We track completable futures for starting replication groups and wait for
> them before we start stopping alive replication groups.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
