This is an automated email from the ASF dual-hosted git repository.
timoninmaxim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new ab278862bd3 IGNITE-18783 Fix DistributedProcess hangs on
AssertionError (#10537)
ab278862bd3 is described below
commit ab278862bd322e0d2d76557c263c6c1a7b97375c
Author: Maksim Timonin <[email protected]>
AuthorDate: Thu Feb 16 11:43:38 2023 +0300
IGNITE-18783 Fix DistributedProcess hangs on AssertionError (#10537)
---
.../managers/encryption/GridEncryptionManager.java | 8 +-
.../managers/encryption/GroupKeyChangeProcess.java | 6 +-
.../snapshot/IgniteSnapshotManager.java | 4 +-
.../snapshot/SnapshotRestoreProcess.java | 14 +-
.../util/distributed/DistributedProcess.java | 55 ++++---
.../internal/util/distributed/FullMessage.java | 6 +-
.../util/distributed/SingleNodeMessage.java | 6 +-
.../util/DistributedProcessErrorHandlingTest.java | 169 +++++++++++++++++++++
.../ignite/testsuites/IgniteUtilSelfTestSuite.java | 2 +
9 files changed, 228 insertions(+), 42 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GridEncryptionManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GridEncryptionManager.java
index 8e1f5be7b7e..f1f5f11c553 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GridEncryptionManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GridEncryptionManager.java
@@ -1568,7 +1568,7 @@ public class GridEncryptionManager extends
GridManagerAdapter<EncryptionSpi> imp
* @param res Results.
* @param err Errors.
*/
- private void finishPrepareMasterKeyChange(UUID id, Map<UUID, EmptyResult>
res, Map<UUID, Exception> err) {
+ private void finishPrepareMasterKeyChange(UUID id, Map<UUID, EmptyResult>
res, Map<UUID, Throwable> err) {
if (!err.isEmpty()) {
if (masterKeyChangeRequest != null &&
masterKeyChangeRequest.requestId().equals(id))
masterKeyChangeRequest = null;
@@ -1613,7 +1613,7 @@ public class GridEncryptionManager extends
GridManagerAdapter<EncryptionSpi> imp
* @param res Results.
* @param err Errors.
*/
- private void finishPerformMasterKeyChange(UUID id, Map<UUID, EmptyResult>
res, Map<UUID, Exception> err) {
+ private void finishPerformMasterKeyChange(UUID id, Map<UUID, EmptyResult>
res, Map<UUID, Throwable> err) {
completeMasterKeyChangeFuture(id, err);
}
@@ -1621,7 +1621,7 @@ public class GridEncryptionManager extends
GridManagerAdapter<EncryptionSpi> imp
* @param reqId Request id.
* @param err Exception.
*/
- private void completeMasterKeyChangeFuture(UUID reqId, Map<UUID,
Exception> err) {
+ private void completeMasterKeyChangeFuture(UUID reqId, Map<UUID,
Throwable> err) {
synchronized (opsMux) {
boolean isInitiator = masterKeyChangeFut != null &&
masterKeyChangeFut.id().equals(reqId);
@@ -1629,7 +1629,7 @@ public class GridEncryptionManager extends
GridManagerAdapter<EncryptionSpi> imp
return;
if (!F.isEmpty(err)) {
- Exception e = err.values().stream().findFirst().get();
+ Throwable e = err.values().stream().findFirst().get();
masterKeyChangeFut.onDone(e);
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyChangeProcess.java
b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyChangeProcess.java
index 855d34e22d4..a8b63bce215 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyChangeProcess.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/managers/encryption/GroupKeyChangeProcess.java
@@ -273,7 +273,7 @@ class GroupKeyChangeProcess {
* @param res Results.
* @param err Errors.
*/
- private void finishPrepare(UUID id, Map<UUID, EmptyResult> res, Map<UUID,
Exception> err) {
+ private void finishPrepare(UUID id, Map<UUID, EmptyResult> res, Map<UUID,
Throwable> err) {
if (!err.isEmpty()) {
if (req != null && req.requestId().equals(id))
req = null;
@@ -318,7 +318,7 @@ class GroupKeyChangeProcess {
* @param res Results.
* @param err Errors.
*/
- private void finishPerform(UUID id, Map<UUID, EmptyResult> res, Map<UUID,
Exception> err) {
+ private void finishPerform(UUID id, Map<UUID, EmptyResult> res, Map<UUID,
Throwable> err) {
completeFuture(id, err, fut);
}
@@ -328,7 +328,7 @@ class GroupKeyChangeProcess {
* @param fut Key change future.
* @return {@code True} if future was completed by this call.
*/
- private boolean completeFuture(UUID reqId, Map<UUID, Exception> err,
GroupKeyChangeFuture fut) {
+ private boolean completeFuture(UUID reqId, Map<UUID, Throwable> err,
GroupKeyChangeFuture fut) {
boolean isInitiator = fut != null && fut.id().equals(reqId);
if (!isInitiator || fut.isDone())
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
index 212e5ba8b07..4710dca0962 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java
@@ -891,7 +891,7 @@ public class IgniteSnapshotManager extends
GridCacheSharedManagerAdapter
* @param res Results.
* @param err Errors.
*/
- private void processLocalSnapshotStartStageResult(UUID id, Map<UUID,
SnapshotOperationResponse> res, Map<UUID, Exception> err) {
+ private void processLocalSnapshotStartStageResult(UUID id, Map<UUID,
SnapshotOperationResponse> res, Map<UUID, Throwable> err) {
if (cctx.kernalContext().clientNode())
return;
@@ -1106,7 +1106,7 @@ public class IgniteSnapshotManager extends
GridCacheSharedManagerAdapter
* @param res Results.
* @param err Errors.
*/
- private void processLocalSnapshotEndStageResult(UUID id, Map<UUID,
SnapshotOperationResponse> res, Map<UUID, Exception> err) {
+ private void processLocalSnapshotEndStageResult(UUID id, Map<UUID,
SnapshotOperationResponse> res, Map<UUID, Throwable> err) {
SnapshotOperationRequest snpReq = clusterSnpReq;
if (snpReq == null || !F.eq(id, snpReq.requestId()))
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
index 2d489728edc..ca09ee198e7 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java
@@ -771,13 +771,13 @@ public class SnapshotRestoreProcess {
* @param res Results.
* @param errs Errors.
*/
- private void finishPrepare(UUID reqId, Map<UUID,
SnapshotRestoreOperationResponse> res, Map<UUID, Exception> errs) {
+ private void finishPrepare(UUID reqId, Map<UUID,
SnapshotRestoreOperationResponse> res, Map<UUID, Throwable> errs) {
if (ctx.clientNode())
return;
SnapshotRestoreContext opCtx0 = opCtx;
- Exception failure = F.first(errs.values());
+ Throwable failure = F.first(errs.values());
assert opCtx0 != null || failure != null : "Context has not been
created on the node " + ctx.localNodeId();
@@ -1173,13 +1173,13 @@ public class SnapshotRestoreProcess {
* @param res Results.
* @param errs Errors.
*/
- private void finishPreload(UUID reqId, Map<UUID, Boolean> res, Map<UUID,
Exception> errs) {
+ private void finishPreload(UUID reqId, Map<UUID, Boolean> res, Map<UUID,
Throwable> errs) {
if (ctx.clientNode())
return;
SnapshotRestoreContext opCtx0 = opCtx;
- Exception failure = errs.values().stream().findFirst().
+ Throwable failure = errs.values().stream().findFirst().
orElse(checkNodeLeft(opCtx0.nodes(), res.keySet()));
opCtx0.errHnd.accept(failure);
@@ -1232,13 +1232,13 @@ public class SnapshotRestoreProcess {
* @param res Results.
* @param errs Errors.
*/
- private void finishCacheStart(UUID reqId, Map<UUID, Boolean> res,
Map<UUID, Exception> errs) {
+ private void finishCacheStart(UUID reqId, Map<UUID, Boolean> res,
Map<UUID, Throwable> errs) {
if (ctx.clientNode())
return;
SnapshotRestoreContext opCtx0 = opCtx;
- Exception failure = errs.values().stream().findFirst().
+ Throwable failure = errs.values().stream().findFirst().
orElse(checkNodeLeft(opCtx0.nodes(), res.keySet()));
if (failure == null) {
@@ -1375,7 +1375,7 @@ public class SnapshotRestoreProcess {
* @param res Results.
* @param errs Errors.
*/
- private void finishRollback(UUID reqId, Map<UUID, Boolean> res, Map<UUID,
Exception> errs) {
+ private void finishRollback(UUID reqId, Map<UUID, Boolean> res, Map<UUID,
Throwable> errs) {
if (ctx.clientNode())
return;
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
index f0081a5377b..ad0a84d2f42 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java
@@ -96,7 +96,7 @@ public class DistributedProcess<I extends Serializable, R
extends Serializable>
GridKernalContext ctx,
DistributedProcessType type,
Function<I, IgniteInternalFuture<R>> exec,
- CI3<UUID, Map<UUID, R>, Map<UUID, Exception>> finish
+ CI3<UUID, Map<UUID, R>, Map<UUID, Throwable>> finish
) {
this(ctx, type, exec, finish, (id, req) -> new InitMessage<>(id, type,
req));
}
@@ -112,7 +112,7 @@ public class DistributedProcess<I extends Serializable, R
extends Serializable>
GridKernalContext ctx,
DistributedProcessType type,
Function<I, IgniteInternalFuture<R>> exec,
- CI3<UUID, Map<UUID, R>, Map<UUID, Exception>> finish,
+ CI3<UUID, Map<UUID, R>, Map<UUID, Throwable>> finish,
BiFunction<UUID, I, ? extends InitMessage<I>> initMsgFactory
) {
this.ctx = ctx;
@@ -146,22 +146,32 @@ public class DistributedProcess<I extends Serializable, R
extends Serializable>
if (crd.isLocal())
initCoordinator(p, topVer);
- IgniteInternalFuture<R> fut = exec.apply((I)msg.request());
+ try {
+ IgniteInternalFuture<R> fut = exec.apply((I)msg.request());
- fut.listen(f -> {
- if (f.error() != null)
- p.resFut.onDone(f.error());
- else
- p.resFut.onDone(f.result());
+ fut.listen(f -> {
+ if (f.error() != null)
+ p.resFut.onDone(f.error());
+ else
+ p.resFut.onDone(f.result());
- if (!ctx.clientNode()) {
- assert crd != null;
+ if (!ctx.clientNode()) {
+ assert crd != null;
- sendSingleMessage(p);
- }
- });
+ sendSingleMessage(p);
+ }
+ });
+
+ p.initFut.onDone();
+ }
+ catch (Throwable err) {
+ U.error(log, "Failed to handle InitMessage [id=" + p.id + ']',
err);
- p.initFut.onDone();
+ p.resFut.onDone(err);
+
+ if (!ctx.clientNode())
+ sendSingleMessage(p);
+ }
});
ctx.discovery().setCustomEventListener(FullMessage.class, (topVer,
snd, msg0) -> {
@@ -180,9 +190,15 @@ public class DistributedProcess<I extends Serializable, R
extends Serializable>
return;
}
- finish.apply(p.id, msg.result(), msg.error());
-
- processes.remove(msg.processId());
+ try {
+ finish.apply(p.id, msg.result(), msg.error());
+ }
+ catch (Throwable err) {
+ U.error(log, "Failed to handle FullMessage [id=" + p.id + ']',
err);
+ }
+ finally {
+ processes.remove(msg.processId());
+ }
});
ctx.io().addMessageListener(GridTopic.TOPIC_DISTRIBUTED_PROCESS,
(nodeId, msg0, plc) -> {
@@ -274,8 +290,7 @@ public class DistributedProcess<I extends Serializable, R
extends Serializable>
private void sendSingleMessage(Process p) {
assert p.resFut.isDone();
- SingleNodeMessage<R> singleMsg = new SingleNodeMessage<>(p.id, type,
p.resFut.result(),
- (Exception)p.resFut.error());
+ SingleNodeMessage<R> singleMsg = new SingleNodeMessage<>(p.id, type,
p.resFut.result(), p.resFut.error());
UUID crdId = p.crdId;
@@ -333,7 +348,7 @@ public class DistributedProcess<I extends Serializable, R
extends Serializable>
private void finishProcess(Process p) {
HashMap<UUID, R> res = new HashMap<>();
- HashMap<UUID, Exception> err = new HashMap<>();
+ HashMap<UUID, Throwable> err = new HashMap<>();
p.singleMsgs.forEach((uuid, msg) -> {
if (msg.hasError())
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/FullMessage.java
b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/FullMessage.java
index 852cfe7b056..3522c463def 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/FullMessage.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/FullMessage.java
@@ -54,7 +54,7 @@ public class FullMessage<R extends Serializable> implements
DiscoveryCustomMessa
private Map<UUID, R> res;
/** Errors. */
- private Map<UUID, Exception> err;
+ private Map<UUID, Throwable> err;
/**
* @param processId Process id.
@@ -62,7 +62,7 @@ public class FullMessage<R extends Serializable> implements
DiscoveryCustomMessa
* @param res Results.
* @param err Errors
*/
- public FullMessage(UUID processId, DistributedProcessType type, Map<UUID,
R> res, Map<UUID, Exception> err) {
+ public FullMessage(UUID processId, DistributedProcessType type, Map<UUID,
R> res, Map<UUID, Throwable> err) {
this.processId = processId;
this.type = type.ordinal();
this.res = res;
@@ -106,7 +106,7 @@ public class FullMessage<R extends Serializable> implements
DiscoveryCustomMessa
}
/** @return Nodes errors. */
- public Map<UUID, Exception> error() {
+ public Map<UUID, Throwable> error() {
return err;
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java
b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java
index 6d46e600502..56f1062f98a 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/SingleNodeMessage.java
@@ -51,7 +51,7 @@ public class SingleNodeMessage<R extends Serializable>
implements Message {
private R resp;
/** Error. */
- private Exception err;
+ private Throwable err;
/** Empty constructor for marshalling purposes. */
public SingleNodeMessage() {
@@ -63,7 +63,7 @@ public class SingleNodeMessage<R extends Serializable>
implements Message {
* @param resp Single node response.
* @param err Error.
*/
- public SingleNodeMessage(UUID processId, DistributedProcessType type, R
resp, Exception err) {
+ public SingleNodeMessage(UUID processId, DistributedProcessType type, R
resp, Throwable err) {
this.processId = processId;
this.type = type.ordinal();
this.resp = resp;
@@ -194,7 +194,7 @@ public class SingleNodeMessage<R extends Serializable>
implements Message {
}
/** @return Error. */
- public Exception error() {
+ public Throwable error() {
return err;
}
}
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/util/DistributedProcessErrorHandlingTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/util/DistributedProcessErrorHandlingTest.java
new file mode 100644
index 00000000000..674157e4328
--- /dev/null
+++
b/modules/core/src/test/java/org/apache/ignite/internal/util/DistributedProcessErrorHandlingTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.StopNodeFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.distributed.DistributedProcess;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static
org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.TEST_PROCESS;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+
+/** */
+@RunWith(Parameterized.class)
+public class DistributedProcessErrorHandlingTest extends
GridCommonAbstractTest {
+ /** */
+ private static final int SRV_NODES = 3;
+
+ /** If {@code true} then client fails, otherwise server node fails. */
+ @Parameterized.Parameter
+ public boolean failClient;
+
+ /** */
+ @Parameterized.Parameters(name = "failClient={0}")
+ public static Iterable<Boolean> params() {
+ return F.asList(false, true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String
instanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(instanceName);
+
+ cfg.setFailureHandler(new StopNodeFailureHandler());
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ startGrids(SRV_NODES);
+
+ startClientGrid();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() {
+ stopAllGrids();
+ }
+
+ /** */
+ @Test
+ public void testBackgroundExecFailureHandled() throws Exception {
+ checkDistributedProcess((ign, latch) ->
+ new DistributedProcess<>(ign.context(), TEST_PROCESS,
+ req -> runAsync(() -> {
+ failOnNode(ign); // Fails processing request in a spawned
thread.
+
+ return 0;
+ }),
+ (id, res, err) -> {
+ if (failClient)
+ assertEquals(SRV_NODES, res.values().size());
+ else {
+ assertEquals(SRV_NODES - 1, res.values().size());
+ assertEquals(1, err.size());
+ assertTrue(err.get(grid(1).localNode().id())
instanceof AssertionError);
+ }
+
+ latch.countDown();
+ }));
+ }
+
+ /** */
+ @Test
+ public void testExecFailureHandled() throws Exception {
+ checkDistributedProcess((ign, latch) ->
+ new DistributedProcess<>(ign.context(), TEST_PROCESS,
+ req -> {
+ failOnNode(ign); // Fails processing request in the
discovery thread.
+
+ return new GridFinishedFuture<>(0);
+ },
+ (id, res, err) -> {
+ if (failClient)
+ assertEquals(SRV_NODES, res.values().size());
+ else {
+ assertEquals(SRV_NODES - 1, res.values().size());
+ assertEquals(1, err.size());
+ assertTrue(err.get(grid(1).localNode().id())
instanceof AssertionError);
+ }
+
+ latch.countDown();
+ }));
+ }
+
+ /** */
+ @Test
+ public void testFinishFailureHandled() throws Exception {
+ checkDistributedProcess((ign, latch) ->
+ new DistributedProcess<>(ign.context(), TEST_PROCESS,
+ req -> new GridFinishedFuture<>(0),
+ (uuid, res, err) -> {
+ assertEquals(SRV_NODES, res.values().size());
+ latch.countDown();
+
+ failOnNode(ign);
+ }));
+ }
+
+ /** */
+ private void checkDistributedProcess(
+ BiFunction<IgniteEx, CountDownLatch, DistributedProcess<Integer,
Integer>> processFactory
+ ) throws Exception {
+ DistributedProcess<Integer, Integer> process = null;
+
+ CountDownLatch latch = new CountDownLatch(SRV_NODES + 1);
+
+ for (Ignite g: G.allGrids())
+ process = processFactory.apply((IgniteEx)g, latch);
+
+ process.start(UUID.randomUUID(), 0);
+
+ assertTrue(latch.await(5, TimeUnit.SECONDS));
+
+ // Just checks that node is alive.
+ grid(1).cluster().state(ClusterState.INACTIVE);
+
+ awaitPartitionMapExchange();
+
+ waitForTopology(SRV_NODES + 1);
+ }
+
+ /** Checks whether to fail on specified node. */
+ private void failOnNode(IgniteEx ign) {
+ if (failClient)
+ assert !ign.configuration().isClientMode();
+ else
+ assert getTestIgniteInstanceIndex(ign.name()) != 1;
+ }
+}
diff --git
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index 5126a479aea..02667237d4c 100644
---
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.IgniteVersionUtilsSelfTest;
import org.apache.ignite.internal.pagemem.impl.PageIdUtilsSelfTest;
import org.apache.ignite.internal.util.BasicRateLimiterTest;
import org.apache.ignite.internal.util.DistributedProcessCoordinatorLeftTest;
+import org.apache.ignite.internal.util.DistributedProcessErrorHandlingTest;
import org.apache.ignite.internal.util.GridArraysSelfTest;
import org.apache.ignite.internal.util.GridConcurrentMultiPairQueueTest;
import org.apache.ignite.internal.util.GridCountDownCallbackTest;
@@ -143,6 +144,7 @@ import org.junit.runners.Suite;
GridCountDownCallbackTest.class,
+ DistributedProcessErrorHandlingTest.class,
DistributedProcessCoordinatorLeftTest.class,
BasicRateLimiterTest.class,
