This is an automated email from the ASF dual-hosted git repository.
sdanilov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new c4b214c7a0 IGNITE-18710 Make default for delay to remove from logical
topology non-zero (#1636)
c4b214c7a0 is described below
commit c4b214c7a0166692b299b4427b77396c90b68604
Author: Roman Puchkovskiy <[email protected]>
AuthorDate: Tue Feb 7 13:15:48 2023 +0400
IGNITE-18710 Make default for delay to remove from logical topology
non-zero (#1636)
---
.../ignite/internal/cli/IntegrationTestBase.java | 7 +++--
.../repl/executor/ItIgnitePicocliCommandsTest.java | 2 +-
.../management/BaseItClusterManagementTest.java | 2 +-
.../management/ClusterManagementGroupManager.java | 1 +
.../ClusterManagementConfigurationSchema.java | 3 +-
.../internal/AbstractClusterIntegrationTest.java | 31 +++++++++++++++++--
.../java/org/apache/ignite/internal/Cluster.java | 33 +++++++++++++++++---
.../internal/compute/ItLogicalTopologyTest.java | 36 ++++++++++++++++++++--
8 files changed, 98 insertions(+), 17 deletions(-)
diff --git
a/modules/cli/src/integrationTest/java/org/apache/ignite/internal/cli/IntegrationTestBase.java
b/modules/cli/src/integrationTest/java/org/apache/ignite/internal/cli/IntegrationTestBase.java
index b809518b6a..eca4c7a2f3 100644
---
a/modules/cli/src/integrationTest/java/org/apache/ignite/internal/cli/IntegrationTestBase.java
+++
b/modules/cli/src/integrationTest/java/org/apache/ignite/internal/cli/IntegrationTestBase.java
@@ -101,8 +101,8 @@ public class IntegrationTestBase extends
BaseIgniteAbstractTest {
+ " }\n"
+ "}";
- /** Template for node bootstrap config with Scalecube settings for fast
failure detection. */
- protected static final String FAST_SWIM_NODE_BOOTSTRAP_CFG_TEMPLATE = "{\n"
+ /** Template for node bootstrap config with Scalecube and Logical Topology
settings for fast failure detection. */
+ protected static final String
FAST_FAILURE_DETECTION_NODE_BOOTSTRAP_CFG_TEMPLATE = "{\n"
+ " network: {\n"
+ " port: {},\n"
+ " nodeFinder: {\n"
@@ -117,7 +117,8 @@ public class IntegrationTestBase extends
BaseIgniteAbstractTest {
+ " gossipInterval: 10\n"
+ " },\n"
+ " }\n"
- + " }\n"
+ + " },"
+ + " cluster.failoverTimeout: 100\n"
+ "}";
/** Futures that are going to be completed when all nodes are started and
the cluster is initialized. */
diff --git
a/modules/cli/src/integrationTest/java/org/apache/ignite/internal/cli/core/repl/executor/ItIgnitePicocliCommandsTest.java
b/modules/cli/src/integrationTest/java/org/apache/ignite/internal/cli/core/repl/executor/ItIgnitePicocliCommandsTest.java
index 72aaa3d807..122a9befcc 100644
---
a/modules/cli/src/integrationTest/java/org/apache/ignite/internal/cli/core/repl/executor/ItIgnitePicocliCommandsTest.java
+++
b/modules/cli/src/integrationTest/java/org/apache/ignite/internal/cli/core/repl/executor/ItIgnitePicocliCommandsTest.java
@@ -89,7 +89,7 @@ public class ItIgnitePicocliCommandsTest extends
CliCommandTestInitializedIntegr
@Override
protected String nodeBootstrapConfigTemplate() {
- return FAST_SWIM_NODE_BOOTSTRAP_CFG_TEMPLATE;
+ return FAST_FAILURE_DETECTION_NODE_BOOTSTRAP_CFG_TEMPLATE;
}
@BeforeEach
diff --git
a/modules/cluster-management/src/integrationTest/java/org/apache/ignite/internal/cluster/management/BaseItClusterManagementTest.java
b/modules/cluster-management/src/integrationTest/java/org/apache/ignite/internal/cluster/management/BaseItClusterManagementTest.java
index d24d5070eb..0f5cb7359b 100644
---
a/modules/cluster-management/src/integrationTest/java/org/apache/ignite/internal/cluster/management/BaseItClusterManagementTest.java
+++
b/modules/cluster-management/src/integrationTest/java/org/apache/ignite/internal/cluster/management/BaseItClusterManagementTest.java
@@ -44,7 +44,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
public class BaseItClusterManagementTest extends BaseIgniteAbstractTest {
private static final int PORT_BASE = 10000;
- @InjectConfiguration
+ @InjectConfiguration("mock.failoverTimeout: 100")
private static ClusterManagementConfiguration cmgConfiguration;
@InjectConfiguration
diff --git
a/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/ClusterManagementGroupManager.java
b/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/ClusterManagementGroupManager.java
index 70767928da..bf0c7e8ff5 100644
---
a/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/ClusterManagementGroupManager.java
+++
b/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/ClusterManagementGroupManager.java
@@ -386,6 +386,7 @@ public class ClusterManagementGroupManager implements
IgniteComponent {
.filter(node ->
!physicalTopologyIds.contains(node.id()))
.collect(toUnmodifiableSet());
+ // TODO: IGNITE-18681 - respect removal timeout.
return nodesToRemove.isEmpty() ? completedFuture(null) :
service.removeFromCluster(nodesToRemove);
})
.thenApply(v -> service);
diff --git
a/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/configuration/ClusterManagementConfigurationSchema.java
b/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/configuration/ClusterManagementConfigurationSchema.java
index ab38f24f27..4dc156b411 100644
---
a/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/configuration/ClusterManagementConfigurationSchema.java
+++
b/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/configuration/ClusterManagementConfigurationSchema.java
@@ -38,8 +38,7 @@ public class ClusterManagementConfigurationSchema {
*/
@Value(hasDefault = true)
@Range(min = 0)
- // TODO: IGNITE-18630 - change this to a sensible default.
- public long failoverTimeout = 0;
+ public long failoverTimeout = TimeUnit.SECONDS.toMillis(30);
/** Maximum amount of time a validated node that has not yet completed the
join is allowed to remain validated (ms). */
@Value(hasDefault = true)
diff --git
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/AbstractClusterIntegrationTest.java
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/AbstractClusterIntegrationTest.java
index 0f18443650..6a891e4f39 100644
---
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/AbstractClusterIntegrationTest.java
+++
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/AbstractClusterIntegrationTest.java
@@ -19,6 +19,8 @@ package org.apache.ignite.internal;
import static
org.apache.ignite.internal.sql.engine.util.CursorUtils.getAllFromCursor;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
import java.nio.file.Path;
import java.util.List;
import org.apache.ignite.Ignite;
@@ -33,6 +35,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.platform.commons.support.ReflectionSupport;
/**
* Abstract integration test that starts and stops a cluster.
@@ -55,8 +58,8 @@ public abstract class AbstractClusterIntegrationTest extends
BaseIgniteAbstractT
+ " }\n"
+ "}";
- /** Template for node bootstrap config with Scalecube settings for fast
failure detection. */
- protected static final String FAST_SWIM_NODE_BOOTSTRAP_CFG_TEMPLATE = "{\n"
+ /** Template for node bootstrap config with Scalecube and Logical Topology
settings for fast failure detection. */
+ protected static final String
FAST_FAILURE_DETECTION_NODE_BOOTSTRAP_CFG_TEMPLATE = "{\n"
+ " network: {\n"
+ " port: {},\n"
+ " nodeFinder: {\n"
@@ -71,7 +74,8 @@ public abstract class AbstractClusterIntegrationTest extends
BaseIgniteAbstractT
+ " gossipInterval: 10\n"
+ " },\n"
+ " }\n"
- + " }\n"
+ + " },"
+ + " cluster.failoverTimeout: 100\n"
+ "}";
protected Cluster cluster;
@@ -109,6 +113,16 @@ public abstract class AbstractClusterIntegrationTest
extends BaseIgniteAbstractT
cluster.startAndInit(initialNodes());
}
+ private String invokeArglessMethod(Class<?> testClass, String methodName) {
+ Method method = ReflectionSupport.findMethod(testClass,
methodName).orElseThrow();
+
+ if (!Modifier.isStatic(method.getModifiers())) {
+ throw new IllegalStateException(methodName + " is expected to be
static");
+ }
+
+ return (String) ReflectionSupport.invokeMethod(method, null);
+ }
+
@AfterEach
@Timeout(60)
void shutdownCluster() {
@@ -143,6 +157,17 @@ public abstract class AbstractClusterIntegrationTest
extends BaseIgniteAbstractT
return cluster.startNode(nodeIndex);
}
+ /**
+ * Starts an Ignite node with the given index.
+ *
+ * @param nodeIndex Zero-based index (used to build node name).
+ * @param nodeBootstrapConfigTemplate Bootstrap config template to use for
this node.
+ * @return Started Ignite node.
+ */
+ protected final IgniteImpl startNode(int nodeIndex, String
nodeBootstrapConfigTemplate) {
+ return cluster.startNode(nodeIndex, nodeBootstrapConfigTemplate);
+ }
+
/**
* Stops a node by index.
*
diff --git
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/Cluster.java
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/Cluster.java
index 67bfde67c5..69c4a61147 100644
---
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/Cluster.java
+++
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/Cluster.java
@@ -88,7 +88,7 @@ public class Cluster {
private final Path workDir;
- private final String nodeBootstrapConfigTemplate;
+ private final String defaultNodeBootstrapConfigTemplate;
/** Cluster nodes. */
private final List<IgniteImpl> nodes = new CopyOnWriteArrayList<>();
@@ -110,10 +110,10 @@ public class Cluster {
/**
* Creates a new cluster with the given bootstrap config.
*/
- public Cluster(TestInfo testInfo, Path workDir, String
nodeBootstrapConfigTemplate) {
+ public Cluster(TestInfo testInfo, Path workDir, String
defaultNodeBootstrapConfigTemplate) {
this.testInfo = testInfo;
this.workDir = workDir;
- this.nodeBootstrapConfigTemplate = nodeBootstrapConfigTemplate;
+ this.defaultNodeBootstrapConfigTemplate =
defaultNodeBootstrapConfigTemplate;
}
/**
@@ -142,12 +142,23 @@ public class Cluster {
}
/**
- * Start a cluster node and return its startup future.
+ * Starts a cluster node with the default bootstrap config template and
returns its startup future.
*
* @param nodeIndex Index of the nodex to start.
* @return Future that will be completed when the node starts.
*/
public CompletableFuture<IgniteImpl> startClusterNode(int nodeIndex) {
+ return startClusterNode(nodeIndex, defaultNodeBootstrapConfigTemplate);
+ }
+
+ /**
+ * Starts a cluster node and returns its startup future.
+ *
+ * @param nodeIndex Index of the nodex to start.
+ * @param nodeBootstrapConfigTemplate Bootstrap config template to use for
this node.
+ * @return Future that will be completed when the node starts.
+ */
+ public CompletableFuture<IgniteImpl> startClusterNode(int nodeIndex,
String nodeBootstrapConfigTemplate) {
String nodeName = testNodeName(testInfo, nodeIndex);
String config =
IgniteStringFormatter.format(nodeBootstrapConfigTemplate, BASE_PORT +
nodeIndex, CONNECT_NODE_ADDR);
@@ -204,10 +215,22 @@ public class Cluster {
* is not initialized, the node is returned in a state in which it is
ready to join the cluster).
*/
public IgniteImpl startNode(int index) {
+ return startNode(index, defaultNodeBootstrapConfigTemplate);
+ }
+
+ /**
+ * Starts a new node with the given index.
+ *
+ * @param index Node index.
+ * @param nodeBootstrapConfigTemplate Bootstrap config template to use for
this node.
+ * @return Started node (if the cluster is already initialized, the node
is returned when it joins the cluster; if it
+ * is not initialized, the node is returned in a state in which it is
ready to join the cluster).
+ */
+ public IgniteImpl startNode(int index, String nodeBootstrapConfigTemplate)
{
IgniteImpl newIgniteNode;
try {
- newIgniteNode = startClusterNode(index).get(10, TimeUnit.SECONDS);
+ newIgniteNode = startClusterNode(index,
nodeBootstrapConfigTemplate).get(10, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
diff --git
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/compute/ItLogicalTopologyTest.java
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/compute/ItLogicalTopologyTest.java
index dc25c73540..a4e05a3d7e 100644
---
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/compute/ItLogicalTopologyTest.java
+++
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/compute/ItLogicalTopologyTest.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.compute;
import static
org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition;
import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -27,10 +28,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.ignite.Ignite;
import org.apache.ignite.internal.AbstractClusterIntegrationTest;
+import org.apache.ignite.internal.Cluster.NodeKnockout;
import org.apache.ignite.internal.app.IgniteImpl;
+import
org.apache.ignite.internal.cluster.management.configuration.ClusterManagementConfiguration;
import
org.apache.ignite.internal.cluster.management.topology.api.LogicalTopologyEventListener;
import
org.apache.ignite.internal.cluster.management.topology.api.LogicalTopologySnapshot;
import org.apache.ignite.internal.network.message.ScaleCubeMessage;
@@ -64,7 +69,7 @@ class ItLogicalTopologyTest extends
AbstractClusterIntegrationTest {
@Override
protected String getNodeBootstrapConfigTemplate() {
- return FAST_SWIM_NODE_BOOTSTRAP_CFG_TEMPLATE;
+ return FAST_FAILURE_DETECTION_NODE_BOOTSTRAP_CFG_TEMPLATE;
}
@Test
@@ -143,7 +148,6 @@ class ItLogicalTopologyTest extends
AbstractClusterIntegrationTest {
if (appearedNode.name().equals(secondIgnite.name())) {
secondIgniteAppeared.countDown();
}
-
}
});
@@ -170,6 +174,34 @@ class ItLogicalTopologyTest extends
AbstractClusterIntegrationTest {
assertTrue(secondIgniteDisappeared.await(10, TimeUnit.SECONDS), "Did
not see second node leaving in time");
}
+ @Test
+ void nodeDoesNotLeaveLogicalTopologyImmediatelyAfterBeingLostBySwim()
throws Exception {
+ IgniteImpl entryNode = node(0);
+
+ setInfiniteClusterFailoverTimeout(entryNode);
+
+ startNode(1);
+
+ entryNode.logicalTopologyService().addEventListener(listener);
+
+ // Knock the node out without allowing it to say good bye.
+ cluster.knockOutNode(1, NodeKnockout.PARTITION_NETWORK);
+
+ // 1 second is usually insufficient on my machine to get an event,
even if it's produced. On the CI we
+ // should probably account for spurious delays due to other processes,
hence 2 seconds.
+ waitForCondition(() -> !events.isEmpty(), 2_000);
+
+ assertThat(events, is(empty()));
+ }
+
+ private static void setInfiniteClusterFailoverTimeout(IgniteImpl node)
+ throws InterruptedException, ExecutionException, TimeoutException {
+
node.nodeConfiguration().getConfiguration(ClusterManagementConfiguration.KEY)
+ .failoverTimeout()
+ .update(Long.MAX_VALUE)
+ .get(10, TimeUnit.SECONDS);
+ }
+
private static class Event {
private final boolean appeared;
private final ClusterNode node;
