This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a commit to branch ignite-18171 in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit e95a4a992a6f86beeee2336a6f308682d6d5a139 Author: amashenkov <[email protected]> AuthorDate: Wed Nov 30 18:05:26 2022 +0300 wip. --- .../ItNodeRestartTest.java} | 229 ++++++---------- .../{ => cluster}/ItNodeStartStopTest.java | 301 +++++++++------------ 2 files changed, 217 insertions(+), 313 deletions(-) diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/ItNodeStartStopTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/cluster/ItNodeRestartTest.java similarity index 75% copy from modules/runner/src/integrationTest/java/org/apache/ignite/internal/ItNodeStartStopTest.java copy to modules/runner/src/integrationTest/java/org/apache/ignite/internal/cluster/ItNodeRestartTest.java index 377059b8e2..86e194d014 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/ItNodeStartStopTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/cluster/ItNodeRestartTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.internal; +package org.apache.ignite.internal.cluster; import static org.apache.ignite.internal.sql.engine.util.CursorUtils.getAllFromCursor; import static org.apache.ignite.internal.testframework.IgniteTestUtils.assertThrowsWithCause; @@ -27,10 +27,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import io.micronaut.configuration.picocli.MicronautFactory; import io.micronaut.context.ApplicationContext; import io.micronaut.context.env.Environment; -import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; -import java.lang.invoke.MethodHandles; import java.lang.reflect.Method; import java.nio.file.Path; import java.util.ArrayList; @@ -40,11 +38,13 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.BiPredicate; @@ -58,12 +58,12 @@ import org.apache.ignite.internal.app.EnvironmentDefaultValueProvider; import org.apache.ignite.internal.app.IgniteImpl; import org.apache.ignite.internal.cli.commands.TopLevelCliCommand; import org.apache.ignite.internal.hlc.HybridTimestamp; -import org.apache.ignite.internal.replicator.ReplicaManager; import org.apache.ignite.internal.sql.engine.QueryContext; import org.apache.ignite.internal.sql.engine.QueryProperty; import org.apache.ignite.internal.sql.engine.property.PropertiesHolder; import org.apache.ignite.internal.sql.engine.session.SessionId; import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest; +import org.apache.ignite.internal.testframework.IgniteTestUtils; import org.apache.ignite.internal.testframework.WithSystemProperty; import org.apache.ignite.internal.testframework.WorkDirectory; import org.apache.ignite.internal.testframework.WorkDirectoryExtension; @@ -75,11 +75,11 @@ import org.hamcrest.Matchers; import org.hamcrest.text.IsEmptyString; import org.jetbrains.annotations.Nullable; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestFactory; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; @@ -87,14 +87,17 @@ import org.mockito.Mockito; import picocli.CommandLine; /** - * Test node start/stop in different scenarios and validate grid components behavior depending on availability/absence of quorums. + * Test node restart in different scenarios and validate grid components behavior depending on availability/absence of quorums. */ -//TODO: Fix expected messages in assertThrows +// TODO: Fix expected messages in assertThrows // TODO: Create 2 distribution zones, which spans a single node and both data nodes, and a tables in these zones. @ExtendWith(WorkDirectoryExtension.class) @WithSystemProperty(key = "org.jline.terminal.dumb", value = "true") -public class ItNodeStartStopTest extends BaseIgniteAbstractTest { +public class ItNodeRestartTest extends BaseIgniteAbstractTest { public static final int NODE_JOIN_WAIT_TIMEOUT = 5; + public static final Runnable NOOP = () -> { + }; + /** Work directory. */ @WorkDirectory private static Path WORK_DIR; @@ -173,36 +176,30 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { ); /** Cluster nodes. */ - private final Map<String, Ignite> clusterNodes = new HashMap<>(); + private final Map<String, Future<Ignite>> clusterNodes = new HashMap<>(); /** Runs after each test sequence. */ @BeforeEach - public void beforeEach() throws Exception { + public void beforeEach() { List<CompletableFuture<Ignite>> futures = new ArrayList<>(); // Start nodes. - nodesCfg.forEach((k, v) -> futures.add(IgnitionManager.start(k, v, WORK_DIR.resolve(k)))); + for (Entry<String, String> entry : nodesCfg.entrySet()) { + String nodeName = entry.getKey(); + String nodeConfig = entry.getValue(); + + futures.add(IgnitionManager.start(nodeName, nodeConfig, WORK_DIR.resolve(nodeName))); + } // Init cluster. - //TODO: Use dedicated node for metastorage. - IgnitionManager.init(CMG_NODE, List.of(CMG_NODE /* METASTORAGE_NODE */), List.of(CMG_NODE), "cluster"); + IgnitionManager.init(CMG_NODE, List.of(METASTORAGE_NODE), List.of(CMG_NODE), "cluster"); - for (CompletableFuture<Ignite> future : futures) { - assertThat(future, willCompleteSuccessfully()); - } + assertThat(CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)), willCompleteSuccessfully()); // Create tables. IgniteImpl node = (IgniteImpl) futures.get(0).join(); sql(node, null, "CREATE TABLE tbl1 (id INT PRIMARY KEY, val INT) WITH partitions = 1"); - for (CompletableFuture<Ignite> f : futures) { - ReplicaManager replicaMgr = (ReplicaManager) MethodHandles.privateLookupIn(IgniteImpl.class, MethodHandles.lookup()) - .findVarHandle(IgniteImpl.class, "replicaMgr", ReplicaManager.class) - .get(f.get()); - - assertTrue(DATA_NODE.equals(f.get().name()) ^ replicaMgr.startedGroups().isEmpty()); - } - sql(node, null, "INSERT INTO tbl1(id, val) VALUES (1,1)"); // Shutdown cluster. @@ -213,7 +210,7 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { /** Runs after each test sequence. */ @AfterEach - public void afterEach() throws IOException { + public void afterEach() { log.info("Stop all nodes."); clusterNodes.keySet().forEach(IgnitionManager::stop); @@ -230,47 +227,6 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { && (!DATA_NODE_2.equals(nodeName) || grid.contains(DATA_NODE)); // Data nodes are interchangeable. } - /** - * Test factory for testing node startup order. - * - * @return JUnit tests. - * @see #checkNodeStartupSequence() () - */ - @TestFactory - public Stream<DynamicNode> gridStartupTestFactory() { - return GridGenerator.generateStartupSequences( - nodesCfg.keySet(), - nodeFilter() - ).stream() - .map(nodes -> { - ArrayList<DynamicNode> tests = new ArrayList<>(); - - for (int i = 0; i < nodes.size(); i++) { - String nodeName = nodes.get(i); - boolean last = (i == nodes.size() - 1); - boolean first = (i == 0); - - tests.add(createTest( - "Start " + nodeLabels.get(nodeName), - () -> { - assert !first || clusterNodes.isEmpty(); - - startNode(nodeName); - }, - this::checkNodeStartupSequence, - () -> { - if (last) { - stopNodes(nodes); - } - } - )); - } - - return DynamicContainer.dynamicContainer("Start sequence " + - nodes.stream().map(nodeLabels::get).collect(Collectors.joining("-")), tests); - }); - } - /** * Test factory for testing single node restart. * @@ -291,38 +247,32 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { ArrayList<DynamicNode> tests = new ArrayList<>(); for (int i = 0; i < nodes.size(); i++) { String nodeName = nodes.get(i); + boolean last = (i == nodes.size() - 1); boolean first = i == 0; + Runnable setup = first ? () -> { + nodes.forEach(this::startNode); + stopNode(nodeName); + } : () -> stopNode(nodeName); + tests.add(createTest( - "Stop " + nodeLabels.get(nodeName), - () -> { - if (first) { - assert clusterNodes.isEmpty(); - - startNodes(nodes); - } - stopNode(nodeName); - }, - this::checkNodeRestart, - () -> { - } + "Stopped " + nodeLabels.get(nodeName), + setup, + NOOP, + this::checkNodeRestart )); tests.add(createTest( - "Start " + nodeLabels.get(nodeName), + "Started " + nodeLabels.get(nodeName), () -> startNode(nodeName), - this::checkNodeRestart, - () -> { - if (last) { - stopNodes(nodes); - } - } + last ? this::stopCluster : NOOP, + this::checkNodeRestart )); } - return DynamicContainer.dynamicContainer("Grid [" + - nodes.stream().map(nodeLabels::get).collect(Collectors.joining(", ")) + ']', tests); + return DynamicContainer.dynamicContainer("Grid " + + clusterNodesToString(nodes), tests); }) .filter(Objects::nonNull); } @@ -332,9 +282,10 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { validateNodeJoin(); - Assumptions.assumeTrue(clusterNodes.containsKey(CMG_NODE), "CMG must start first"); + if (!clusterNodes.containsKey(METASTORAGE_NODE)) { + return; // There is no node startup future finished, as nodes wait for Metastorage on start. + } - validateDistributionZone(); validateDDL(); validateROTransaction(); validateRWTransaction(); @@ -344,7 +295,6 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { log.info("Node restart test: cluster=[" + String.join(", ", new TreeSet<>(clusterNodes.keySet())) + ']'); validateNodeJoin(); - validateDistributionZone(); validateDDL(); validateROTransaction(); validateRWTransaction(); @@ -355,25 +305,32 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { CompletableFuture<Ignite> fut = IgnitionManager.start(NEW_NODE, NEW_NODE_CONFIG, WORK_DIR.resolve(NEW_NODE)); if (!clusterNodes.containsKey(CMG_NODE)) { - // Only CMG promotes a new node to logical topology assertThrowsWithCause(() -> fut.get(NODE_JOIN_WAIT_TIMEOUT, TimeUnit.SECONDS), TimeoutException.class); - assertTrue(validateTopologyHasNode("physical", NEW_NODE)); - assertThrowsWithCause(() -> validateTopologyHasNode("logical", NEW_NODE), IgniteException.class); + assertTrue(validateNodeEnterTopology("physical", NEW_NODE)); + assertThrowsWithCause(() -> validateNodeEnterTopology("logical", NEW_NODE), IgniteException.class); + + return; + } else if (!clusterNodes.containsKey(METASTORAGE_NODE)) { + // Node future can't complete as some components requires Metastorage on start. + assertThrowsWithCause(() -> fut.get(NODE_JOIN_WAIT_TIMEOUT, TimeUnit.SECONDS), TimeoutException.class); + + assertTrue(validateNodeEnterTopology("physical", NEW_NODE)); + assertTrue(validateNodeEnterTopology("logical", NEW_NODE)); return; } assertThat(fut, willCompleteSuccessfully()); - assertTrue(validateTopologyHasNode("physical", ((IgniteImpl) fut.join()).id())); - assertTrue(validateTopologyHasNode("logical", ((IgniteImpl) fut.join()).id())); + assertTrue(validateNodeEnterTopology("physical", ((IgniteImpl) fut.join()).id())); + assertTrue(validateNodeEnterTopology("logical", ((IgniteImpl) fut.join()).id())); } finally { IgnitionManager.stop(NEW_NODE); } } - private static boolean validateTopologyHasNode(String topologyType, String nodeId) { + private static boolean validateNodeEnterTopology(String topologyType, String nodeId) { StringWriter out = new StringWriter(); StringWriter err = new StringWriter(); @@ -388,33 +345,12 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { return Pattern.compile("\\b" + nodeId + "\\b").matcher(out.toString()).find(); } - private void validateDistributionZone() { - if (!clusterNodes.containsKey(METASTORAGE_NODE)) { - //TODO: creating distribution zone fails. - return; - } - if (!clusterNodes.containsKey(DATA_NODE) && !clusterNodes.containsKey(DATA_NODE_2)) { - //TODO: creating distribution zone fails. - return; - } - - try { - if (!clusterNodes.containsKey(DATA_NODE_2)) { - //TODO: creating distribution zone that spans on the DATA_NODE only will success. - //TODO: creating distribution zone that spans DATA_NODE and DATA_NODE2 will fail. - return; - } - //TODO: creating distribution zones will success. - } finally { - //TODO: drop distribution zones. - } - } - private void validateDDL() { - Ignite node = getNode(); String createTableCommand = "CREATE TABLE tempTbl (id INT PRIMARY KEY, val INT) WITH partitions = 1"; String dropTableCommand = "DROP TABLE IF EXISTS tempTbl"; + Ignite node = getNode(); + try { if (!clusterNodes.containsKey(METASTORAGE_NODE)) { assertThrowsWithCause(() -> sql(node, null, createTableCommand), IgniteException.class); @@ -434,8 +370,9 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { try { if (!clusterNodes.containsKey(DATA_NODE) && !clusterNodes.containsKey(DATA_NODE_2)) { assertThrowsWithCause(() -> sql(node, roTx, "SELECT * FROM tbl1"), IgniteException.class); + return; - } else if (clusterNodes.containsKey(DATA_NODE)) { + } else if (!clusterNodes.containsKey(DATA_NODE_2)) { // Use fake transaction with a timestamp from the past. Transaction tx0 = Mockito.spy(roTx); Mockito.when(tx0.readTimestamp()).thenReturn(new HybridTimestamp(1L, 0)); @@ -502,7 +439,19 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { /** Get cluster node. */ private Ignite getNode() { - return clusterNodes.values().iterator().next(); + try { + if (clusterNodes.containsKey(DATA_NODE)) { + clusterNodes.get(DATA_NODE).get(); + } else if (clusterNodes.containsKey(DATA_NODE_2)) { + clusterNodes.get(DATA_NODE_2).get(); + } + + return clusterNodes.values().iterator().next().get(); + } catch (Throwable th) { + IgniteTestUtils.sneakyThrow(th); + + return null; + } } /** @@ -510,56 +459,52 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { * * @param testName Test name. * @param setUpRunnable SetUp action. - * @param testRunnable Test action. * @param tearDownRunnable TearDown action. + * @param testRunnable Test action. * @return JUnit test node. */ - private DynamicTest createTest(String testName, Runnable setUpRunnable, Runnable testRunnable, Runnable tearDownRunnable) { + private DynamicTest createTest(String testName, Runnable setUpRunnable, Runnable tearDownRunnable, Runnable testRunnable) { return DynamicTest.dynamicTest(testName, () -> { TestInfoImpl info = new TestInfoImpl(testName); try { - setUpRunnable.run(); setupBase(info, WORK_DIR); + setUpRunnable.run(); testRunnable.run(); - } finally { - tearDownBase(info); tearDownRunnable.run(); + tearDownBase(info); } }); } + private static String clusterNodesToString(List<String> nodes) { + return '[' + nodes.stream().map(nodeLabels::get).collect(Collectors.joining(", ")) + ']'; + } + private void startNode(String nodeName) { - CompletableFuture<Ignite> node = IgnitionManager.start(nodeName, nodesCfg.get(nodeName), WORK_DIR.resolve(nodeName)); + CompletableFuture<Ignite> fut = IgnitionManager.start(nodeName, nodesCfg.get(nodeName), WORK_DIR.resolve(nodeName)); - clusterNodes.put(nodeName, node.join()); + clusterNodes.put(nodeName, fut); } private void stopNode(String nodeName) { - Ignite rmv = clusterNodes.remove(nodeName); + Future<?> rmv = clusterNodes.remove(nodeName); assert rmv != null; IgnitionManager.stop(nodeName); } - private void stopNodes(List<String> nodes) { - for (int i = nodes.size() - 1; i >= 0; i--) { - stopNode(nodes.get(i)); - } - } - - private void startNodes(List<String> nodes) { - for (String node : nodes) { - startNode(node); - } + private void stopCluster() { + clusterNodes.keySet().forEach(IgnitionManager::stop); + clusterNodes.clear(); } - protected List<List<Object>> sql(Ignite node, @Nullable Transaction tx, String sql, Object... args) { + protected static List<List<Object>> sql(Ignite node, @Nullable Transaction tx, String sql, Object... args) { var queryEngine = ((IgniteImpl) node).queryEngine(); - SessionId sessionId = queryEngine.createSession(15_000, PropertiesHolder.fromMap( + SessionId sessionId = queryEngine.createSession(5_000, PropertiesHolder.fromMap( Map.of(QueryProperty.DEFAULT_SCHEMA, "PUBLIC") )); @@ -660,7 +605,7 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { @Override public Optional<Class<?>> getTestClass() { - return Optional.of(ItNodeStartStopTest.class); + return Optional.of(ItNodeRestartTest.class); } @Override diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/ItNodeStartStopTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/cluster/ItNodeStartStopTest.java similarity index 71% rename from modules/runner/src/integrationTest/java/org/apache/ignite/internal/ItNodeStartStopTest.java rename to modules/runner/src/integrationTest/java/org/apache/ignite/internal/cluster/ItNodeStartStopTest.java index 377059b8e2..425baa80ff 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/ItNodeStartStopTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/cluster/ItNodeStartStopTest.java @@ -15,13 +15,14 @@ * limitations under the License. */ -package org.apache.ignite.internal; +package org.apache.ignite.internal.cluster; import static org.apache.ignite.internal.sql.engine.util.CursorUtils.getAllFromCursor; import static org.apache.ignite.internal.testframework.IgniteTestUtils.assertThrowsWithCause; import static org.apache.ignite.internal.testframework.IgniteTestUtils.await; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import io.micronaut.configuration.picocli.MicronautFactory; @@ -40,11 +41,10 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.BiPredicate; @@ -64,6 +64,7 @@ import org.apache.ignite.internal.sql.engine.QueryProperty; import org.apache.ignite.internal.sql.engine.property.PropertiesHolder; import org.apache.ignite.internal.sql.engine.session.SessionId; import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest; +import org.apache.ignite.internal.testframework.IgniteTestUtils; import org.apache.ignite.internal.testframework.WithSystemProperty; import org.apache.ignite.internal.testframework.WorkDirectory; import org.apache.ignite.internal.testframework.WorkDirectoryExtension; @@ -73,6 +74,7 @@ import org.apache.ignite.tx.Transaction; import org.apache.ignite.tx.TransactionException; import org.hamcrest.Matchers; import org.hamcrest.text.IsEmptyString; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assumptions; @@ -91,10 +93,14 @@ import picocli.CommandLine; */ //TODO: Fix expected messages in assertThrows // TODO: Create 2 distribution zones, which spans a single node and both data nodes, and a tables in these zones. +@SuppressWarnings("ThrowableNotThrown") @ExtendWith(WorkDirectoryExtension.class) @WithSystemProperty(key = "org.jline.terminal.dumb", value = "true") public class ItNodeStartStopTest extends BaseIgniteAbstractTest { - public static final int NODE_JOIN_WAIT_TIMEOUT = 5; + public static final int NODE_JOIN_WAIT_TIMEOUT = 500; + public static final Runnable NOOP = () -> { + }; + /** Work directory. */ @WorkDirectory private static Path WORK_DIR; @@ -173,11 +179,17 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { ); /** Cluster nodes. */ - private final Map<String, Ignite> clusterNodes = new HashMap<>(); + private final Map<String, CompletableFuture<Ignite>> clusterNodes = new HashMap<>(); /** Runs after each test sequence. */ @BeforeEach - public void beforeEach() throws Exception { + public void before() throws Exception { + assert clusterNodes.isEmpty(); + + for (String name : nodesCfg.keySet()) { + IgniteUtils.deleteIfExists(WORK_DIR.resolve(name)); + } + List<CompletableFuture<Ignite>> futures = new ArrayList<>(); // Start nodes. @@ -185,7 +197,7 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { // Init cluster. //TODO: Use dedicated node for metastorage. - IgnitionManager.init(CMG_NODE, List.of(CMG_NODE /* METASTORAGE_NODE */), List.of(CMG_NODE), "cluster"); + IgnitionManager.init(CMG_NODE, List.of(METASTORAGE_NODE), List.of(CMG_NODE), "cluster"); for (CompletableFuture<Ignite> future : futures) { assertThat(future, willCompleteSuccessfully()); @@ -234,7 +246,6 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { * Test factory for testing node startup order. * * @return JUnit tests. - * @see #checkNodeStartupSequence() () */ @TestFactory public Stream<DynamicNode> gridStartupTestFactory() { @@ -243,137 +254,74 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { nodeFilter() ).stream() .map(nodes -> { - ArrayList<DynamicNode> tests = new ArrayList<>(); + List<DynamicNode> scenarioSteps = new ArrayList<>(); + List<String> startedNodes = new ArrayList<>(); for (int i = 0; i < nodes.size(); i++) { String nodeName = nodes.get(i); - boolean last = (i == nodes.size() - 1); - boolean first = (i == 0); + startedNodes.add(nodeName); - tests.add(createTest( - "Start " + nodeLabels.get(nodeName), - () -> { - assert !first || clusterNodes.isEmpty(); - - startNode(nodeName); - }, - this::checkNodeStartupSequence, - () -> { - if (last) { - stopNodes(nodes); - } - } - )); - } + boolean first = (i == 0); + boolean last = (i >= (nodes.size() - 1)); - return DynamicContainer.dynamicContainer("Start sequence " + - nodes.stream().map(nodeLabels::get).collect(Collectors.joining("-")), tests); - }); - } + Runnable setup = () -> { + assert !first || clusterNodes.isEmpty(); - /** - * Test factory for testing single node restart. - * - * @return JUnit tests. - * @see #checkNodeRestart() () - */ - @TestFactory - public Stream<? extends DynamicNode> nodeRestartTestFactory() { - return GridGenerator.generateGrids( - nodesCfg.keySet(), - nodeFilter() // Data nodes are interchangeable. - ).stream() - .map(nodes -> { - if (nodes.size() != 3) { - return null; //TODO: remove this - } + startNode(nodeName); + }; - ArrayList<DynamicNode> tests = new ArrayList<>(); - for (int i = 0; i < nodes.size(); i++) { - String nodeName = nodes.get(i); - boolean last = (i == nodes.size() - 1); - boolean first = i == 0; - - tests.add(createTest( - "Stop " + nodeLabels.get(nodeName), - () -> { - if (first) { - assert clusterNodes.isEmpty(); - - startNodes(nodes); - } - stopNode(nodeName); - }, - this::checkNodeRestart, - () -> { - } - )); + Runnable tearDown = last ? () -> stopCluster(nodes) : NOOP; - tests.add(createTest( - "Start " + nodeLabels.get(nodeName), - () -> startNode(nodeName), - this::checkNodeRestart, - () -> { - if (last) { - stopNodes(nodes); - } - } + scenarioSteps.add(DynamicContainer.dynamicContainer( + "Started " + clusterNodesToString(startedNodes), + List.of( + createDynamicTest("checkNewNodeJoin", setup, this::checkNewNodeJoin, NOOP), + createDynamicTest("checkDDL", NOOP, this::checkDDL, NOOP), + createDynamicTest("checkROTransaction", NOOP, this::checkROTransacton, NOOP), +// createTest("validateRWTransaction", NOOP, NOOP, this::validateRWTransaction), + createDynamicTest("checkImplicitRWTransaction", NOOP, this::checkImplicitRWTransaction, tearDown) + ) )); } - return DynamicContainer.dynamicContainer("Grid [" + - nodes.stream().map(nodeLabels::get).collect(Collectors.joining(", ")) + ']', tests); - }) - .filter(Objects::nonNull); - } - - public void checkNodeStartupSequence() { - log.info("Node startup sequence test: cluster=[" + String.join(", ", new TreeSet<>(clusterNodes.keySet())) + ']'); - - validateNodeJoin(); - - Assumptions.assumeTrue(clusterNodes.containsKey(CMG_NODE), "CMG must start first"); - - validateDistributionZone(); - validateDDL(); - validateROTransaction(); - validateRWTransaction(); - } - - public void checkNodeRestart() { - log.info("Node restart test: cluster=[" + String.join(", ", new TreeSet<>(clusterNodes.keySet())) + ']'); - - validateNodeJoin(); - validateDistributionZone(); - validateDDL(); - validateROTransaction(); - validateRWTransaction(); + return DynamicContainer.dynamicContainer("Start sequence " + clusterNodesToString(nodes), scenarioSteps); + }); } - private void validateNodeJoin() { + private void checkNewNodeJoin() { try { CompletableFuture<Ignite> fut = IgnitionManager.start(NEW_NODE, NEW_NODE_CONFIG, WORK_DIR.resolve(NEW_NODE)); if (!clusterNodes.containsKey(CMG_NODE)) { - // Only CMG promotes a new node to logical topology - assertThrowsWithCause(() -> fut.get(NODE_JOIN_WAIT_TIMEOUT, TimeUnit.SECONDS), TimeoutException.class); + assertThrowsWithCause(() -> fut.get(NODE_JOIN_WAIT_TIMEOUT, TimeUnit.MILLISECONDS), TimeoutException.class); - assertTrue(validateTopologyHasNode("physical", NEW_NODE)); - assertThrowsWithCause(() -> validateTopologyHasNode("logical", NEW_NODE), IgniteException.class); + assertTrue(validateNodeEnterTopology("physical", NEW_NODE)); + + // CMG holds logical topology state. + assertThrowsWithCause(() -> validateNodeEnterTopology("logical", NEW_NODE), IgniteException.class); + + return; + } else if (!clusterNodes.containsKey(METASTORAGE_NODE)) { + // Node future can't complete as some components requires Metastorage on start. + assertThrowsWithCause(() -> fut.get(NODE_JOIN_WAIT_TIMEOUT, TimeUnit.MILLISECONDS), TimeoutException.class); + + assertTrue(validateNodeEnterTopology("physical", NEW_NODE)); + assertFalse( + validateNodeEnterTopology("logical", NEW_NODE)); //TODO: Is Metastore required to promote node to logical topology? return; } assertThat(fut, willCompleteSuccessfully()); - assertTrue(validateTopologyHasNode("physical", ((IgniteImpl) fut.join()).id())); - assertTrue(validateTopologyHasNode("logical", ((IgniteImpl) fut.join()).id())); + assertTrue(validateNodeEnterTopology("physical", ((IgniteImpl) fut.join()).id())); + assertTrue(validateNodeEnterTopology("logical", ((IgniteImpl) fut.join()).id())); } finally { IgnitionManager.stop(NEW_NODE); } } - private static boolean validateTopologyHasNode(String topologyType, String nodeId) { + private static boolean validateNodeEnterTopology(String topologyType, String nodeId) { StringWriter out = new StringWriter(); StringWriter err = new StringWriter(); @@ -388,86 +336,72 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { return Pattern.compile("\\b" + nodeId + "\\b").matcher(out.toString()).find(); } - private void validateDistributionZone() { - if (!clusterNodes.containsKey(METASTORAGE_NODE)) { - //TODO: creating distribution zone fails. - return; - } - if (!clusterNodes.containsKey(DATA_NODE) && !clusterNodes.containsKey(DATA_NODE_2)) { - //TODO: creating distribution zone fails. - return; - } - - try { - if (!clusterNodes.containsKey(DATA_NODE_2)) { - //TODO: creating distribution zone that spans on the DATA_NODE only will success. - //TODO: creating distribution zone that spans DATA_NODE and DATA_NODE2 will fail. - return; - } - //TODO: creating distribution zones will success. - } finally { - //TODO: drop distribution zones. - } - } - - private void validateDDL() { - Ignite node = getNode(); + public void checkDDL() { String createTableCommand = "CREATE TABLE tempTbl (id INT PRIMARY KEY, val INT) WITH partitions = 1"; String dropTableCommand = "DROP TABLE IF EXISTS tempTbl"; - try { - if (!clusterNodes.containsKey(METASTORAGE_NODE)) { - assertThrowsWithCause(() -> sql(node, null, createTableCommand), IgniteException.class); - return; - } + Ignite node = getNode(); + try { sql(node, null, createTableCommand); } finally { sql(node, null, dropTableCommand); } } - public void validateROTransaction() { + public void checkROTransacton() { Ignite node = getNode(); Transaction roTx = node.transactions().readOnly().begin(); try { if (!clusterNodes.containsKey(DATA_NODE) && !clusterNodes.containsKey(DATA_NODE_2)) { assertThrowsWithCause(() -> sql(node, roTx, "SELECT * FROM tbl1"), IgniteException.class); + return; - } else if (clusterNodes.containsKey(DATA_NODE)) { + } else if (!clusterNodes.containsKey(DATA_NODE_2)) { // Use fake transaction with a timestamp from the past. Transaction tx0 = Mockito.spy(roTx); Mockito.when(tx0.readTimestamp()).thenReturn(new HybridTimestamp(1L, 0)); - assertThrowsWithCause(() -> sql(node, tx0, "SELECT * FROM tbl1"), IgniteException.class); +// assertThrowsWithCause(() -> sql(node, tx0, "SELECT * FROM tbl1"), IgniteException.class); + + sql(node, roTx, "SELECT * FROM tbl1"); return; } - assertThrowsWithCause(() -> sql(node, roTx, "SELECT * FROM tbl1"), IgniteException.class); + sql(node, roTx, "SELECT * FROM tbl1"); } finally { roTx.rollback(); } } - public void validateImplicitRWTransaction() { + public void checkImplicitRWTransaction() { Ignite node = getNode(); - if (!clusterNodes.containsKey(DATA_NODE) || !clusterNodes.containsKey(DATA_NODE_2)) { - assertThrowsWithCause( - () -> sql(node, null, "INSERT INTO tbl1 VALUES (5, 5)"), - TransactionException.class, - "Failed to get the primary replica"); + // TODO: Bound table distribution zone to data nodes and uncomment. + // if (!clusterNodes.containsKey(DATA_NODE) || !clusterNodes.containsKey(DATA_NODE_2)) { + if (clusterNodes.size() <= 2 || !clusterNodes.containsKey(DATA_NODE)) { + + try { + assertThrowsWithCause( + () -> sql(node, null, "INSERT INTO tbl1 VALUES (2, -2)"), + TransactionException.class, + "Failed to get the primary replica"); + } finally { + sql(node, null, "DELETE FROM tbl1 WHERE tbl1.id = 2"); + } return; } sql(node, null, "INSERT INTO tbl1 VALUES (2, 2)"); - assertThat(sql(node, null, "SELECT * FROM tbl1").size(), Matchers.equalTo(2)); - - sql(node, null, "DELETE FROM tbl1 WHERE tbl1.id = 2"); + try { + assertThat(sql(node, null, "SELECT * FROM tbl1").size(), Matchers.equalTo(2)); + } finally { + sql(node, null, "DELETE FROM tbl1 WHERE tbl1.id = 2"); + } } public void validateRWTransaction() { @@ -477,14 +411,15 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { Transaction tx = node.transactions().readOnly().begin(); try { assertThrowsWithCause( - () -> sql(node, tx, "INSERT INTO tbl1 VALUES (5, 5)"), + () -> sql(node, tx, "INSERT INTO tbl1 VALUES (2, -2)"), TransactionException.class, "Failed to get the primary replica"); - - return; } finally { tx.rollback(); + sql(node, null, "DELETE FROM tbl1 WHERE tbl1.id = 2"); } + + return; } Transaction tx = node.transactions().readOnly().begin(); @@ -501,8 +436,31 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { } /** Get cluster node. */ - private Ignite getNode() { - return clusterNodes.values().iterator().next(); + private @NotNull Ignite getNode() { + try { + CompletableFuture<Ignite> nodeFut; +// if (clusterNodes.containsKey(DATA_NODE)) { +// nodeFut = clusterNodes.get(DATA_NODE); +// } else if (clusterNodes.containsKey(DATA_NODE_2)) { +// nodeFut = clusterNodes.get(DATA_NODE_2); +// } else { +// } + nodeFut = clusterNodes.values().iterator().next(); + + if (clusterNodes.containsKey(METASTORAGE_NODE)) + return nodeFut.join(); + + return nodeFut.get(NODE_JOIN_WAIT_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (TimeoutException ex) { + Assumptions.assumeTrue(clusterNodes.containsKey(METASTORAGE_NODE), + "No Ignite object instances available, because node initialization requires Metastorage."); + + IgniteTestUtils.sneakyThrow(ex); + } catch (Throwable th) { + IgniteTestUtils.sneakyThrow(th); + } + + return null; } /** @@ -514,52 +472,53 @@ public class ItNodeStartStopTest extends BaseIgniteAbstractTest { * @param tearDownRunnable TearDown action. * @return JUnit test node. */ - private DynamicTest createTest(String testName, Runnable setUpRunnable, Runnable testRunnable, Runnable tearDownRunnable) { + private DynamicTest createDynamicTest(String testName, Runnable setUpRunnable, Runnable testRunnable, Runnable tearDownRunnable) { return DynamicTest.dynamicTest(testName, () -> { TestInfoImpl info = new TestInfoImpl(testName); try { - setUpRunnable.run(); setupBase(info, WORK_DIR); + setUpRunnable.run(); testRunnable.run(); - } finally { - tearDownBase(info); tearDownRunnable.run(); + tearDownBase(info); } }); } + private static String clusterNodesToString(Collection<String> nodes) { + return '[' + nodes.stream().map(nodeLabels::get).collect(Collectors.joining(", ")) + ']'; + } + private void startNode(String nodeName) { - CompletableFuture<Ignite> node = IgnitionManager.start(nodeName, nodesCfg.get(nodeName), WORK_DIR.resolve(nodeName)); + String nodeConfig = nodesCfg.get(nodeName); + + CompletableFuture<Ignite> node = IgnitionManager.start(nodeName, nodeConfig, WORK_DIR.resolve(nodeName)); - clusterNodes.put(nodeName, node.join()); + clusterNodes.put(nodeName, node); } private void stopNode(String nodeName) { - Ignite rmv = clusterNodes.remove(nodeName); + Future<?> rmv = clusterNodes.remove(nodeName); assert rmv != null; IgnitionManager.stop(nodeName); } - private void stopNodes(List<String> nodes) { + private void stopCluster(List<String> nodes) { for (int i = nodes.size() - 1; i >= 0; i--) { stopNode(nodes.get(i)); } - } - private void startNodes(List<String> nodes) { - for (String node : nodes) { - startNode(node); - } + assert clusterNodes.isEmpty(); } protected List<List<Object>> sql(Ignite node, @Nullable Transaction tx, String sql, Object... args) { var queryEngine = ((IgniteImpl) node).queryEngine(); - SessionId sessionId = queryEngine.createSession(15_000, PropertiesHolder.fromMap( + SessionId sessionId = queryEngine.createSession(5_000, PropertiesHolder.fromMap( Map.of(QueryProperty.DEFAULT_SCHEMA, "PUBLIC") ));
