This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a commit to branch ignite-18171 in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit 2d94efe18753c0d8b64926eec09c605fdd2666f8 Author: amashenkov <[email protected]> AuthorDate: Thu Nov 24 17:04:02 2022 +0300 Wip. --- .../ignite/internal/ItNodeStartStopTest.java | 486 +++++++++++++++------ 1 file changed, 347 insertions(+), 139 deletions(-) diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/ItNodeStartStopTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/ItNodeStartStopTest.java index 9bfdd00d12..415c6bce1f 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/ItNodeStartStopTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/ItNodeStartStopTest.java @@ -17,238 +17,446 @@ package org.apache.ignite.internal; -import static org.junit.jupiter.api.Assertions.fail; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; +import static org.hamcrest.MatcherAssert.assertThat; import java.lang.reflect.Method; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; import java.util.function.BiPredicate; -import java.util.function.Consumer; import java.util.function.Predicate; -import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgnitionManager; import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest; import org.apache.ignite.internal.testframework.WorkDirectory; import org.apache.ignite.internal.testframework.WorkDirectoryExtension; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.function.Executable; +/** + * Test node start/stop in different scenarios and validate grid components behavior depending on availability/absence of quorums. + */ @ExtendWith(WorkDirectoryExtension.class) public class ItNodeStartStopTest extends BaseIgniteAbstractTest { /** Work directory. */ @WorkDirectory private static Path WORK_DIR; + private static final String connectionAddr = "\"localhost:3344\", \"localhost:3345\", \"localhost:3346\""; + + /** Cluster management group node name. */ + private static final String CMG_NODE = "C"; + /** MetaStorage group node name. */ + private static final String METASTORAGE_NODE = "M"; + /** Data node 1. */ + private static final String DATA_NODE = "D"; + /** Data node 2. */ + private static final String DATA_NODE_2 = "D2"; + /** Nodes configurations. */ - private final Map<String, String> nodesCfg = Map.of( - "C", "", - "M", "", - "D", "", - "D2", "" - ); // Node name -> config. + private static final Map<String, String> nodesCfg = Map.of( + CMG_NODE, "{\n" + + " \"network\": {\n" + + " \"port\":3344,\n" + + " \"nodeFinder\":{\n" + + " \"netClusterNodes\": [ " + connectionAddr + " ]\n" + + " }\n" + + " }\n" + + "}", + METASTORAGE_NODE, "{\n" + + " \"network\": {\n" + + " \"port\":3345,\n" + + " \"nodeFinder\":{\n" + + " \"netClusterNodes\": [ " + connectionAddr + " ]\n" + + " }\n" + + " }\n" + + "}", + DATA_NODE, "{\n" + + " \"network\": {\n" + + " \"port\":3346,\n" + + " \"nodeFinder\":{\n" + + " \"netClusterNodes\": [ " + connectionAddr + " ]\n" + + " }\n" + + " }\n" + + "}", + DATA_NODE_2, "{\n" + + " \"network\": {\n" + + " \"port\":3347,\n" + + " \"nodeFinder\":{\n" + + " \"netClusterNodes\": [ " + connectionAddr + " ]\n" + + " }\n" + + " }\n" + + "}" + ); /** Cluster nodes. */ - private final List<String> clusterNodes = new ArrayList<>(); // TODO: replace with Map<String, Ignite> (nodeName->node) ? + private final Map<String, Ignite> clusterNodes = new HashMap<>(); + + + /** Runs after each test sequence. */ + @BeforeEach + public void beforeAll() { + log.info("Init cluster."); + + List<CompletableFuture<Ignite>> futures = new ArrayList<>(); + + nodesCfg.forEach((k, v) -> futures.add(IgnitionManager.start(k, v, WORK_DIR.resolve(k)))); + + //TODO: Fix metastore group + IgnitionManager.init(CMG_NODE, List.of(CMG_NODE /* METASTORAGE_NODE */), List.of(CMG_NODE), "cluster"); + + // TODO: Create distribution zones: spans both nodes, spans a single node. + // TODO: Create tables in these distribution zone + add data. + + for (CompletableFuture<Ignite> future : futures) { + assertThat(future, willCompleteSuccessfully()); + } + + for (int i = futures.size() - 1; i >= 0; i--) { + IgnitionManager.stop(futures.get(i).join().name()); + } + } - /** Runs after each sequence. */ + /** Runs after each test sequence. */ @AfterEach public void after() { -// clusterNodes.forEach(node -> IgnitionManager.stop(node.name())); + log.info("Stop all nodes."); + + clusterNodes.keySet().forEach(IgnitionManager::stop); clusterNodes.clear(); } + /** Filter out duplicates and invalid grids. */ + private static BiPredicate<String, Set<String>> nodeFilter() { + return (nodeName, grid) -> (!grid.isEmpty() || CMG_NODE.equals(nodeName)) // CMG node always starts first. + && (!DATA_NODE_2.equals(nodeName) || grid.contains(DATA_NODE)); // Data nodes are interchangeable. + } + + /** + * Test factory for testing node startup order. + * + * @return JUnit tests. + * @see #checkNodeStartupSequence() () + */ + @TestFactory + public Stream<DynamicNode> gridStartupTestFactory() { + return GridGenerator.generateStartupSequences( + nodesCfg.keySet(), + nodeFilter() + ).stream() + .map(nodes -> { + ArrayList<DynamicNode> tests = new ArrayList<>(); + + for (int i = 0; i < nodes.size(); i++) { + String nodeName = nodes.get(i); + boolean last = (i == nodes.size() - 1); + + tests.add(createTest( + "Start " + nodeName, + () -> startNode(nodeName), + this::checkNodeStartupSequence, + () -> { + if (last) { + stopNodes(nodes); + } + } + )); + } + + return DynamicContainer.dynamicContainer("Start sequence " + String.join("-", nodes), tests); + }); + } + /** - * Test factory. + * Test factory for testing single node restart. * - * @return Tests. + * @return JUnit tests. + * @see #checkNodeRestart() () */ @TestFactory - public Stream<DynamicTest> factory() { - return new NodesStartStopGenerator( - this::startNode, - this::stopNode, - testExecutor(this::testStartStopNodes), // Wrap test method with logging executor. - nodesCfg.keySet(), - ItNodeStartStopTest::isValidNode - ).build(); + public Stream<DynamicNode> nodeRestartTestFactory() { + return GridGenerator.generateGrids( + nodesCfg.keySet(), + nodeFilter() // Data nodes are interchangeable. + ).stream() + .map(nodes -> { + ArrayList<DynamicNode> tests = new ArrayList<>(); + for (int i = 0; i < nodes.size(); i++) { + String nodeName = nodes.get(i); + boolean last = (i == nodes.size() - 1); + boolean first = i == 0; + + tests.add(createTest( + "Stop " + nodeName, + () -> { + if (first) { + startNodes(nodes); + } + stopNode(nodeName); + }, + this::checkNodeRestart, + () -> { + } + )); + + tests.add(createTest( + "Start " + nodeName, + () -> startNode(nodeName), + this::checkNodeRestart, + () -> { + if (last) { + stopNodes(nodes); + } + } + )); + } + + return DynamicContainer.dynamicContainer("Grid [" + String.join(", ", nodes) + ']', tests); + }); } - private static boolean isValidNode(String nodeName, Set<String> grid) { - return (!grid.isEmpty() || "C".equals(nodeName)) // CMG node always starts first. - && (!"D2".equals(nodeName) || grid.contains("D")); // Data nodes are interchangeable. + public void checkNodeStartupSequence() { + log.info("Node startup sequence test: cluster=[" + String.join(", ", new TreeSet<>(clusterNodes.keySet())) + ']'); + + validateNodeJoin(); + + Assumptions.assumeTrue(clusterNodes.containsKey(CMG_NODE), "CMG must start first"); + + validateDistributionZone(); + validateDDL(); + validateROTransaction(); + validateRWTransaction(); } - private void testStartStopNodes() { - System.out.println("Grid state: " + String.join("", String.join(", ", clusterNodes))); + public void checkNodeRestart() { + log.info("Node restart test: cluster=[" + String.join(", ", new TreeSet<>(clusterNodes.keySet())) + ']'); + + validateNodeJoin(); + validateDistributionZone(); + validateDDL(); + validateROTransaction(); + validateRWTransaction(); } - private void startNode(String nodeName) { - System.out.println("Starting node: " + nodeName); -// CompletableFuture<Ignite> node = IgnitionManager.start(nodeName, nodesCfg.get(nodeName), WORK_DIR.resolve(nodeName)); -// -// clusterNodes.add(node.join()); - clusterNodes.add(nodeName); + private void validateNodeJoin() { + if (!clusterNodes.containsKey(CMG_NODE)) { + //TODO: add node and check it can't join. + return; + } + + if (!clusterNodes.containsKey(METASTORAGE_NODE)) { + //TODO: add node and check it joins, but logical topology is unchanged. + } else { + //TODO: add node and check it joins and updates logical topology. + } } - private void stopNode(String nodeName) { - System.out.println("Stopping node: " + nodeName); -// Node node = clusterNodes.stream().filter(n -> nodeName.equals(n.name())).findFirst().orElseThrow(); -// IgnitionManager.stop(nodeName); + private void validateDistributionZone() { + if (!clusterNodes.containsKey(METASTORAGE_NODE)) { + //TODO: creating distribution zone fails. + return; + } + + if (!clusterNodes.containsKey(DATA_NODE) && !clusterNodes.containsKey(DATA_NODE_2)) { + //TODO: creating distribution zone fails. + return; + } - clusterNodes.remove(nodeName); + try { + if (clusterNodes.containsKey(DATA_NODE) && !clusterNodes.containsKey(DATA_NODE_2)) { + //TODO: creating distribution zone that spans unavailable node will fails. + //TODO: creating distribution zone that spans on the DATA_NODE only will success. + return; + } + + //TODO: creating distribution zone that spans unavailable node will fails. + } finally { + //TODO: drop distribution zone. + } + } + + private void validateDDL() { + if (!clusterNodes.containsKey(METASTORAGE_NODE)) { + //TODO: create table and check it fails. + return; + } + + if (!clusterNodes.containsKey(DATA_NODE) && !clusterNodes.containsKey(DATA_NODE_2)) { + //TODO: create table and check it fails because distribution zone must have quorum. + return; + } + + try { + if (clusterNodes.containsKey(DATA_NODE) && !clusterNodes.containsKey(DATA_NODE_2)) { + //TODO: creating table in distribution zone that spans all data nodes will fails. + //TODO: creating table in distribution zone that spans DATA_NODE only will success. + } + + //TODO: creating table will success. + } finally { + //TODO: drop table. + } + } + + public void validateROTransaction() { + if (!clusterNodes.containsKey(DATA_NODE) && !clusterNodes.containsKey(DATA_NODE_2)) { + //TODO: table RO transaction will fails. + return; + } + + //TODO: table RO transaction will success. + } + + public void validateRWTransaction() { + if (clusterNodes.containsKey(DATA_NODE) && clusterNodes.containsKey(DATA_NODE_2)) { + //TODO: table RW transaction will success. + return; + } + + //TODO: table RO transaction will fails. } /** - * Test sequence generator. + * Creates JUnit test node. + * + * @param testName Test name. + * @param setUpRunnable SetUp action. + * @param testRunnable Test action. + * @param tearDownRunnable TearDown action. + * @return JUnit test node. */ - private Consumer<TestInfo> testExecutor(Runnable testRunnable) { - return (info) -> { + private DynamicTest createTest(String testName, Runnable setUpRunnable, Runnable testRunnable, Runnable tearDownRunnable) { + return DynamicTest.dynamicTest(testName, () -> { + TestInfoImpl info = new TestInfoImpl(testName); try { + setUpRunnable.run(); setupBase(info, WORK_DIR); testRunnable.run(); } finally { tearDownBase(info); + tearDownRunnable.run(); } - }; + }); } - /** - * Filter out grids that where already seen before. - */ - static class VisitedFilter implements Predicate<Set<String>> { + private void startNode(String nodeName) { + CompletableFuture<Ignite> node = IgnitionManager.start(nodeName, nodesCfg.get(nodeName), WORK_DIR.resolve(nodeName)); - private final Set<Set<String>> visitedGrids = new HashSet<>(); + clusterNodes.put(nodeName, node.join()); + } - @Override - public boolean test(Set<String> g) { - return visitedGrids.add(new HashSet<>(g)); + private void stopNode(String nodeName) { + Ignite rmv = clusterNodes.remove(nodeName); + + assert rmv != null; + + IgnitionManager.stop(nodeName); + } + + private void stopNodes(List<String> nodes) { + for (int i = nodes.size() - 1; i >= 0; i--) { + stopNode(nodes.get(i)); } } - public static class NodesStartStopGenerator { + private void startNodes(List<String> nodes) { + for (String node : nodes) { + startNode(node); + } + } - private final Consumer<String> nodeStarter; - private final Consumer<String> nodeStopper; - private final Consumer<TestInfo> testMethodBody; + /** + * Grids configurations generator. + */ + static class GridGenerator { + static List<List<String>> generateStartupSequences(Collection<String> nodes, BiPredicate<String, Set<String>> nodeFilter) { + return new GridGenerator(nodeFilter, grid -> grid.size() == nodes.size()).generate(nodes); + } + + static List<List<String>> generateGrids(Collection<String> nodes, BiPredicate<String, Set<String>> nodeFilter) { + Predicate<Set<String>> filter = new UniqueSetFilter<>(); + filter = filter.and(grid -> grid.size() > 1); + + return new GridGenerator(nodeFilter, filter).generate(nodes); + } - private final Set<String> currentGrid = new TreeSet<>(); - private final Predicate<Set<String>> gridFilter = new VisitedFilter(); // Duplicates filter. + private final LinkedHashSet<String> currentGrid = new LinkedHashSet<>(); + private final List<List<String>> gridStartSequences = new ArrayList<>(); private final BiPredicate<String, Set<String>> nodeFilter; + private final Predicate<Set<String>> gridFilter; - private final Set<String> gridNodes; - - /** - * Creates test sequence generator. - * - * @param nodeStarter Function, starts node by node name. - * @param nodeStopper Function, stops node by node name. - * @param testMethodBody Test method body executor. - * @param nodes Node names. - * @param nodeFilter Node filter accepts node name to start and current grid state. - */ - NodesStartStopGenerator( - Consumer<String> nodeStarter, - Consumer<String> nodeStopper, - Consumer<TestInfo> testMethodBody, - Set<String> nodes, - BiPredicate<String, Set<String>> nodeFilter - ) { - this.nodeStarter = nodeStarter; - this.nodeStopper = nodeStopper; - this.testMethodBody = testMethodBody; + private GridGenerator(BiPredicate<String, Set<String>> nodeFilter, Predicate<Set<String>> gridFilter) { this.nodeFilter = nodeFilter; - this.gridNodes = nodes; + this.gridFilter = gridFilter; } - /** - * Generates test sequence for all valid grid configurations that can be created with given nodes. - * - * @return Tests sequence. - */ - public Stream<DynamicTest> build() { - List<Executable> actionSequence = new ArrayList<>(); - - generate(actionSequence::add, gridNodes); - - return actionSequence.stream() - .map(action -> { - if (action instanceof TestExecutable) { - TestExecutable namedAction = (TestExecutable) action; - - return DynamicTest.dynamicTest(namedAction.getDisplayName(), namedAction); - } else { - try { // Execute instantly and proceed with the next. - action.execute(); - } catch (Throwable e) { - fail(e); - } - - return null; - } - }) - .filter(Objects::nonNull); + /** Generates tests execution sequence recursively. */ + List<List<String>> generate(Collection<String> nodes) { + generate0(nodes); + + return gridStartSequences; } /** Generates tests execution sequence recursively. */ - private void generate(Consumer<Executable> actionCollector, Set<String> availableNodes) { - for (String nodeName : availableNodes) { - if (!nodeFilter.test(nodeName, currentGrid)) { + private void generate0(Collection<String> availableNodes) { + if (gridFilter.test(currentGrid)) { + gridStartSequences.add(new ArrayList<>(currentGrid)); // Copy mutable collection. + } + + for (String node : availableNodes) { + if (!nodeFilter.test(node, currentGrid)) { continue; // Skip node from adding to the current grid. } - String gridStateString = currentGrid.stream().map(Objects::toString).collect(Collectors.joining(", ")); - String prevGridState = '[' + gridStateString + "]"; - String nextGridState = currentGrid.isEmpty() ? '[' + nodeName + ']' : '[' + gridStateString + ", " + nodeName + ']'; + currentGrid.add(node); - actionCollector.accept(() -> nodeStarter.accept(nodeName)); - currentGrid.add(nodeName); + HashSet<String> unusedNodes = new HashSet<>(availableNodes); + unusedNodes.remove(node); - actionCollector.accept(new TestExecutable(prevGridState + " -> " + nextGridState, testMethodBody)); + generate(unusedNodes); - if (availableNodes.size() > 1) { - Set<String> otherNodes = new HashSet<>(availableNodes); - otherNodes.remove(nodeName); - if (gridFilter.test(otherNodes)) { // Avoid generating duplicate subsequences. - generate(actionCollector, otherNodes); - } - } - - currentGrid.remove(nodeName); - actionCollector.accept(() -> nodeStopper.accept(nodeName)); - - if (!currentGrid.isEmpty()) { - actionCollector.accept(new TestExecutable(nextGridState + " -> " + prevGridState, testMethodBody)); - } + currentGrid.remove(node); } } + } - static class TestExecutable implements TestInfo, Executable { - private final Consumer<TestInfo> delegate; + /** Filters out non-unique sets. */ + private static class UniqueSetFilter<T> implements Predicate<Set<T>> { + final Set<Set<T>> seenBefore = new HashSet<>(); + @Override + public boolean test(Set<T> s) { + return seenBefore.add(new HashSet<>(s)); // Copy mutable collection. + } + } + + /** Test info implementation for dynamic tests. */ + static class TestInfoImpl implements TestInfo { private final String name; - TestExecutable(String name, Consumer<TestInfo> delegate) { + TestInfoImpl(String name) { this.name = name; - this.delegate = delegate; - } - - @Override - public void execute() throws Throwable { - delegate.accept(this); } @Override
