This is an automated email from the ASF dual-hosted git repository.
sergey-chugunov-1985 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 09339c025b7 IGNITE-29083 Add a JUnit base for multi-DC network split
tests (#13625)
09339c025b7 is described below
commit 09339c025b796ee25308e49688ff8b7fe7211a45
Author: Maksim Davydov <[email protected]>
AuthorDate: Tue Oct 6 10:13:30 2026 +0300
IGNITE-29083 Add a JUnit base for multi-DC network split tests (#13625)
---
.../cache/distributed/dht/MdcDcIsolationTest.java | 165 ++++++
.../dht/MdcTopologySplitAbstractTest.java | 606 +++++++++++++++++++++
.../IgniteTopologyValidatorTestSuite.java | 3 +
3 files changed, 774 insertions(+)
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/MdcDcIsolationTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/MdcDcIsolationTest.java
new file mode 100644
index 00000000000..749a45849e0
--- /dev/null
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/MdcDcIsolationTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.topology.MdcTopologyValidator;
+import org.junit.Test;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+
+/**
+ * Cuts one DC off the cluster: the segment the topology validator lets write
keeps writing, the cut-off DC only reads,
+ * and after the cut-off DC restarts every DC holds the same data. Also splits
three DCs three ways, where no segment
+ * writes.
+ */
+public class MdcDcIsolationTest extends MdcTopologySplitAbstractTest {
+ /** */
+ private static final int KEYS = 10;
+
+ /** DCs of the current test. */
+ private List<String> dcs;
+
+ /** {@inheritDoc} */
+ @Override protected List<String> dataCenters() {
+ return dcs;
+ }
+
+ /** Three DCs with majority validation: DC3 alone is a minority, DC1 and
DC2 keep writing. */
+ @Test
+ public void testIsolatedDcOfThree() throws Exception {
+ checkIsolation(Arrays.asList(DC1, DC2, DC3), majorityValidator(DC1,
DC2, DC3), DC3, DC1);
+ }
+
+ /** Three DCs with majority validation: DC1 alone is a minority too, no DC
is privileged. */
+ @Test
+ public void testIsolatedFirstDcOfThree() throws Exception {
+ checkIsolation(Arrays.asList(DC1, DC2, DC3), majorityValidator(DC1,
DC2, DC3), DC1, DC2);
+ }
+
+ /** Three DCs with majority validation split three ways: no segment
writes, every segment reads. */
+ @Test
+ public void testThreeWaySplit() throws Exception {
+ dcs = Arrays.asList(DC1, DC2, DC3);
+
+ MdcTopologyValidator validator = majorityValidator(DC1, DC2, DC3);
+
+ startCluster();
+
+ Map<String, Map<Integer, Integer>> expected = fill(DC1, validator);
+
+ splitInto(Arrays.asList(Arrays.asList(DC1), Arrays.asList(DC2),
Arrays.asList(DC3)));
+
+ for (String dc : dcs) {
+ for (String cacheName : expected.keySet()) {
+ IgniteCache<Integer, Integer> cache =
client(dc).cache(cacheName);
+
+ assertWriteRejected(() -> cache.put(KEYS, KEYS));
+
+ assertEquals(Integer.valueOf(1), cache.get(1));
+ }
+ }
+
+ restartAllSegmentsExcept(DC1);
+
+ for (Map.Entry<String, Map<Integer, Integer>> e : expected.entrySet())
+ assertDataInEveryDc(e.getKey(), e.getValue());
+
+ assertPartitionsSame(idleVerify(client(DC1),
expected.keySet().toArray(new String[0])));
+ }
+
+ /** Two DCs with DC1 as the main one: DC2 cut off from it only reads. */
+ @Test
+ public void testIsolatedDcOfTwo() throws Exception {
+ checkIsolation(Arrays.asList(DC1, DC2), mainDcValidator(DC1), DC2,
DC1);
+ }
+
+ /**
+ * @param dcs DCs of the cluster.
+ * @param validator Topology validator of the caches.
+ * @param isolatedDc DC to cut off; must end up read-only.
+ * @param writerDc DC that keeps writing.
+ */
+ private void checkIsolation(
+ List<String> dcs,
+ MdcTopologyValidator validator,
+ String isolatedDc,
+ String writerDc
+ ) throws Exception {
+ this.dcs = dcs;
+
+ startCluster();
+
+ Map<String, Map<Integer, Integer>> expected = fill(writerDc,
validator);
+
+ split(isolatedDc);
+
+ for (Map.Entry<String, Map<Integer, Integer>> e : expected.entrySet())
{
+ IgniteCache<Integer, Integer> writer =
client(writerDc).cache(e.getKey());
+ IgniteCache<Integer, Integer> isolated =
client(isolatedDc).cache(e.getKey());
+
+ writer.put(KEYS, KEYS);
+ e.getValue().put(KEYS, KEYS);
+
+ assertEquals(Integer.valueOf(0), writer.get(0));
+
+ assertWriteRejected(() -> isolated.put(KEYS + 1, KEYS + 1));
+
+ assertEquals(Integer.valueOf(1), isolated.get(1));
+ }
+
+ restartAllSegmentsExcept(writerDc);
+
+ for (Map.Entry<String, Map<Integer, Integer>> e : expected.entrySet())
+ assertDataInEveryDc(e.getKey(), e.getValue());
+
+ assertPartitionsSame(idleVerify(client(writerDc),
expected.keySet().toArray(new String[0])));
+ }
+
+ /**
+ * Creates an atomic and a transactional cache and writes {@link #KEYS}
keys into each.
+ *
+ * @param dc DC whose client creates and fills the caches.
+ * @param validator Topology validator of the caches.
+ * @return Content of each cache by its name.
+ */
+ private Map<String, Map<Integer, Integer>> fill(String dc,
MdcTopologyValidator validator) {
+ Map<String, Map<Integer, Integer>> expected = new HashMap<>();
+
+ for (CacheAtomicityMode mode : new CacheAtomicityMode[] {ATOMIC,
TRANSACTIONAL}) {
+ IgniteCache<Integer, Integer> cache =
client(dc).createCache(cacheConfiguration(mode.name(), mode, validator));
+
+ Map<Integer, Integer> data = new HashMap<>();
+
+ for (int key = 0; key < KEYS; key++)
+ data.put(key, key);
+
+ cache.putAll(data);
+
+ expected.put(mode.name(), data);
+ }
+
+ return expected;
+ }
+}
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/MdcTopologySplitAbstractTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/MdcTopologySplitAbstractTest.java
new file mode 100644
index 00000000000..98127f21469
--- /dev/null
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/MdcTopologySplitAbstractTest.java
@@ -0,0 +1,606 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.affinity.rendezvous.MdcAffinityBackupFilter;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
+import
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.topology.MdcTopologyValidator;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_DATA_CENTER_ID;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ * Base for tests that cut the network between data centers (DCs) of one
cluster.
+ *
+ * <p>The cluster has {@link #serversPerDc()} servers and one client in each
DC of {@link #dataCenters()}.
+ * Servers {@code 0 .. dcs * serversPerDc - 1} go DC by DC; the client of DC
{@code i} has index
+ * {@code dcs * serversPerDc + i}. A client knows only the servers of its own
DC, so it stays in its DC's segment
+ * of a split.</p>
+ *
+ * <p>{@link #split(String...)} cuts the given DCs off from the rest, and
{@link #splitInto(List)} splits the DCs into
+ * any number of segments: discovery connections between segments fail, and
communication messages between them are
+ * held. Either then waits, with a timeout, until every node sees exactly its
own segment.
+ * {@link #restartAllSegmentsExcept(String)} drops the held messages
(delivering them would replay one segment's view
+ * on another) and restarts every segment except the given DC's segment; the
restarted nodes rejoin that segment.
+ * Subclasses can hold more messages while the cluster is split through
+ * {@link #blockMessage(ClusterNode, ClusterNode, Message)}.</p>
+ *
+ * <p>Unlike {@link IgniteCacheTopologySplitAbstractTest#splitAndWait()},
nothing here waits for one exact topology
+ * version without a timeout, so a split that ends in an unexpected topology
fails the test instead of hanging it.</p>
+ */
+public abstract class MdcTopologySplitAbstractTest extends
IgniteCacheTopologySplitAbstractTest {
+ /** */
+ protected static final String DC1 = "DC1";
+
+ /** */
+ protected static final String DC2 = "DC2";
+
+ /** */
+ protected static final String DC3 = "DC3";
+
+ /** Time for the segments of a split to see only themselves, and for the
cluster to become whole after a restart. */
+ protected static final long TOPOLOGY_TIMEOUT = 30_000;
+
+ /** */
+ private static final String LOCAL_IP = "127.0.0.1";
+
+ /** Message of the exception a write gets when the topology validator
rejects it. */
+ private static final String VALIDATOR_REJECTION = "cache topology is not
valid";
+
+ /**
+ * Segment number of each DC in the current split, from 0: DCs with the
same number are in one segment. Empty
+ * when the cluster is whole.
+ */
+ private volatile Map<String, Integer> segmentNumByDc =
Collections.emptyMap();
+
+ /** @return DCs of the cluster, in the order their servers are numbered. */
+ protected abstract List<String> dataCenters();
+
+ /** @return Number of servers in each DC. */
+ protected int serversPerDc() {
+ return 2;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ int idx = getTestIgniteInstanceIndex(igniteInstanceName);
+
+ String dc = dataCenterOf(idx);
+
+ cfg.setUserAttributes(F.asMap(IGNITE_DATA_CENTER_ID, dc));
+
+ TcpDiscoverySpi disco = new MdcSplitDiscoverySpi(dc);
+
+
disco.setReconnectCount(((TcpDiscoverySpi)cfg.getDiscoverySpi()).getReconnectCount());
+
+ if (isServer(idx)) {
+ disco.setLocalPort(discoveryPort(idx));
+ disco.setLocalPortRange(0);
+ disco.setIpFinder(new
TcpDiscoveryVmIpFinder().setAddresses(serverAddresses(dataCenters())));
+ }
+ else
+ disco.setIpFinder(new
TcpDiscoveryVmIpFinder().setAddresses(serverAddresses(Collections.singleton(dc))));
+
+ cfg.setDiscoverySpi(disco);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ segmentNumByDc = Collections.emptyMap();
+
+ stopAllGrids();
+
+ super.afterTest();
+ }
+
+ /**
+ * Starts every server, then a client in each DC.
+ *
+ * @throws Exception If failed.
+ */
+ protected void startCluster() throws Exception {
+ startGridsMultiThreaded(serverCount());
+
+ for (String dc : dataCenters())
+ startClientGrid(clientIndex(dc));
+
+ awaitPartitionMapExchange();
+ }
+
+ /**
+ * Cuts the given DCs off from the rest of the cluster and waits until
each segment sees only itself.
+ *
+ * @param dcs DCs to cut off.
+ */
+ protected void split(String... dcs) throws Exception {
+ List<String> cut = Arrays.asList(dcs);
+
+ List<String> rest = new ArrayList<>(dataCenters());
+
+ rest.removeAll(cut);
+
+ splitInto(Arrays.asList(rest, cut));
+ }
+
+ /**
+ * Splits the DCs into the given segments and waits until each segment
sees only itself.
+ *
+ * @param segments Segments of the split: two at least, together holding
every DC exactly once.
+ */
+ protected void splitInto(List<List<String>> segments) throws Exception {
+ assertTrue("Cluster is split already: " + segmentNumByDc,
segmentNumByDc.isEmpty());
+ assertTrue("A split needs two segments at least: " + segments,
segments.size() >= 2);
+
+ Map<String, Integer> split = new HashMap<>();
+
+ for (int seg = 0; seg < segments.size(); seg++) {
+ assertFalse("Every segment must have a DC: " + segments,
segments.get(seg).isEmpty());
+
+ for (String dc : segments.get(seg)) {
+ assertTrue("Unknown DC: " + dc, dataCenters().contains(dc));
+ assertNull("DC is listed twice: " + dc, split.put(dc, seg));
+ }
+ }
+
+ assertEquals("Every DC must be in a segment: " + segments,
dataCenters().size(), split.size());
+
+ log.info(">>> Splitting DCs into " + segments);
+
+ Map<String, Integer> segmentMap = Collections.unmodifiableMap(split);
+
+ // Hold communication first, then cut discovery: no message crosses
the split once any node sees it.
+ for (Ignite ignite : G.allGrids())
+ communication(ignite).blockMessages(new
MdcSplitBlocker(ignite.cluster().localNode(), segmentMap));
+
+ segmentNumByDc = segmentMap;
+
+ long start = System.nanoTime();
+
+ awaitSegmentsSeeThemselves();
+
+ log.info(">>> Split done in " +
TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start) + " ms");
+ }
+
+ /**
+ * Restarts every node of every segment except the segment of the given
DC, and waits until the cluster is whole
+ * again. The restarted nodes rejoin the remaining segment and rebalance
from it, so pass a DC of the segment whose
+ * data must survive, normally the one that kept writing. The segments
form separate rings, which never merge on
+ * their own. Every held message is dropped and every message filter is
removed, including those added by
+ * {@link #blockMessage}.
+ *
+ * @param dc DC whose segment is not restarted.
+ */
+ protected void restartAllSegmentsExcept(String dc) throws Exception {
+ Map<String, Integer> split = segmentNumByDc;
+
+ assertFalse("Cluster is not split", split.isEmpty());
+ assertTrue("Unknown DC: " + dc, split.containsKey(dc));
+
+ Integer keptSegment = split.get(dc);
+
+ List<String> restartDcs = dataCenters().stream()
+ .filter(d -> !split.get(d).equals(keptSegment))
+ .collect(Collectors.toList());
+
+ log.info(">>> Restarting DCs " + restartDcs + ", keeping the segment
of " + dc);
+
+ List<Integer> restart = new ArrayList<>();
+
+ for (String restartDc : restartDcs) {
+ restart.addAll(serverIndexes(restartDc));
+ restart.add(clientIndex(restartDc));
+ }
+
+ for (int idx : restart)
+ stopGrid(idx, true);
+
+ segmentNumByDc = Collections.emptyMap();
+
+ for (Ignite ignite : G.allGrids())
+ communication(ignite).stopBlock(false);
+
+ for (int idx : restart) {
+ if (isServer(idx))
+ startGrid(idx);
+ }
+
+ for (int idx : restart) {
+ if (!isServer(idx))
+ startClientGrid(idx);
+ }
+
+ int total = serverCount() + dataCenters().size();
+
+ assertTrue("Cluster did not become whole: " + topologyViews(),
+ GridTestUtils.waitForCondition(() -> G.allGrids().stream()
+ .allMatch(ignite -> ignite.cluster().nodes().size() == total),
TOPOLOGY_TIMEOUT));
+
+ awaitPartitionMapExchange();
+
+ log.info(">>> Cluster is whole again");
+ }
+
+ /**
+ * @param name Cache name.
+ * @param atomicityMode Atomicity mode.
+ * @param validator Topology validator, or {@code null} for none.
+ * @return Cache configuration that keeps one copy of each partition in
every DC.
+ */
+ protected CacheConfiguration<Integer, Integer> cacheConfiguration(
+ String name,
+ CacheAtomicityMode atomicityMode,
+ @Nullable MdcTopologyValidator validator
+ ) {
+ int dcs = dataCenters().size();
+
+ int backups = dcs - 1;
+
+ return new CacheConfiguration<Integer, Integer>(name)
+ .setAtomicityMode(atomicityMode)
+ .setWriteSynchronizationMode(FULL_SYNC)
+ .setBackups(backups)
+ .setAffinity(new RendezvousAffinityFunction(false, 32)
+ .setAffinityBackupFilter(new MdcAffinityBackupFilter(dcs,
backups)))
+ .setTopologyValidator(validator);
+ }
+
+ /**
+ * @param dcs Every DC of the cluster, an odd number of them.
+ * @return Validator that lets a segment write while it sees a majority of
the DCs.
+ */
+ protected static MdcTopologyValidator majorityValidator(String... dcs) {
+ // The validator accepts an even set too, and then an even split
leaves no segment that writes.
+ assertTrue("Majority validation needs an odd number of DCs, use
mainDcValidator() for an even one: " +
+ Arrays.toString(dcs), dcs.length % 2 == 1);
+
+ MdcTopologyValidator validator = new MdcTopologyValidator();
+
+ validator.setDatacenters(new HashSet<>(Arrays.asList(dcs)));
+
+ return validator;
+ }
+
+ /**
+ * @param mainDc Main DC.
+ * @return Validator that lets a segment write while it sees the main DC.
+ */
+ protected static MdcTopologyValidator mainDcValidator(String mainDc) {
+ MdcTopologyValidator validator = new MdcTopologyValidator();
+
+ validator.setMainDatacenter(mainDc);
+
+ return validator;
+ }
+
+ /**
+ * Checks that a write is rejected by the topology validator. The
rejection surfaces as
+ * {@link CacheInvalidStateException} for atomic caches and explicit
transactions, and wrapped into a
+ * {@code TransactionRollbackException} for implicit transactional writes,
so the cause chain is searched.
+ * Other reasons for the same exception (lost partitions, inactive or
read-only cluster) don't count.
+ *
+ * @param write Write that must be rejected.
+ */
+ protected static void assertWriteRejected(Runnable write) {
+ Throwable err = GridTestUtils.assertThrowsWithCause(write,
Exception.class);
+
+ assertTrue("Unexpected rejection: " + X.getFullStackTrace(err),
+ X.hasCause(err, VALIDATOR_REJECTION,
CacheInvalidStateException.class));
+ }
+
+ /**
+ * Checks that every DC keeps a copy of every key with the expected value.
+ *
+ * @param cacheName Cache name.
+ * @param expected Expected cache content.
+ */
+ protected void assertDataInEveryDc(String cacheName, Map<Integer, Integer>
expected) {
+ assertEquals(expected.size(),
client(dataCenters().get(0)).cache(cacheName).size(CachePeekMode.PRIMARY));
+
+ for (String dc : dataCenters()) {
+ Map<Integer, Integer> dcData = new HashMap<>();
+
+ for (int idx : serverIndexes(dc)) {
+ IgniteCache<Integer, Integer> cache =
grid(idx).cache(cacheName);
+
+ for (Integer key : expected.keySet()) {
+ Integer val = cache.localPeek(key, CachePeekMode.PRIMARY,
CachePeekMode.BACKUP);
+
+ if (val != null)
+ assertNull("Key has two copies in " + dc + ": " + key,
dcData.put(key, val));
+ }
+ }
+
+ assertEquals("Data in " + dc, expected, dcData);
+ }
+ }
+
+ /** @return Number of servers. */
+ protected int serverCount() {
+ return dataCenters().size() * serversPerDc();
+ }
+
+ /**
+ * @param dc DC.
+ * @return Indexes of the DC's servers.
+ */
+ protected List<Integer> serverIndexes(String dc) {
+ int first = dataCenters().indexOf(dc) * serversPerDc();
+
+ assertTrue("Unknown DC: " + dc, first >= 0);
+
+ List<Integer> idxs = new ArrayList<>(serversPerDc());
+
+ for (int i = 0; i < serversPerDc(); i++)
+ idxs.add(first + i);
+
+ return idxs;
+ }
+
+ /**
+ * @param dc DC.
+ * @return Index of the DC's client.
+ */
+ protected int clientIndex(String dc) {
+ int dcIdx = dataCenters().indexOf(dc);
+
+ assertTrue("Unknown DC: " + dc, dcIdx >= 0);
+
+ return serverCount() + dcIdx;
+ }
+
+ /**
+ * @param dc DC.
+ * @return The DC's client.
+ */
+ protected IgniteEx client(String dc) {
+ return grid(clientIndex(dc));
+ }
+
+ /**
+ * Lets a subclass hold more messages while the cluster is split, in
addition to those crossing the split.
+ * Held messages are dropped by {@link #restartAllSegmentsExcept(String)}.
+ *
+ * @param locNode Sending node.
+ * @param rmtNode Receiving node.
+ * @param msg Message.
+ * @return {@code True} to hold the message.
+ */
+ protected boolean blockMessage(ClusterNode locNode, ClusterNode rmtNode,
Message msg) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean segmented() {
+ return !segmentNumByDc.isEmpty();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean isBlocked(int locPort, int rmtPort) {
+ // Not used: the discovery SPI here decides by the local node's DC,
since a client has no port of its own.
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * @param dc1 DC.
+ * @param dc2 DC.
+ * @return {@code True} if the DCs are in different segments of the
current split.
+ */
+ private boolean dcsAreInDifferentSegments(String dc1, String dc2) {
+ Map<String, Integer> split = segmentNumByDc;
+
+ return !split.isEmpty() && !split.get(dc1).equals(split.get(dc2));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int segment(ClusterNode node) {
+ // Segment of the node in the current split, or 0 when the cluster is
whole. There can be more than the
+ // template's two segments: its splitAndWait(), which expects two, is
not used.
+ return segmentNumByDc.getOrDefault(node.dataCenterId(), 0);
+ }
+
+ /** Waits until every node sees exactly the nodes of its own segment, and
its last exchange is done. */
+ private void awaitSegmentsSeeThemselves() throws Exception {
+ Map<Integer, Set<UUID>> segments = new HashMap<>();
+
+ for (Ignite ignite : G.allGrids()) {
+ ClusterNode node = ignite.cluster().localNode();
+
+ segments.computeIfAbsent(segment(node), s -> new
HashSet<>()).add(node.id());
+ }
+
+ boolean done = GridTestUtils.waitForCondition(() ->
G.allGrids().stream().allMatch(ignite -> {
+ Set<UUID> seen =
ignite.cluster().nodes().stream().map(ClusterNode::id).collect(Collectors.toSet());
+
+ if
(!seen.equals(segments.get(segment(ignite.cluster().localNode()))))
+ return false;
+
+ GridDhtPartitionsExchangeFuture exchFut =
+
((IgniteEx)ignite).context().cache().context().exchange().lastTopologyFuture();
+
+ return exchFut != null && exchFut.isDone()
+ && exchFut.topologyVersion().topologyVersion() ==
ignite.cluster().topologyVersion();
+ }), TOPOLOGY_TIMEOUT);
+
+ assertTrue("Segments of the split did not separate within " +
TOPOLOGY_TIMEOUT + " ms: " + topologyViews(),
+ done);
+ }
+
+ /** @return What each node sees, for failure messages. */
+ private String topologyViews() {
+ StringBuilder sb = new StringBuilder();
+
+ for (Ignite ignite : G.allGrids()) {
+ sb.append("\n ").append(ignite.name())
+ .append("
[dc=").append(ignite.cluster().localNode().dataCenterId())
+ .append(", topVer=").append(ignite.cluster().topologyVersion())
+ .append(", sees=").append(ignite.cluster().nodes().stream()
+ .map(n -> n.dataCenterId() + (n.isClient() ? "c" : "s"))
+ .sorted()
+ .collect(Collectors.joining(" ")))
+ .append(']');
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * @param idx Node index.
+ * @return {@code True} if the index belongs to a server.
+ */
+ private boolean isServer(int idx) {
+ return idx < serverCount();
+ }
+
+ /**
+ * @param idx Node index.
+ * @return DC of the node.
+ */
+ private String dataCenterOf(int idx) {
+ int dcIdx = isServer(idx) ? idx / serversPerDc() : idx - serverCount();
+
+ assertTrue("No DC for node " + idx, dcIdx < dataCenters().size());
+
+ return dataCenters().get(dcIdx);
+ }
+
+ /**
+ * @param idx Server index.
+ * @return Discovery port of the server.
+ */
+ private static int discoveryPort(int idx) {
+ return TcpDiscoverySpi.DFLT_PORT + idx;
+ }
+
+ /**
+ * @param port Port.
+ * @return DC of the server listening on the discovery port, or {@code
null} if it is not a server's port.
+ */
+ @Nullable private String dataCenterOfPort(int port) {
+ int idx = port - TcpDiscoverySpi.DFLT_PORT;
+
+ return idx >= 0 && isServer(idx) ? dataCenterOf(idx) : null;
+ }
+
+ /**
+ * @param dcs DCs.
+ * @return Discovery addresses of the DCs' servers.
+ */
+ private Collection<String> serverAddresses(Collection<String> dcs) {
+ List<String> addrs = new ArrayList<>();
+
+ for (String dc : dcs) {
+ for (int idx : serverIndexes(dc))
+ addrs.add(LOCAL_IP + ':' + discoveryPort(idx));
+ }
+
+ return addrs;
+ }
+
+ /**
+ * @param ignite Node.
+ * @return Its communication SPI.
+ */
+ private static TestRecordingCommunicationSpi communication(Ignite ignite) {
+ return
(TestRecordingCommunicationSpi)ignite.configuration().getCommunicationSpi();
+ }
+
+ /** Holds communication messages between the segments of a split, and
those {@link #blockMessage} picks. */
+ private class MdcSplitBlocker implements IgniteBiPredicate<ClusterNode,
Message> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Local node. */
+ private final ClusterNode locNode;
+
+ /** Segment number of each DC in the split. */
+ private final Map<String, Integer> segmentNumByDc;
+
+ /**
+ * @param locNode Local node.
+ * @param segmentNumByDc Segment number of each DC in the split.
+ */
+ MdcSplitBlocker(ClusterNode locNode, Map<String, Integer>
segmentNumByDc) {
+ this.locNode = locNode;
+ this.segmentNumByDc = segmentNumByDc;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean apply(ClusterNode node, Message msg) {
+ return
!segmentNumByDc.get(locNode.dataCenterId()).equals(segmentNumByDc.get(node.dataCenterId()))
+ || blockMessage(locNode, node, msg);
+ }
+ }
+
+ /**
+ * Discovery SPI that fails connections from its node to servers in
another segment of the current split.
+ * The segment is taken from the node's DC, not from its local port: a
client has no port of its own.
+ */
+ private class MdcSplitDiscoverySpi extends SplitTcpDiscoverySpi {
+ /** DC of the local node. */
+ private final String dc;
+
+ /** @param dc DC of the local node. */
+ MdcSplitDiscoverySpi(String dc) {
+ this.dc = dc;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean segmented(InetSocketAddress sockAddr) {
+ String rmtDc = dataCenterOfPort(sockAddr.getPort());
+
+ return rmtDc != null && dcsAreInDifferentSegments(dc, rmtDc);
+ }
+ }
+}
diff --git
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteTopologyValidatorTestSuite.java
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteTopologyValidatorTestSuite.java
index ba3e55667f2..5d401973527 100644
---
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteTopologyValidatorTestSuite.java
+++
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteTopologyValidatorTestSuite.java
@@ -34,6 +34,7 @@ import
org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorReplic
import
org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorReplicatedAtomicCacheTest;
import
org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorReplicatedTxCacheGroupsTest;
import
org.apache.ignite.internal.processors.cache.IgniteTopologyValidatorReplicatedTxCacheTest;
+import
org.apache.ignite.internal.processors.cache.distributed.dht.MdcDcIsolationTest;
import org.apache.ignite.testframework.GridTestUtils;
/**
@@ -64,6 +65,8 @@ public class IgniteTopologyValidatorTestSuite {
GridTestUtils.addTestIfNeeded(suite,
IgniteTopologyValidatorGridSplitCacheTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite,
CacheTopologyValidatorProviderTest.class, ignoredTests);
+ GridTestUtils.addTestIfNeeded(suite, MdcDcIsolationTest.class,
ignoredTests);
+
return suite;
}
}