This is an automated email from the ASF dual-hosted git repository.
vpyatkov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new b520ad3bc3 IGNITE-20561 Change condition for
DistributionZonesUtil#triggerKeyConditionForZonesChanges to solve inconsistency
issues (#2743)
b520ad3bc3 is described below
commit b520ad3bc337b8ca051ae88764d6ae084f6dcc09
Author: Mirza Aliev <[email protected]>
AuthorDate: Wed Oct 25 14:36:06 2023 +0400
IGNITE-20561 Change condition for
DistributionZonesUtil#triggerKeyConditionForZonesChanges to solve inconsistency
issues (#2743)
---
.../distributionzones/DistributionZoneManager.java | 25 ++++----
.../distributionzones/DistributionZonesUtil.java | 61 ++++++++++--------
...ibutionZoneManagerConfigurationChangesTest.java | 22 -------
...butionZoneManagerLogicalTopologyEventsTest.java | 10 ++-
.../DistributionZonesTestUtil.java | 23 -------
...niteDistributionZoneManagerNodeRestartTest.java | 74 ++++++++++++++++++++++
6 files changed, 123 insertions(+), 92 deletions(-)
diff --git
a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
index 0e8bd9c4f2..982c7d7f09 100644
---
a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
+++
b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZoneManager.java
@@ -29,12 +29,13 @@ import static
org.apache.ignite.internal.catalog.commands.CatalogUtils.INFINITE_
import static
org.apache.ignite.internal.catalog.events.CatalogEvent.ZONE_ALTER;
import static
org.apache.ignite.internal.catalog.events.CatalogEvent.ZONE_CREATE;
import static org.apache.ignite.internal.catalog.events.CatalogEvent.ZONE_DROP;
+import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.conditionForZoneCreation;
+import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.conditionForZoneRemoval;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.createZoneManagerExecutor;
-import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.deleteDataNodesAndUpdateTriggerKeys;
+import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.deleteDataNodesAndTriggerKeys;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.extractChangeTriggerRevision;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.extractDataNodes;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.toDataNodesMap;
-import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.triggerKeyConditionForZonesChanges;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.triggerScaleUpScaleDownKeysCondition;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.updateDataNodesAndScaleDownTriggerKey;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.updateDataNodesAndScaleUpTriggerKey;
@@ -106,6 +107,7 @@ import org.apache.ignite.internal.metastorage.WatchListener;
import org.apache.ignite.internal.metastorage.dsl.CompoundCondition;
import org.apache.ignite.internal.metastorage.dsl.Condition;
import org.apache.ignite.internal.metastorage.dsl.Iif;
+import org.apache.ignite.internal.metastorage.dsl.SimpleCondition;
import org.apache.ignite.internal.metastorage.dsl.StatementResult;
import org.apache.ignite.internal.metastorage.dsl.Update;
import org.apache.ignite.internal.thread.NamedThreadFactory;
@@ -509,8 +511,8 @@ public class DistributionZoneManager implements
IgniteComponent {
/**
* Method initialise data nodes value for the specified zone, also sets
{@code revision} to the
- * {@link DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)}, {@link
DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)} and
- * {@link DistributionZonesUtil#zonesChangeTriggerKey(int)} if it passes
the condition. It is called on the first creation of a zone.
+ * {@link DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)} and
{@link DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)}
+ * if it passes the condition. It is called on the first creation of a
zone.
*
* @param zoneId Unique id of a zone
* @param revision Revision of an event that has triggered this method.
@@ -527,9 +529,8 @@ public class DistributionZoneManager implements
IgniteComponent {
}
try {
- // Update data nodes for a zone only if the revision of the event
is newer than value in that trigger key,
- // so we do not react on a stale events
- CompoundCondition triggerKeyCondition =
triggerKeyConditionForZonesChanges(revision, zoneId);
+ // Update data nodes for a zone only if the corresponding data
nodes keys weren't initialised in ms yet.
+ CompoundCondition triggerKeyCondition =
conditionForZoneCreation(zoneId);
Update dataNodesAndTriggerKeyUpd =
updateDataNodesAndTriggerKeys(zoneId, revision,
toBytes(toDataNodesMap(dataNodes)));
@@ -578,9 +579,9 @@ public class DistributionZoneManager implements
IgniteComponent {
}
try {
- CompoundCondition triggerKeyCondition =
triggerKeyConditionForZonesChanges(revision, zoneId);
+ SimpleCondition triggerKeyCondition =
conditionForZoneRemoval(zoneId);
- Update removeKeysUpd = deleteDataNodesAndUpdateTriggerKeys(zoneId,
revision);
+ Update removeKeysUpd = deleteDataNodesAndTriggerKeys(zoneId,
revision);
Iif iif = iif(triggerKeyCondition, removeKeysUpd,
ops().yield(false));
@@ -887,8 +888,7 @@ public class DistributionZoneManager implements
IgniteComponent {
/**
* Method updates data nodes value for the specified zone after scale up
timer timeout, sets {@code revision} to the
- * {@link DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)} and
- * {@link DistributionZonesUtil#zonesChangeTriggerKey(int)} if it passes
the condition.
+ * {@link DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)} if it
passes the condition.
*
* @param zoneId Unique id of a zone
* @param revision Revision of an event that has triggered this method.
@@ -998,8 +998,7 @@ public class DistributionZoneManager implements
IgniteComponent {
/**
* Method updates data nodes value for the specified zone after scale down
timer timeout, sets {@code revision} to the
- * {@link DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)} and
- * {@link DistributionZonesUtil#zonesChangeTriggerKey(int)} if it passes
the condition.
+ * {@link DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)} if it
passes the condition.
*
* @param zoneId Unique id of a zone
* @param revision Revision of an event that has triggered this method.
diff --git
a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZonesUtil.java
b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZonesUtil.java
index 1ea89d2085..5d050674cf 100644
---
a/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZonesUtil.java
+++
b/modules/distribution-zones/src/main/java/org/apache/ignite/internal/distributionzones/DistributionZonesUtil.java
@@ -22,8 +22,9 @@ import static java.util.stream.Collectors.toMap;
import static java.util.stream.Collectors.toSet;
import static
org.apache.ignite.internal.catalog.commands.CatalogUtils.DEFAULT_FILTER;
import static org.apache.ignite.internal.metastorage.dsl.Conditions.and;
+import static org.apache.ignite.internal.metastorage.dsl.Conditions.exists;
import static org.apache.ignite.internal.metastorage.dsl.Conditions.notExists;
-import static org.apache.ignite.internal.metastorage.dsl.Conditions.or;
+import static
org.apache.ignite.internal.metastorage.dsl.Conditions.notTombstone;
import static org.apache.ignite.internal.metastorage.dsl.Conditions.value;
import static org.apache.ignite.internal.metastorage.dsl.Operations.ops;
import static org.apache.ignite.internal.metastorage.dsl.Operations.put;
@@ -98,9 +99,6 @@ public class DistributionZonesUtil {
/** Key prefix for zones' logical topology version. */
private static final String DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_VERSION =
DISTRIBUTION_ZONES_LOGICAL_TOPOLOGY_PREFIX + "version";
- /** Key prefix, needed for processing the event about zone's update was
triggered only once. */
- private static final String DISTRIBUTION_ZONES_CHANGE_TRIGGER_KEY_PREFIX =
"distributionZones.change.trigger.";
-
/** Key prefix that represents {@link ZoneState#topologyAugmentationMap()}
in the Vault.*/
private static final String
DISTRIBUTION_ZONES_TOPOLOGY_AUGMENTATION_VAULT_PREFIX =
"vault.distributionZones.topologyAugmentation.";
@@ -187,14 +185,6 @@ public class DistributionZonesUtil {
return
Integer.parseInt(strKey.substring(DISTRIBUTION_ZONE_DATA_NODES_VALUE_PREFIX.length()));
}
- /**
- * The key needed for processing an event about zone's creation and
deletion.
- * With this key we can be sure that event was triggered only once.
- */
- public static ByteArray zonesChangeTriggerKey(int zoneId) {
- return new ByteArray(DISTRIBUTION_ZONES_CHANGE_TRIGGER_KEY_PREFIX +
zoneId);
- }
-
/**
* The key needed for processing an event about zone's data node
propagation on scale up.
* With this key we can be sure that event was triggered only once.
@@ -271,19 +261,29 @@ public class DistributionZonesUtil {
}
/**
- * Condition for updating {@link
DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)} key.
- * Update only if the revision of the event is newer than value in that
trigger key.
+ * Condition for creating all data nodes' related keys in Meta Storage.
Condition passes only when
+ * {@link DistributionZonesUtil#zoneDataNodesKey(int)} not exists and not
a tombstone in the Meta Storage.
*
- * @param revision Event revision.
+ * @param zoneId Distribution zone id
* @return Update condition.
*/
- static CompoundCondition triggerKeyConditionForZonesChanges(long revision,
int zoneId) {
- return or(
- notExists(zonesChangeTriggerKey(zoneId)),
-
value(zonesChangeTriggerKey(zoneId)).lt(ByteUtils.longToBytes(revision))
+ static CompoundCondition conditionForZoneCreation(int zoneId) {
+ return and(
+ notExists(zoneDataNodesKey(zoneId)),
+ notTombstone(zoneDataNodesKey(zoneId))
);
}
+ /**
+ * Condition for removing all data nodes' related keys in Meta Storage.
+ *
+ * @param zoneId Distribution zone id
+ * @return Update condition.
+ */
+ static SimpleCondition conditionForZoneRemoval(int zoneId) {
+ return exists(zoneDataNodesKey(zoneId));
+ }
+
/**
* Condition for updating {@link
DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)} key.
* Update only if the revision of the event is newer than value in that
trigger key.
@@ -328,6 +328,14 @@ public class DistributionZonesUtil {
).yield(true);
}
+ /**
+ * Updates data nodes value for a zone and set {@code revision} to {@link
DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)}.
+ *
+ * @param zoneId Distribution zone id
+ * @param revision Revision of the event.
+ * @param nodes Data nodes.
+ * @return Update command for the meta storage.
+ */
static Update updateDataNodesAndScaleDownTriggerKey(int zoneId, long
revision, byte[] nodes) {
return ops(
put(zoneDataNodesKey(zoneId), nodes),
@@ -337,8 +345,8 @@ public class DistributionZonesUtil {
/**
- * Updates data nodes value for a zone and set {@code revision} to {@link
DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)},
- * {@link DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)} and
{@link DistributionZonesUtil#zonesChangeTriggerKey(int)}.
+ * Updates data nodes value for a zone and set {@code revision} to {@link
DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)} and
+ * {@link DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)}.
*
* @param zoneId Distribution zone id
* @param revision Revision of the event.
@@ -349,26 +357,23 @@ public class DistributionZonesUtil {
return ops(
put(zoneDataNodesKey(zoneId), nodes),
put(zoneScaleUpChangeTriggerKey(zoneId),
ByteUtils.longToBytes(revision)),
- put(zoneScaleDownChangeTriggerKey(zoneId),
ByteUtils.longToBytes(revision)),
- put(zonesChangeTriggerKey(zoneId),
ByteUtils.longToBytes(revision))
+ put(zoneScaleDownChangeTriggerKey(zoneId),
ByteUtils.longToBytes(revision))
).yield(true);
}
/**
* Deletes data nodes, {@link
DistributionZonesUtil#zoneScaleUpChangeTriggerKey(int)},
- * {@link DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)} values
for a zone. Also sets {@code revision} to
- * {@link DistributionZonesUtil#zonesChangeTriggerKey(int)}.
+ * {@link DistributionZonesUtil#zoneScaleDownChangeTriggerKey(int)} values
for a zone.
*
* @param zoneId Distribution zone id
* @param revision Revision of the event.
* @return Update command for the meta storage.
*/
- static Update deleteDataNodesAndUpdateTriggerKeys(int zoneId, long
revision) {
+ static Update deleteDataNodesAndTriggerKeys(int zoneId, long revision) {
return ops(
remove(zoneDataNodesKey(zoneId)),
remove(zoneScaleUpChangeTriggerKey(zoneId)),
- remove(zoneScaleDownChangeTriggerKey(zoneId)),
- put(zonesChangeTriggerKey(zoneId),
ByteUtils.longToBytes(revision))
+ remove(zoneScaleDownChangeTriggerKey(zoneId))
).yield(true);
}
diff --git
a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerConfigurationChangesTest.java
b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerConfigurationChangesTest.java
index ac1b42e369..ca5509592e 100644
---
a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerConfigurationChangesTest.java
+++
b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerConfigurationChangesTest.java
@@ -22,9 +22,7 @@ import static
java.util.concurrent.CompletableFuture.completedFuture;
import static java.util.stream.Collectors.toSet;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertDataNodesForZoneWithAttributes;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertZoneScaleUpChangeTriggerKey;
-import static
org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertZonesChangeTriggerKey;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
-import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesChangeTriggerKey;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesGlobalStateRevision;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyVault;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyVersionKey;
@@ -53,7 +51,6 @@ import
org.apache.ignite.internal.cluster.management.topology.api.LogicalTopolog
import
org.apache.ignite.internal.cluster.management.topology.api.LogicalTopologySnapshot;
import org.apache.ignite.internal.hlc.HybridClock;
import org.apache.ignite.internal.hlc.HybridClockImpl;
-import org.apache.ignite.internal.hlc.HybridTimestamp;
import
org.apache.ignite.internal.metastorage.impl.StandaloneMetaStorageManager;
import
org.apache.ignite.internal.metastorage.server.SimpleInMemoryKeyValueStorage;
import org.apache.ignite.internal.testframework.IgniteAbstractTest;
@@ -160,8 +157,6 @@ public class
DistributionZoneManagerConfigurationChangesTest extends IgniteAbstr
assertDataNodesForZoneWithAttributes(zoneId,
nodes.stream().map(NodeWithAttributes::node).collect(toSet()), keyValueStorage);
assertZoneScaleUpChangeTriggerKey(2L, zoneId, keyValueStorage);
-
- assertZonesChangeTriggerKey(2, zoneId, keyValueStorage);
}
@Test
@@ -188,23 +183,6 @@ public class
DistributionZoneManagerConfigurationChangesTest extends IgniteAbstr
assertZoneScaleUpChangeTriggerKey(2L, zoneId, keyValueStorage);
assertZoneScaleUpChangeTriggerKey(4L, zoneId2, keyValueStorage);
- assertZonesChangeTriggerKey(2L, zoneId, keyValueStorage);
- assertZonesChangeTriggerKey(4L, zoneId2, keyValueStorage);
- }
-
- @Test
- void testZoneDeleteDoNotRemoveMetaStorageKey() throws Exception {
- createZone(ZONE_NAME);
-
- int zoneId = getZoneId(ZONE_NAME);
-
- assertDataNodesForZoneWithAttributes(zoneId,
nodes.stream().map(NodeWithAttributes::node).collect(toSet()), keyValueStorage);
-
- keyValueStorage.put(zonesChangeTriggerKey(zoneId).bytes(),
longToBytes(100), HybridTimestamp.MIN_VALUE);
-
- dropZone(ZONE_NAME);
-
- assertDataNodesForZoneWithAttributes(zoneId,
nodes.stream().map(NodeWithAttributes::node).collect(toSet()), keyValueStorage);
}
private void createZone(String zoneName) {
diff --git
a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerLogicalTopologyEventsTest.java
b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerLogicalTopologyEventsTest.java
index 9f58aca736..122acad907 100644
---
a/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerLogicalTopologyEventsTest.java
+++
b/modules/distribution-zones/src/test/java/org/apache/ignite/internal/distributionzones/DistributionZoneManagerLogicalTopologyEventsTest.java
@@ -23,9 +23,9 @@ import static
org.apache.ignite.internal.distributionzones.DistributionZonesTest
import static
org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertLogicalTopology;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertLogicalTopologyVersion;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.mockVaultZonesLogicalTopologyKey;
-import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesChangeTriggerKey;
+import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyVersionKey;
-import static org.apache.ignite.internal.util.ByteUtils.longToBytes;
+import static org.apache.ignite.internal.util.ByteUtils.toBytes;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.Collections;
@@ -208,11 +208,9 @@ public class
DistributionZoneManagerLogicalTopologyEventsTest extends BaseDistri
//TODO: IGNITE-19955 Check if this test is needed.
@Test
void testStaleVaultRevisionOnZoneManagerStart() throws Exception {
- long revision = 100;
-
int defaultZoneId = getZoneId(DEFAULT_ZONE_NAME);
- keyValueStorage.put(zonesChangeTriggerKey(defaultZoneId).bytes(),
longToBytes(revision), HybridTimestamp.MIN_VALUE);
+ keyValueStorage.put(zoneDataNodesKey(defaultZoneId).bytes(),
toBytes(Collections.emptyMap()), HybridTimestamp.MIN_VALUE);
Set<LogicalNode> nodes = Set.of(
new LogicalNode(new ClusterNodeImpl("node1", "node1",
NetworkAddress.from("127.0.0.1:127")), Collections.emptyMap()),
@@ -223,6 +221,6 @@ public class
DistributionZoneManagerLogicalTopologyEventsTest extends BaseDistri
startDistributionZoneManager();
- assertDataNodesForZone(defaultZoneId, null, keyValueStorage);
+ assertDataNodesForZone(defaultZoneId, Collections.emptySet(),
keyValueStorage);
}
}
diff --git
a/modules/distribution-zones/src/testFixtures/java/org/apache/ignite/internal/distributionzones/DistributionZonesTestUtil.java
b/modules/distribution-zones/src/testFixtures/java/org/apache/ignite/internal/distributionzones/DistributionZonesTestUtil.java
index e766b7d6fa..e70a2da287 100644
---
a/modules/distribution-zones/src/testFixtures/java/org/apache/ignite/internal/distributionzones/DistributionZonesTestUtil.java
+++
b/modules/distribution-zones/src/testFixtures/java/org/apache/ignite/internal/distributionzones/DistributionZonesTestUtil.java
@@ -23,7 +23,6 @@ import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleDownChangeTriggerKey;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpChangeTriggerKey;
-import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesChangeTriggerKey;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesGlobalStateRevision;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyKey;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesLogicalTopologyVault;
@@ -277,28 +276,6 @@ public class DistributionZonesTestUtil {
);
}
- /**
- * Asserts {@link DistributionZonesUtil#zonesChangeTriggerKey(int)}
revision.
- *
- * @param revision Revision.
- * @param zoneId Zone id.
- * @param keyValueStorage Key-value storage.
- * @throws InterruptedException If thread was interrupted.
- */
- public static void assertZonesChangeTriggerKey(
- long revision,
- int zoneId,
- KeyValueStorage keyValueStorage
- ) throws InterruptedException {
- assertValueInStorage(
- keyValueStorage,
- zonesChangeTriggerKey(zoneId).bytes(),
- ByteUtils::bytesToLong,
- revision,
- 1000
- );
- }
-
/**
* Asserts {@link DistributionZonesUtil#zonesLogicalTopologyKey()} value.
*
diff --git
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/distribution/zones/ItIgniteDistributionZoneManagerNodeRestartTest.java
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/distribution/zones/ItIgniteDistributionZoneManagerNodeRestartTest.java
index c63edadf05..8534e955d7 100644
---
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/distribution/zones/ItIgniteDistributionZoneManagerNodeRestartTest.java
+++
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/distribution/zones/ItIgniteDistributionZoneManagerNodeRestartTest.java
@@ -25,19 +25,25 @@ import static
org.apache.ignite.internal.catalog.commands.CatalogUtils.IMMEDIATE
import static
org.apache.ignite.internal.catalog.commands.CatalogUtils.INFINITE_TIMER_VALUE;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertDataNodesFromManager;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.assertValueInStorage;
+import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.dataNodes;
+import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.toDataNodesMap;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneDataNodesKey;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zoneScaleUpChangeTriggerKey;
import static
org.apache.ignite.internal.distributionzones.DistributionZonesUtil.zonesGlobalStateRevision;
import static
org.apache.ignite.internal.testframework.IgniteTestUtils.testNodeName;
import static
org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition;
+import static
org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willBe;
import static org.apache.ignite.internal.util.ByteUtils.bytesToLong;
import static org.apache.ignite.internal.util.ByteUtils.fromBytes;
+import static org.apache.ignite.internal.util.ByteUtils.toBytes;
import static org.apache.ignite.internal.util.IgniteUtils.startsWith;
import static
org.apache.ignite.utils.ClusterServiceTestUtils.defaultSerializationRegistry;
+import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
@@ -47,8 +53,10 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.function.Consumer;
@@ -80,7 +88,9 @@ import
org.apache.ignite.internal.distributionzones.DistributionZonesUtil;
import org.apache.ignite.internal.distributionzones.Node;
import org.apache.ignite.internal.distributionzones.NodeWithAttributes;
import org.apache.ignite.internal.hlc.HybridClockImpl;
+import org.apache.ignite.internal.lang.ByteArray;
import org.apache.ignite.internal.manager.IgniteComponent;
+import org.apache.ignite.internal.metastorage.Entry;
import org.apache.ignite.internal.metastorage.MetaStorageManager;
import
org.apache.ignite.internal.metastorage.impl.StandaloneMetaStorageManager;
import org.apache.ignite.internal.metastorage.server.If;
@@ -346,6 +356,70 @@ public class
ItIgniteDistributionZoneManagerNodeRestartTest extends BaseIgniteRe
assertEquals(logicalTopology,
distributionZoneManager.logicalTopology());
}
+ @Test
+ public void testCreationZoneWhenDataNodesAreDeletedIsNotSuccessful()
throws Exception {
+ PartialNode node = startPartialNode(0);
+
+ node.logicalTopology().putNode(A);
+ node.logicalTopology().putNode(B);
+
+ node.logicalTopology().putNode(C);
+
+ Set<NodeWithAttributes> logicalTopology = Stream.of(A, B, C)
+ .map(n -> new NodeWithAttributes(n.name(), n.id(),
n.attributes()))
+ .collect(toSet());
+
+ DistributionZoneManager distributionZoneManager =
getDistributionZoneManager(node);
+ DistributionZoneManager finalDistributionZoneManager =
distributionZoneManager;
+
+ assertTrue(waitForCondition(() ->
logicalTopology.equals(finalDistributionZoneManager.logicalTopology()),
TIMEOUT_MILLIS));
+
+ int zoneId = getZoneId(node, DEFAULT_ZONE_NAME);
+
+ assertValueInStorage(
+ metastore,
+ zoneDataNodesKey(zoneId),
+ (v) ->
dataNodes(fromBytes(v)).stream().map(Node::nodeName).collect(toSet()),
+ Set.of(A.name(), B.name(), C.name()),
+ TIMEOUT_MILLIS
+ );
+
+ metastore = findComponent(node.startedComponents(),
MetaStorageManager.class);
+
+ byte[][] dataNodeKey = new byte[1][1];
+
+ // In this mock we catch invocation of
DistributionZoneManager.initDataNodesAndTriggerKeysInMetaStorage, where
condition is based
+ // on presence of data node key in ms. After that we make this data
node as a tombstone, so when logic of creation of a zone is
+ // run, there won't be any initialisation of data nodes keys. We try
to imitate concurrent removal of a zone.
+ doAnswer(invocation -> {
+ ByteArray dataNodeKeyForZone = new ByteArray(dataNodeKey[0]);
+
+ // Here we remove data nodes value for newly created zone, so it
is tombstone
+ metastore.put(dataNodeKeyForZone,
toBytes(toDataNodesMap(emptySet()))).get();
+
+ metastore.remove(dataNodeKeyForZone).get();
+
+ return invocation.callRealMethod();
+ }).when(metastore).invoke(argThat(iif -> {
+ If iif1 = MetaStorageWriteHandler.toIf(iif);
+
+ byte[][] keysFromIf = iif1.cond().keys();
+
+ Optional<byte[]> dataNodeKeyOptional = Arrays.stream(keysFromIf)
+ .filter(op -> startsWith(op, zoneDataNodesKey().bytes()))
+ .findFirst();
+
+ dataNodeKeyOptional.ifPresent(bytes -> dataNodeKey[0] = bytes);
+
+ return dataNodeKeyOptional.isPresent();
+ }));
+
+ createZone(node, "zone1", INFINITE_TIMER_VALUE, INFINITE_TIMER_VALUE);
+
+ // Assert that after creation of a zone, data nodes are still
tombstone, but not the logical topology, as for default zone.
+ assertThat(metastore.get(new
ByteArray(dataNodeKey[0])).thenApply(Entry::tombstone), willBe(true));
+ }
+
@Test
@Disabled("https://issues.apache.org/jira/browse/IGNITE-20559";)
public void testGlobalStateRevisionUpdatedCorrectly() throws Exception {
