This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
The following commit(s) were added to refs/heads/master by this push:
new ce33735dd [apache/helix] -- Fix PreferenceList Ordering Changes during
Maintenance Mode (#2778)
ce33735dd is described below
commit ce33735dda1dd8614cb8d84094119de1a4b30484
Author: Himanshu Kandwal <[email protected]>
AuthorDate: Tue Mar 19 15:49:55 2024 -0700
[apache/helix] -- Fix PreferenceList Ordering Changes during Maintenance
Mode (#2778)
Fixed PreferenceList Ordering Changes (indeterminism) during Maintenance
Mode.
---
.../rebalancer/MaintenanceRebalancer.java | 31 ++++++-
.../rebalancer/TestMaintenanceRebalancer.java | 60 +++++++++++++
...MaintenanceRebalancer.ComputeNewIdealState.json | 98 ++++++++++++++++++++++
3 files changed, 187 insertions(+), 2 deletions(-)
diff --git
a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java
index 9f04de535..9b5b4f929 100644
---
a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java
+++
b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java
@@ -28,6 +28,7 @@ import
org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
import org.apache.helix.controller.stages.CurrentStateOutput;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.Partition;
+import org.apache.helix.model.StateModelDefinition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,12 +56,38 @@ public class MaintenanceRebalancer extends
SemiAutoRebalancer<ResourceController
// One principal is to prohibit DROP -> OFFLINE and OFFLINE -> DROP state
transitions.
// Derived preference list from current state with state priority
+ StateModelDefinition stateModelDef =
clusterData.getStateModelDef(currentIdealState.getStateModelDefRef());
+
for (Partition partition : currentStateMap.keySet()) {
Map<String, String> stateMap = currentStateMap.get(partition);
List<String> preferenceList = new ArrayList<>(stateMap.keySet());
+
+ /**
+ * This sorting preserves the ordering of current state hosts in the
order of current IS pref list
+ * Example:
+ * ideal state pref-list: [A, B, C]
+ * current-state: {
+ * A: FOLLOWER,
+ * B: LEADER,
+ * C: FOLLOWER
+ * }
+ * Lets say newPrefList = new ArrayList<>(current-state.keySet()) => [C,
B, A]
+ *
+ * Sort 1: Sort based on preference-list order:
+ * --------------------------------------------------------
+ * newPrefList = [C, B, A] => [A, B, C]
+ */
Collections.sort(preferenceList, new
PreferenceListNodeComparator(stateMap,
-
clusterData.getStateModelDef(currentIdealState.getStateModelDefRef()),
- Collections.<String>emptyList()));
+ stateModelDef,
currentIdealState.getPreferenceList(partition.getPartitionName())));
+
+ /**
+ * Sort 2: Sort based on state-priority order:
+ * --------------------------------------------------------
+ * newPrefList = [A, B, C] => [B, A, C]
+ * Here, A will be 2nd and C will be third always as both have same
priority so original (pref-list) order will be maintained.
+ */
+ preferenceList.sort(new StatePriorityComparator(stateMap,
stateModelDef));
+
currentIdealState.setPreferenceList(partition.getPartitionName(),
preferenceList);
}
LOG.info(String
diff --git
a/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestMaintenanceRebalancer.java
b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestMaintenanceRebalancer.java
new file mode 100644
index 000000000..50cdd418f
--- /dev/null
+++
b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestMaintenanceRebalancer.java
@@ -0,0 +1,60 @@
+package org.apache.helix.controller.rebalancer;
+
+import java.util.List;
+import java.util.Map;
+
+import
org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
+import org.apache.helix.controller.stages.CurrentStateOutput;
+import org.apache.helix.model.IdealState;
+import org.apache.helix.model.MasterSlaveSMD;
+import org.apache.helix.model.Partition;
+import org.apache.helix.util.TestInputLoader;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestMaintenanceRebalancer {
+
+ private static final String RESOURCE_NAME = "testResource";
+ private static final String PARTITION_NAME = "testResourcePartition";
+
+ @Test(dataProvider = "TestComputeIdealStateInput")
+ public void testComputeIdealState(String comment, String stateModelName,
List<String> liveInstances,
+ List<String> preferenceList, Map<String, String> currentStateMap,
List<String> expectedPrefList) {
+ System.out.println("Test case comment: " + comment);
+ MaintenanceRebalancer rebalancer = new MaintenanceRebalancer();
+
+ Partition partition = new Partition(PARTITION_NAME);
+ CurrentStateOutput currentStateOutput = new CurrentStateOutput();
+ for (String instance : currentStateMap.keySet()) {
+ currentStateOutput.setCurrentState(RESOURCE_NAME, partition, instance,
currentStateMap.get(instance));
+ }
+
+ IdealState currentIdealState = new IdealState(RESOURCE_NAME);
+ currentIdealState.setRebalanceMode(IdealState.RebalanceMode.FULL_AUTO);
+
currentIdealState.setRebalancerClassName("org.apache.helix.controller.rebalancer.waged.WagedRebalancer");
+ currentIdealState.setStateModelDefRef(stateModelName);
+ currentIdealState.setPreferenceList(PARTITION_NAME, preferenceList);
+
+ ResourceControllerDataProvider dataCache =
mock(ResourceControllerDataProvider.class);
+
when(dataCache.getStateModelDef("MasterSlave")).thenReturn(MasterSlaveSMD.build());
+
+ IdealState updatedIdealState = rebalancer
+ .computeNewIdealState(RESOURCE_NAME, currentIdealState,
currentStateOutput, dataCache);
+
+ List<String> partitionPrefList =
updatedIdealState.getPreferenceList(PARTITION_NAME);
+ Assert.assertTrue(partitionPrefList.equals(expectedPrefList));
+ }
+
+ @DataProvider(name = "TestComputeIdealStateInput")
+ public Object[][] loadTestComputeIdealStateInput() {
+ final String[] params = {
+ "comment", "stateModel", "liveInstances", "preferenceList",
"currentStateMap", "expectedPreferenceList"
+ };
+ return
TestInputLoader.loadTestInputs("MaintenanceRebalancer.ComputeNewIdealState.json",
params);
+ }
+
+}
diff --git
a/helix-core/src/test/resources/MaintenanceRebalancer.ComputeNewIdealState.json
b/helix-core/src/test/resources/MaintenanceRebalancer.ComputeNewIdealState.json
new file mode 100644
index 000000000..9212b90bd
--- /dev/null
+++
b/helix-core/src/test/resources/MaintenanceRebalancer.ComputeNewIdealState.json
@@ -0,0 +1,98 @@
+[
+ {
+ "comment": "[Case 1] Same state - No (A, B, C) -> (A, C, B) for different
hash order",
+ "stateModel": "MasterSlave",
+ "liveInstances": [],
+ "preferenceList": [
+ "lKmBBEPisM", "8PNpMU7EgT", "HOs3rOFCad"
+ ],
+ "currentStateMap": {
+ "lKmBBEPisM" : "MASTER",
+ "8PNpMU7EgT" : "SLAVE",
+ "HOs3rOFCad" : "SLAVE"
+ },
+ "expectedPreferenceList": [
+ "lKmBBEPisM", "8PNpMU7EgT", "HOs3rOFCad"
+ ]
+ },
+ {
+ "comment": "[Case 2] Same state - No (A, B, C) -> (A, C, B) for different
hash order",
+ "stateModel": "MasterSlave",
+ "liveInstances": [],
+ "preferenceList": [
+ "Q3ZZuCeBpu", "QY9U91XBYo", "GMYBW20gmC"
+ ],
+ "currentStateMap": {
+ "Q3ZZuCeBpu" : "MASTER",
+ "QY9U91XBYo" : "SLAVE",
+ "GMYBW20gmC" : "SLAVE"
+ },
+ "expectedPreferenceList": [
+ "Q3ZZuCeBpu", "QY9U91XBYo", "GMYBW20gmC"
+ ]
+ },
+ {
+ "comment": "[Case 3] second becomes leader, (A, B, C) -> (B, A, C) for
different hash order",
+ "stateModel": "MasterSlave",
+ "liveInstances": [],
+ "preferenceList": [
+ "lKmBBEPisM", "8PNpMU7EgT", "HOs3rOFCad"
+ ],
+ "currentStateMap": {
+ "lKmBBEPisM" : "SLAVE",
+ "8PNpMU7EgT" : "MASTER",
+ "HOs3rOFCad" : "SLAVE"
+ },
+ "expectedPreferenceList": [
+ "8PNpMU7EgT", "lKmBBEPisM", "HOs3rOFCad"
+ ]
+ },
+ {
+ "comment": "[Case 4] second becomes leader, (A, B, C) -> (B, A, C) for
different hash order",
+ "stateModel": "MasterSlave",
+ "liveInstances": [],
+ "preferenceList": [
+ "Q3ZZuCeBpu", "QY9U91XBYo", "GMYBW20gmC"
+ ],
+ "currentStateMap": {
+ "Q3ZZuCeBpu" : "SLAVE",
+ "QY9U91XBYo" : "MASTER",
+ "GMYBW20gmC" : "SLAVE"
+ },
+ "expectedPreferenceList": [
+ "QY9U91XBYo", "Q3ZZuCeBpu", "GMYBW20gmC"
+ ]
+ },
+ {
+ "comment": "[Case 5] leader becomes offline, (A, B, C) -> (B, C, A) for
different hash order",
+ "stateModel": "MasterSlave",
+ "liveInstances": [],
+ "preferenceList": [
+ "lKmBBEPisM", "8PNpMU7EgT", "HOs3rOFCad"
+ ],
+ "currentStateMap": {
+ "lKmBBEPisM" : "OFFLINE",
+ "8PNpMU7EgT" : "MASTER",
+ "HOs3rOFCad" : "SLAVE"
+ },
+ "expectedPreferenceList": [
+ "8PNpMU7EgT", "HOs3rOFCad", "lKmBBEPisM"
+ ]
+ },
+ {
+ "comment": "[Case 6] leader becomes offline, (A, B, C) -> (B, C, A) for
different hash order",
+ "stateModel": "MasterSlave",
+ "liveInstances": [],
+ "preferenceList": [
+ "Q3ZZuCeBpu", "QY9U91XBYo", "GMYBW20gmC"
+ ],
+ "currentStateMap": {
+ "Q3ZZuCeBpu" : "OFFLINE",
+ "QY9U91XBYo" : "MASTER",
+ "GMYBW20gmC" : "SLAVE"
+ },
+ "expectedPreferenceList": [
+ "QY9U91XBYo", "GMYBW20gmC", "Q3ZZuCeBpu"
+ ]
+ }
+]
