jiajunwang commented on a change in pull request #632: Asynchronously
calculating the Baseline
URL: https://github.com/apache/helix/pull/632#discussion_r360542385
##########
File path:
helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestMixedModeAutoRebalance.java
##########
@@ -111,112 +109,125 @@ public void beforeClass() throws Exception {
};
}
- protected void createResource(String stateModel, int numPartition, int
replica,
+ protected void createResource(String dbName, String stateModel, int
numPartition, int replica,
boolean delayEnabled, String rebalanceStrategy) {
if (delayEnabled) {
- createResourceWithDelayedRebalance(CLUSTER_NAME, DB_NAME, stateModel,
numPartition, replica,
+ createResourceWithDelayedRebalance(CLUSTER_NAME, dbName, stateModel,
numPartition, replica,
replica - 1, 200, rebalanceStrategy);
} else {
- createResourceWithDelayedRebalance(CLUSTER_NAME, DB_NAME, stateModel,
numPartition, replica,
+ createResourceWithDelayedRebalance(CLUSTER_NAME, dbName, stateModel,
numPartition, replica,
replica, 0, rebalanceStrategy);
}
}
@Test(dataProvider = "stateModels")
public void testUserDefinedPreferenceListsInFullAuto(String stateModel,
boolean delayEnabled,
String rebalanceStrateyName) throws Exception {
- createResource(stateModel, _PARTITIONS, _replica, delayEnabled,
+ String dbName = "Test-DB-" + stateModel;
+ createResource(dbName, stateModel, _PARTITIONS, _replica, delayEnabled,
rebalanceStrateyName);
- IdealState idealState =
-
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
DB_NAME);
- Map<String, List<String>> userDefinedPreferenceLists =
idealState.getPreferenceLists();
- List<String> userDefinedPartitions = new ArrayList<>();
- for (String partition : userDefinedPreferenceLists.keySet()) {
- List<String> preferenceList = new ArrayList<>();
- for (int k = _replica; k >= 0; k--) {
- String instance = _participants.get(k).getInstanceName();
- preferenceList.add(instance);
+ try {
+ IdealState idealState =
+
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
dbName);
+ Map<String, List<String>> userDefinedPreferenceLists =
idealState.getPreferenceLists();
+ List<String> userDefinedPartitions = new ArrayList<>();
+ for (String partition : userDefinedPreferenceLists.keySet()) {
+ List<String> preferenceList = new ArrayList<>();
+ for (int k = _replica; k >= 0; k--) {
+ String instance = _participants.get(k).getInstanceName();
+ preferenceList.add(instance);
+ }
+ userDefinedPreferenceLists.put(partition, preferenceList);
+ userDefinedPartitions.add(partition);
}
- userDefinedPreferenceLists.put(partition, preferenceList);
- userDefinedPartitions.add(partition);
- }
-
- ResourceConfig resourceConfig =
- new
ResourceConfig.Builder(DB_NAME).setPreferenceLists(userDefinedPreferenceLists).build();
- _configAccessor.setResourceConfig(CLUSTER_NAME, DB_NAME, resourceConfig);
- // TODO remove this sleep after fix
https://github.com/apache/helix/issues/526
- Thread.sleep(500);
+ ResourceConfig resourceConfig =
+ new
ResourceConfig.Builder(dbName).setPreferenceLists(userDefinedPreferenceLists).build();
+ _configAccessor.setResourceConfig(CLUSTER_NAME, dbName, resourceConfig);
- Assert.assertTrue(_clusterVerifier.verify(3000));
- verifyUserDefinedPreferenceLists(DB_NAME, userDefinedPreferenceLists,
userDefinedPartitions);
+ // TODO remove this sleep after fix
https://github.com/apache/helix/issues/526
+ Thread.sleep(500);
- while (userDefinedPartitions.size() > 0) {
- IdealState originIS =
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
- DB_NAME);
- Set<String> nonUserDefinedPartitions = new
HashSet<>(originIS.getPartitionSet());
- nonUserDefinedPartitions.removeAll(userDefinedPartitions);
-
- removePartitionFromUserDefinedList(DB_NAME, userDefinedPartitions);
- // TODO: Remove wait once we enable the BestPossibleExternalViewVerifier
for the WAGED rebalancer.
- Thread.sleep(1000);
Assert.assertTrue(_clusterVerifier.verify(3000));
- verifyUserDefinedPreferenceLists(DB_NAME, userDefinedPreferenceLists,
userDefinedPartitions);
- verifyNonUserDefinedAssignment(DB_NAME, originIS,
nonUserDefinedPartitions);
+ verifyUserDefinedPreferenceLists(dbName, userDefinedPreferenceLists,
+ userDefinedPartitions);
+
+ while (userDefinedPartitions.size() > 0) {
+ IdealState originIS =
+
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
dbName);
+ Set<String> nonUserDefinedPartitions = new
HashSet<>(originIS.getPartitionSet());
+ nonUserDefinedPartitions.removeAll(userDefinedPartitions);
+
+ removePartitionFromUserDefinedList(dbName, userDefinedPartitions);
+ // TODO: Remove wait once we enable the
BestPossibleExternalViewVerifier for the WAGED rebalancer.
+ Thread.sleep(1000);
+ Assert.assertTrue(_clusterVerifier.verify(3000));
+ verifyUserDefinedPreferenceLists(dbName, userDefinedPreferenceLists,
+ userDefinedPartitions);
+ verifyNonUserDefinedAssignment(dbName, originIS,
nonUserDefinedPartitions);
+ }
+ } finally {
+ _gSetupTool.getClusterManagementTool().dropResource(CLUSTER_NAME,
dbName);
+ _clusterVerifier.verify(5000);
}
}
@Test
public void testUserDefinedPreferenceListsInFullAutoWithErrors() throws
Exception {
- createResource(BuiltInStateModelDefinitions.MasterSlave.name(), 5,
_replica,
- false, CrushRebalanceStrategy.class.getName());
-
- IdealState idealState =
-
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
DB_NAME);
- Map<String, List<String>> userDefinedPreferenceLists =
idealState.getPreferenceLists();
-
- List<String> newNodes = new ArrayList<>();
- for (int i = NUM_NODE; i < NUM_NODE + _replica; i++) {
- String instance = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
- _gSetupTool.addInstanceToCluster(CLUSTER_NAME, instance);
-
- // start dummy participants
- MockParticipantManager participant =
- new TestMockParticipantManager(ZK_ADDR, CLUSTER_NAME, instance);
- participant.syncStart();
- _participants.add(participant);
- newNodes.add(instance);
- }
-
- List<String> userDefinedPartitions = new ArrayList<>();
- for (String partition : userDefinedPreferenceLists.keySet()) {
- userDefinedPreferenceLists.put(partition, newNodes);
- userDefinedPartitions.add(partition);
- }
+ String dbName = "Test-DB-withErrors";
+ createResource(dbName, BuiltInStateModelDefinitions.MasterSlave.name(), 5,
_replica, false,
+ CrushRebalanceStrategy.class.getName());
+ try {
+ IdealState idealState =
+
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
dbName);
+ Map<String, List<String>> userDefinedPreferenceLists =
idealState.getPreferenceLists();
+
+ List<String> newNodes = new ArrayList<>();
+ for (int i = NUM_NODE; i < NUM_NODE + _replica; i++) {
+ String instance = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
+ _gSetupTool.addInstanceToCluster(CLUSTER_NAME, instance);
+
+ // start dummy participants
+ MockParticipantManager participant =
+ new TestMockParticipantManager(ZK_ADDR, CLUSTER_NAME, instance);
+ participant.syncStart();
+ _participants.add(participant);
+ newNodes.add(instance);
+ }
- ResourceConfig resourceConfig =
- new
ResourceConfig.Builder(DB_NAME).setPreferenceLists(userDefinedPreferenceLists).build();
- _configAccessor.setResourceConfig(CLUSTER_NAME, DB_NAME, resourceConfig);
+ List<String> userDefinedPartitions = new ArrayList<>();
+ for (String partition : userDefinedPreferenceLists.keySet()) {
+ userDefinedPreferenceLists.put(partition, newNodes);
+ userDefinedPartitions.add(partition);
+ }
- TestHelper.verify(() -> {
- ExternalView ev =
-
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME,
DB_NAME);
- if (ev != null) {
- for (String partition : ev.getPartitionSet()) {
- Map<String, String> stateMap = ev.getStateMap(partition);
- if (stateMap.values().contains("ERROR")) {
- return true;
+ ResourceConfig resourceConfig =
+ new
ResourceConfig.Builder(dbName).setPreferenceLists(userDefinedPreferenceLists).build();
+ _configAccessor.setResourceConfig(CLUSTER_NAME, dbName, resourceConfig);
+
+ TestHelper.verify(() -> {
+ ExternalView ev =
+
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME,
dbName);
+ if (ev != null) {
+ for (String partition : ev.getPartitionSet()) {
+ Map<String, String> stateMap = ev.getStateMap(partition);
+ if (stateMap.values().contains("ERROR")) {
+ return true;
+ }
}
}
- }
- return false;
- }, 2000);
-
- ExternalView ev =
-
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME,
DB_NAME);
- IdealState is =
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
- DB_NAME);
- validateMinActiveAndTopStateReplica(is, ev, _replica, NUM_NODE);
+ return false;
+ }, 2000);
+
+ ExternalView ev =
+
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME,
dbName);
+ IdealState is =
+
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
dbName);
+ validateMinActiveAndTopStateReplica(is, ev, _replica, NUM_NODE);
+ } finally {
+ _gSetupTool.getClusterManagementTool().dropResource(CLUSTER_NAME,
dbName);
+ _clusterVerifier.verify(5000);
Review comment:
1. I agree, but I didn't find a good place to put it. If in this file, we
only have 2 references. If move it somewhere else, then different test case
might have a different requirement. So I'd prefer to keep the test case easier
for now.
2. It just needs to ensure the resource is gone.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
