narendly commented on a change in pull request #632: Asynchronously calculating
the Baseline
URL: https://github.com/apache/helix/pull/632#discussion_r360574832
##########
File path:
helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestMixedModeAutoRebalance.java
##########
@@ -111,112 +109,125 @@ public void beforeClass() throws Exception {
};
}
- protected void createResource(String stateModel, int numPartition, int
replica,
+ protected void createResource(String dbName, String stateModel, int
numPartition, int replica,
boolean delayEnabled, String rebalanceStrategy) {
if (delayEnabled) {
- createResourceWithDelayedRebalance(CLUSTER_NAME, DB_NAME, stateModel,
numPartition, replica,
+ createResourceWithDelayedRebalance(CLUSTER_NAME, dbName, stateModel,
numPartition, replica,
replica - 1, 200, rebalanceStrategy);
} else {
- createResourceWithDelayedRebalance(CLUSTER_NAME, DB_NAME, stateModel,
numPartition, replica,
+ createResourceWithDelayedRebalance(CLUSTER_NAME, dbName, stateModel,
numPartition, replica,
replica, 0, rebalanceStrategy);
}
}
@Test(dataProvider = "stateModels")
public void testUserDefinedPreferenceListsInFullAuto(String stateModel,
boolean delayEnabled,
String rebalanceStrateyName) throws Exception {
- createResource(stateModel, _PARTITIONS, _replica, delayEnabled,
+ String dbName = "Test-DB-" + stateModel;
+ createResource(dbName, stateModel, _PARTITIONS, _replica, delayEnabled,
rebalanceStrateyName);
- IdealState idealState =
-
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
DB_NAME);
- Map<String, List<String>> userDefinedPreferenceLists =
idealState.getPreferenceLists();
- List<String> userDefinedPartitions = new ArrayList<>();
- for (String partition : userDefinedPreferenceLists.keySet()) {
- List<String> preferenceList = new ArrayList<>();
- for (int k = _replica; k >= 0; k--) {
- String instance = _participants.get(k).getInstanceName();
- preferenceList.add(instance);
+ try {
+ IdealState idealState =
+
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
dbName);
+ Map<String, List<String>> userDefinedPreferenceLists =
idealState.getPreferenceLists();
+ List<String> userDefinedPartitions = new ArrayList<>();
+ for (String partition : userDefinedPreferenceLists.keySet()) {
+ List<String> preferenceList = new ArrayList<>();
+ for (int k = _replica; k >= 0; k--) {
+ String instance = _participants.get(k).getInstanceName();
+ preferenceList.add(instance);
+ }
+ userDefinedPreferenceLists.put(partition, preferenceList);
+ userDefinedPartitions.add(partition);
}
- userDefinedPreferenceLists.put(partition, preferenceList);
- userDefinedPartitions.add(partition);
- }
-
- ResourceConfig resourceConfig =
- new
ResourceConfig.Builder(DB_NAME).setPreferenceLists(userDefinedPreferenceLists).build();
- _configAccessor.setResourceConfig(CLUSTER_NAME, DB_NAME, resourceConfig);
- // TODO remove this sleep after fix
https://github.com/apache/helix/issues/526
- Thread.sleep(500);
+ ResourceConfig resourceConfig =
+ new
ResourceConfig.Builder(dbName).setPreferenceLists(userDefinedPreferenceLists).build();
+ _configAccessor.setResourceConfig(CLUSTER_NAME, dbName, resourceConfig);
- Assert.assertTrue(_clusterVerifier.verify(3000));
- verifyUserDefinedPreferenceLists(DB_NAME, userDefinedPreferenceLists,
userDefinedPartitions);
+ // TODO remove this sleep after fix
https://github.com/apache/helix/issues/526
+ Thread.sleep(500);
- while (userDefinedPartitions.size() > 0) {
- IdealState originIS =
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
- DB_NAME);
- Set<String> nonUserDefinedPartitions = new
HashSet<>(originIS.getPartitionSet());
- nonUserDefinedPartitions.removeAll(userDefinedPartitions);
-
- removePartitionFromUserDefinedList(DB_NAME, userDefinedPartitions);
- // TODO: Remove wait once we enable the BestPossibleExternalViewVerifier
for the WAGED rebalancer.
- Thread.sleep(1000);
Assert.assertTrue(_clusterVerifier.verify(3000));
- verifyUserDefinedPreferenceLists(DB_NAME, userDefinedPreferenceLists,
userDefinedPartitions);
- verifyNonUserDefinedAssignment(DB_NAME, originIS,
nonUserDefinedPartitions);
+ verifyUserDefinedPreferenceLists(dbName, userDefinedPreferenceLists,
+ userDefinedPartitions);
+
+ while (userDefinedPartitions.size() > 0) {
+ IdealState originIS =
+
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
dbName);
+ Set<String> nonUserDefinedPartitions = new
HashSet<>(originIS.getPartitionSet());
+ nonUserDefinedPartitions.removeAll(userDefinedPartitions);
+
+ removePartitionFromUserDefinedList(dbName, userDefinedPartitions);
+ // TODO: Remove wait once we enable the
BestPossibleExternalViewVerifier for the WAGED rebalancer.
+ Thread.sleep(1000);
+ Assert.assertTrue(_clusterVerifier.verify(3000));
+ verifyUserDefinedPreferenceLists(dbName, userDefinedPreferenceLists,
+ userDefinedPartitions);
+ verifyNonUserDefinedAssignment(dbName, originIS,
nonUserDefinedPartitions);
+ }
+ } finally {
+ _gSetupTool.getClusterManagementTool().dropResource(CLUSTER_NAME,
dbName);
+ _clusterVerifier.verify(5000);
}
}
@Test
public void testUserDefinedPreferenceListsInFullAutoWithErrors() throws
Exception {
- createResource(BuiltInStateModelDefinitions.MasterSlave.name(), 5,
_replica,
- false, CrushRebalanceStrategy.class.getName());
-
- IdealState idealState =
-
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
DB_NAME);
- Map<String, List<String>> userDefinedPreferenceLists =
idealState.getPreferenceLists();
-
- List<String> newNodes = new ArrayList<>();
- for (int i = NUM_NODE; i < NUM_NODE + _replica; i++) {
- String instance = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
- _gSetupTool.addInstanceToCluster(CLUSTER_NAME, instance);
-
- // start dummy participants
- MockParticipantManager participant =
- new TestMockParticipantManager(ZK_ADDR, CLUSTER_NAME, instance);
- participant.syncStart();
- _participants.add(participant);
- newNodes.add(instance);
- }
-
- List<String> userDefinedPartitions = new ArrayList<>();
- for (String partition : userDefinedPreferenceLists.keySet()) {
- userDefinedPreferenceLists.put(partition, newNodes);
- userDefinedPartitions.add(partition);
- }
+ String dbName = "Test-DB-withErrors";
+ createResource(dbName, BuiltInStateModelDefinitions.MasterSlave.name(), 5,
_replica, false,
+ CrushRebalanceStrategy.class.getName());
+ try {
+ IdealState idealState =
+
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
dbName);
+ Map<String, List<String>> userDefinedPreferenceLists =
idealState.getPreferenceLists();
+
+ List<String> newNodes = new ArrayList<>();
+ for (int i = NUM_NODE; i < NUM_NODE + _replica; i++) {
+ String instance = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
+ _gSetupTool.addInstanceToCluster(CLUSTER_NAME, instance);
+
+ // start dummy participants
+ MockParticipantManager participant =
+ new TestMockParticipantManager(ZK_ADDR, CLUSTER_NAME, instance);
+ participant.syncStart();
+ _participants.add(participant);
+ newNodes.add(instance);
+ }
- ResourceConfig resourceConfig =
- new
ResourceConfig.Builder(DB_NAME).setPreferenceLists(userDefinedPreferenceLists).build();
- _configAccessor.setResourceConfig(CLUSTER_NAME, DB_NAME, resourceConfig);
+ List<String> userDefinedPartitions = new ArrayList<>();
+ for (String partition : userDefinedPreferenceLists.keySet()) {
+ userDefinedPreferenceLists.put(partition, newNodes);
+ userDefinedPartitions.add(partition);
+ }
- TestHelper.verify(() -> {
- ExternalView ev =
-
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME,
DB_NAME);
- if (ev != null) {
- for (String partition : ev.getPartitionSet()) {
- Map<String, String> stateMap = ev.getStateMap(partition);
- if (stateMap.values().contains("ERROR")) {
- return true;
+ ResourceConfig resourceConfig =
+ new
ResourceConfig.Builder(dbName).setPreferenceLists(userDefinedPreferenceLists).build();
+ _configAccessor.setResourceConfig(CLUSTER_NAME, dbName, resourceConfig);
+
+ TestHelper.verify(() -> {
+ ExternalView ev =
+
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME,
dbName);
+ if (ev != null) {
+ for (String partition : ev.getPartitionSet()) {
+ Map<String, String> stateMap = ev.getStateMap(partition);
+ if (stateMap.values().contains("ERROR")) {
+ return true;
+ }
}
}
- }
- return false;
- }, 2000);
-
- ExternalView ev =
-
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME,
DB_NAME);
- IdealState is =
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
- DB_NAME);
- validateMinActiveAndTopStateReplica(is, ev, _replica, NUM_NODE);
+ return false;
+ }, 2000);
+
+ ExternalView ev =
+
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME,
dbName);
+ IdealState is =
+
_gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME,
dbName);
+ validateMinActiveAndTopStateReplica(is, ev, _replica, NUM_NODE);
+ } finally {
+ _gSetupTool.getClusterManagementTool().dropResource(CLUSTER_NAME,
dbName);
+ _clusterVerifier.verify(5000);
Review comment:
Okay. As for 2, I actually misread and thought you were doing a
dropCluster(). verify makes sense in this case. Since this is a test, I'm fine
with keeping 5000 for now, but it would be better to create a timeout constant
either inside the verifier or in this class since I do see a lot of magic
numbers here.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
