Fix conflicts and make test passed
Project: http://git-wip-us.apache.org/repos/asf/helix/repo Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/fa587f0b Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/fa587f0b Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/fa587f0b Branch: refs/heads/master Commit: fa587f0bfcb229f773eda764c8d2e3a27c814c96 Parents: fe0b6e8 Author: Junkai Xue <j...@linkedin.com> Authored: Tue Nov 7 17:05:09 2017 -0800 Committer: Junkai Xue <j...@linkedin.com> Committed: Tue Nov 7 18:17:12 2017 -0800 ---------------------------------------------------------------------- .../webapp/TestHelixAdminScenariosRest.java | 6 +- .../apache/helix/webapp/TestResetInstance.java | 2 +- .../helix/webapp/TestResetPartitionState.java | 2 +- .../apache/helix/webapp/TestResetResource.java | 4 +- .../webapp/resources/TestJobQueuesResource.java | 2 +- .../org/apache/helix/agent/TestHelixAgent.java | 4 +- .../java/org/apache/helix/HelixProperty.java | 9 - .../main/java/org/apache/helix/PropertyKey.java | 5 - .../helix/manager/zk/CallbackHandler.java | 70 +- .../ClusterExternalViewVerifier.java | 175 ----- .../ClusterVerifiers/ClusterStateVerifier.java | 739 ------------------- .../tools/ClusterVerifiers/ClusterVerifier.java | 148 ---- .../helix/integration/TestAddClusterV2.java | 2 +- .../TestAddStateModelFactoryAfterConnect.java | 4 +- .../TestAutoRebalanceWithDisabledInstance.java | 114 --- .../helix/integration/TestBasicSpectator.java | 2 +- .../integration/TestCleanupExternalView.java | 2 +- .../integration/TestControllerHistory.java | 59 -- .../TestCorrectnessOnConnectivityLoss.java | 4 +- .../integration/TestCrushAutoRebalance.java | 222 ------ .../TestCrushAutoRebalanceNonRack.java | 216 ------ .../integration/TestDelayedAutoRebalance.java | 452 ------------ .../TestDelayedAutoRebalanceWithRackaware.java | 73 -- .../TestDisableCustomCodeRunner.java | 2 +- .../integration/TestDisableExternalView.java | 2 +- .../helix/integration/TestDisableNode.java | 2 +- .../helix/integration/TestDisablePartition.java | 2 +- .../helix/integration/TestDisableResource.java | 2 +- .../integration/TestDistributedCMMain.java | 4 +- .../TestDistributedClusterController.java | 4 +- .../org/apache/helix/integration/TestDrop.java | 2 + .../helix/integration/TestDropResource.java | 2 +- .../integration/TestEntropyFreeNodeBounce.java | 6 +- .../helix/integration/TestErrorPartition.java | 2 +- .../integration/TestExternalViewUpdates.java | 6 +- .../integration/TestHelixCustomCodeRunner.java | 2 +- .../helix/integration/TestHelixInstanceTag.java | 2 +- .../helix/integration/TestInstanceHistory.java | 56 -- .../integration/TestInvalidAutoIdealState.java | 4 +- .../TestInvalidResourceRebalance.java | 2 +- .../integration/TestNodeOfflineTimeStamp.java | 59 -- .../TestPartitionLevelTransitionConstraint.java | 2 +- .../helix/integration/TestPauseSignal.java | 4 +- .../TestReelectedPipelineCorrectness.java | 4 +- .../helix/integration/TestRenamePartition.java | 2 +- .../helix/integration/TestResetInstance.java | 2 +- .../integration/TestResetPartitionState.java | 2 +- .../helix/integration/TestResetResource.java | 2 +- .../TestResourceWithSamePartitionKey.java | 2 +- .../helix/integration/TestSchemataSM.java | 4 +- .../integration/TestSemiAutoRebalance.java | 229 ------ .../TestSessionExpiryInTransition.java | 4 +- .../helix/integration/TestStandAloneCMMain.java | 2 +- .../TestStandAloneCMSessionExpiry.java | 2 +- .../TestStateTransitionTimeoutWithResource.java | 192 ----- .../helix/integration/TestSwapInstance.java | 2 +- .../integration/TestZkCallbackHandlerLeak.java | 2 +- .../helix/integration/TestZkReconnect.java | 4 +- .../helix/integration/TestZkSessionExpiry.java | 2 +- .../common/ZkStandAloneCMTestBase.java | 6 +- .../controller/TestControllerLiveLock.java | 2 +- .../TestSkipBestPossibleCalculation.java | 8 +- .../manager/TestConsecutiveZkSessionExpiry.java | 4 +- .../manager/TestControllerManager.java | 4 +- .../TestDistributedControllerManager.java | 4 +- .../integration/manager/TestStateModelLeak.java | 4 +- .../manager/TestZkCallbackHandlerLeak.java | 2 +- .../messaging/TestMessageThrottle2.java | 4 +- .../paticipant/TestNonOfflineInitState.java | 4 +- .../paticipant/TestParticipantErrorMessage.java | 4 +- .../paticipant/TestRestartParticipant.java | 4 +- .../paticipant/TestStateTransitionTimeout.java | 4 +- .../TestAutoRebalancePartitionLimit.java | 4 +- .../TestCustomizedIdealStateRebalancer.java | 4 +- .../rebalancer/TestFullAutoNodeTagging.java | 6 +- .../manager/zk/TestLiveInstanceBounce.java | 2 +- .../TestClusterStatusMonitorLifecycle.java | 6 +- .../mbeans/TestResetClusterMetrics.java | 2 +- .../helix/tools/TestClusterStateVerifier.java | 4 +- .../apache/helix/tools/TestHelixAdminCli.java | 6 +- 80 files changed, 144 insertions(+), 2884 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestHelixAdminScenariosRest.java ---------------------------------------------------------------------- diff --git a/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestHelixAdminScenariosRest.java b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestHelixAdminScenariosRest.java index 369c6f0..0b49096 100644 --- a/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestHelixAdminScenariosRest.java +++ b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestHelixAdminScenariosRest.java @@ -41,9 +41,9 @@ import org.apache.helix.model.IdealState.IdealStateProperty; import org.apache.helix.model.InstanceConfig; import org.apache.helix.model.LiveInstance; import org.apache.helix.tools.ClusterSetup; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier.BestPossAndExtViewZkVerifier; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier.MasterNbInExtViewVerifier; +import org.apache.helix.tools.ClusterStateVerifier; +import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier; +import org.apache.helix.tools.ClusterStateVerifier.MasterNbInExtViewVerifier; import org.apache.helix.webapp.resources.ClusterRepresentationUtil; import org.apache.helix.webapp.resources.InstancesResource.ListInstancesWrapper; import org.apache.helix.webapp.resources.JsonParameters; http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetInstance.java ---------------------------------------------------------------------- diff --git a/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetInstance.java b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetInstance.java index 052c3c0..fdcb1e5 100644 --- a/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetInstance.java +++ b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetInstance.java @@ -29,7 +29,7 @@ import org.apache.helix.integration.manager.ClusterControllerManager; import org.apache.helix.integration.manager.MockParticipantManager; import org.apache.helix.mock.participant.ErrTransition; import org.apache.helix.tools.ClusterSetup; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; +import org.apache.helix.tools.ClusterStateVerifier; import org.apache.helix.webapp.resources.JsonParameters; import org.testng.Assert; import org.testng.annotations.Test; http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetPartitionState.java ---------------------------------------------------------------------- diff --git a/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetPartitionState.java b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetPartitionState.java index 79364d2..ddf2ec1 100644 --- a/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetPartitionState.java +++ b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetPartitionState.java @@ -37,7 +37,7 @@ import org.apache.helix.mock.participant.ErrTransition; import org.apache.helix.model.LiveInstance; import org.apache.helix.model.Message; import org.apache.helix.tools.ClusterSetup; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; +import org.apache.helix.tools.ClusterStateVerifier; import org.apache.helix.webapp.resources.JsonParameters; import org.apache.log4j.Logger; import org.testng.Assert; http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetResource.java ---------------------------------------------------------------------- diff --git a/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetResource.java b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetResource.java index eb5d2a5..6746c70 100644 --- a/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetResource.java +++ b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/TestResetResource.java @@ -29,7 +29,7 @@ import org.apache.helix.integration.manager.ClusterControllerManager; import org.apache.helix.integration.manager.MockParticipantManager; import org.apache.helix.mock.participant.ErrTransition; import org.apache.helix.tools.ClusterSetup; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; +import org.apache.helix.tools.ClusterStateVerifier; import org.apache.helix.webapp.resources.JsonParameters; import org.testng.Assert; import org.testng.annotations.Test; @@ -71,7 +71,7 @@ public class TestResetResource extends AdminTestBase { if (i == 0) { participants[i] = - new MockParticipantManager(ZK_ADDR, clusterName, instanceName); + new MockParticipantManager(ZK_ADDR, clusterName, instanceName); participants[i].setTransition(new ErrTransition(errPartitions)); } else { participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName); http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/resources/TestJobQueuesResource.java ---------------------------------------------------------------------- diff --git a/helix-admin-webapp/src/test/java/org/apache/helix/webapp/resources/TestJobQueuesResource.java b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/resources/TestJobQueuesResource.java index 5d82723..5d8a93b 100644 --- a/helix-admin-webapp/src/test/java/org/apache/helix/webapp/resources/TestJobQueuesResource.java +++ b/helix-admin-webapp/src/test/java/org/apache/helix/webapp/resources/TestJobQueuesResource.java @@ -39,7 +39,7 @@ import org.apache.helix.task.TaskFactory; import org.apache.helix.task.TaskStateModelFactory; import org.apache.helix.task.beans.JobBean; import org.apache.helix.task.beans.WorkflowBean; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; +import org.apache.helix.tools.ClusterStateVerifier; import org.apache.helix.webapp.AdminTestBase; import org.apache.helix.webapp.AdminTestHelper; import org.apache.log4j.Logger; http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-agent/src/test/java/org/apache/helix/agent/TestHelixAgent.java ---------------------------------------------------------------------- diff --git a/helix-agent/src/test/java/org/apache/helix/agent/TestHelixAgent.java b/helix-agent/src/test/java/org/apache/helix/agent/TestHelixAgent.java index a0638f0..27b4d36 100644 --- a/helix-agent/src/test/java/org/apache/helix/agent/TestHelixAgent.java +++ b/helix-agent/src/test/java/org/apache/helix/agent/TestHelixAgent.java @@ -34,8 +34,8 @@ import org.apache.helix.model.HelixConfigScope; import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty; import org.apache.helix.model.builder.HelixConfigScopeBuilder; import org.apache.helix.tools.ClusterSetup; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier.BestPossAndExtViewZkVerifier; +import org.apache.helix.tools.ClusterStateVerifier; +import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier; import org.apache.log4j.Logger; import org.testng.Assert; import org.testng.annotations.AfterMethod; http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/main/java/org/apache/helix/HelixProperty.java ---------------------------------------------------------------------- diff --git a/helix-core/src/main/java/org/apache/helix/HelixProperty.java b/helix-core/src/main/java/org/apache/helix/HelixProperty.java index 47806aa..4a3ec5e 100644 --- a/helix-core/src/main/java/org/apache/helix/HelixProperty.java +++ b/helix-core/src/main/java/org/apache/helix/HelixProperty.java @@ -152,15 +152,6 @@ public class HelixProperty { } /** - * Initialize the property with an existing ZNRecord with new record id - * @param record - * @param id - */ - public HelixProperty(ZNRecord record, String id) { - _record = new ZNRecord(record, id); - } - - /** * Get the property identifier * @return the property id */ http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/main/java/org/apache/helix/PropertyKey.java ---------------------------------------------------------------------- diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java index 775db7a..20cb833 100644 --- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java +++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java @@ -329,11 +329,6 @@ public class PropertyKey { return new PropertyKey(ERRORS, Error.class, _clusterName, instanceName); } - public PropertyKey participantHistory(String instanceName) { - return new PropertyKey(INSTANCE_HISTORY, ParticipantHistory.class, _clusterName, - instanceName); - } - /** * Get a property key associated with {@link Error} for an instance under a session * @param instanceName http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java ---------------------------------------------------------------------- diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java b/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java index aa0cbbc..45dcc15 100644 --- a/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java +++ b/helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java @@ -161,41 +161,41 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener { Class listenerClass = null; switch (_changeType) { - case IDEAL_STATE: - listenerClass = IdealStateChangeListener.class; - break; - case INSTANCE_CONFIG: - if (_listener instanceof ConfigChangeListener) { - listenerClass = ConfigChangeListener.class; - } else if (_listener instanceof InstanceConfigChangeListener) { - listenerClass = InstanceConfigChangeListener.class; - } - break; - case CLUSTER_CONFIG: - listenerClass = ClusterConfigChangeListener.class; - break; - case RESOURCE_CONFIG: - listenerClass = ResourceConfigChangeListener.class; - break; - case CONFIG: + case IDEAL_STATE: + listenerClass = IdealStateChangeListener.class; + break; + case INSTANCE_CONFIG: + if (_listener instanceof ConfigChangeListener) { listenerClass = ConfigChangeListener.class; - break; - case LIVE_INSTANCE: - listenerClass = LiveInstanceChangeListener.class; - break; - case CURRENT_STATE: - listenerClass = CurrentStateChangeListener.class; ; - break; - case MESSAGE: - case MESSAGES_CONTROLLER: - listenerClass = MessageListener.class; - break; - case EXTERNAL_VIEW: - case TARGET_EXTERNAL_VIEW: - listenerClass = ExternalViewChangeListener.class; - break; - case CONTROLLER: - listenerClass = ControllerChangeListener.class; + } else if (_listener instanceof InstanceConfigChangeListener) { + listenerClass = InstanceConfigChangeListener.class; + } + break; + case CLUSTER_CONFIG: + listenerClass = ClusterConfigChangeListener.class; + break; + case RESOURCE_CONFIG: + listenerClass = ResourceConfigChangeListener.class; + break; + case CONFIG: + listenerClass = ConfigChangeListener.class; + break; + case LIVE_INSTANCE: + listenerClass = LiveInstanceChangeListener.class; + break; + case CURRENT_STATE: + listenerClass = CurrentStateChangeListener.class; ; + break; + case MESSAGE: + case MESSAGES_CONTROLLER: + listenerClass = MessageListener.class; + break; + case EXTERNAL_VIEW: + case TARGET_EXTERNAL_VIEW: + listenerClass = ExternalViewChangeListener.class; + break; + case CONTROLLER: + listenerClass = ControllerChangeListener.class; } Method callbackMethod = listenerClass.getMethods()[0]; @@ -449,7 +449,7 @@ public class CallbackHandler implements IZkChildListener, IZkDataListener { case CURRENT_STATE: case IDEAL_STATE: case EXTERNAL_VIEW: - case TARGET_EXTERNAL_VIEW:{ + case TARGET_EXTERNAL_VIEW:{ // check if bucketized BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_zkClient); List<ZNRecord> records = baseAccessor.getChildren(path, null, 0); http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterExternalViewVerifier.java ---------------------------------------------------------------------- diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterExternalViewVerifier.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterExternalViewVerifier.java deleted file mode 100644 index 933acc2..0000000 --- a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterExternalViewVerifier.java +++ /dev/null @@ -1,175 +0,0 @@ -package org.apache.helix.tools.ClusterVerifiers; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.helix.controller.pipeline.Stage; -import org.apache.helix.controller.pipeline.StageContext; -import org.apache.helix.controller.stages.AttributeName; -import org.apache.helix.controller.stages.BestPossibleStateCalcStage; -import org.apache.helix.controller.stages.BestPossibleStateOutput; -import org.apache.helix.controller.stages.ClusterDataCache; -import org.apache.helix.controller.stages.ClusterEvent; -import org.apache.helix.controller.stages.CurrentStateComputationStage; -import org.apache.helix.controller.stages.ResourceComputationStage; -import org.apache.helix.manager.zk.ZkClient; -import org.apache.helix.model.ExternalView; -import org.apache.helix.model.Partition; -import org.apache.log4j.Logger; - -/** - * given zk, cluster, and a list of expected live-instances - * check whether cluster's external-view reaches best-possible states - */ - -/** - * This class is deprecated, please use BestPossibleExternalViewVerifier instead. - */ -@Deprecated -public class ClusterExternalViewVerifier extends ClusterVerifier { - private static Logger LOG = Logger.getLogger(ClusterExternalViewVerifier.class); - - final List<String> _expectSortedLiveNodes; // always sorted - - public ClusterExternalViewVerifier(ZkClient zkclient, String clusterName, - List<String> expectLiveNodes) { - super(zkclient, clusterName); - _expectSortedLiveNodes = expectLiveNodes; - Collections.sort(_expectSortedLiveNodes); - } - - boolean verifyLiveNodes(List<String> actualLiveNodes) { - Collections.sort(actualLiveNodes); - return _expectSortedLiveNodes.equals(actualLiveNodes); - } - - /** - * @param externalView - * @param bestPossibleState map of partition to map of instance to state - * @return - */ - boolean verifyExternalView(ExternalView externalView, - Map<Partition, Map<String, String>> bestPossibleState) { - Map<String, Map<String, String>> bestPossibleStateMap = - convertBestPossibleState(bestPossibleState); - // trimBestPossibleState(bestPossibleStateMap); - - Map<String, Map<String, String>> externalViewMap = externalView.getRecord().getMapFields(); - return externalViewMap.equals(bestPossibleStateMap); - } - - static void runStage(ClusterEvent event, Stage stage) throws Exception { - StageContext context = new StageContext(); - stage.init(context); - stage.preProcess(); - stage.process(event); - stage.postProcess(); - } - - BestPossibleStateOutput calculateBestPossibleState(ClusterDataCache cache) throws Exception { - ClusterEvent event = new ClusterEvent("event"); - event.addAttribute("ClusterDataCache", cache); - - List<Stage> stages = new ArrayList<Stage>(); - stages.add(new ResourceComputationStage()); - stages.add(new CurrentStateComputationStage()); - stages.add(new BestPossibleStateCalcStage()); - - for (Stage stage : stages) { - runStage(event, stage); - } - - return event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name()); - } - - /** - * remove empty map and DROPPED state from best possible state - * @param bestPossibleState - */ - // static void trimBestPossibleState(Map<String, Map<String, String>> bestPossibleState) { - // Iterator<Entry<String, Map<String, String>>> iter = bestPossibleState.entrySet().iterator(); - // while (iter.hasNext()) { - // Map.Entry<String, Map<String, String>> entry = iter.next(); - // Map<String, String> instanceStateMap = entry.getValue(); - // if (instanceStateMap.isEmpty()) { - // iter.remove(); - // } else { - // // remove instances with DROPPED state - // Iterator<Map.Entry<String, String>> insIter = instanceStateMap.entrySet().iterator(); - // while (insIter.hasNext()) { - // Map.Entry<String, String> insEntry = insIter.next(); - // String state = insEntry.getValue(); - // if (state.equalsIgnoreCase(HelixDefinedState.DROPPED.toString())) { - // insIter.remove(); - // } - // } - // } - // } - // } - - static Map<String, Map<String, String>> convertBestPossibleState( - Map<Partition, Map<String, String>> bestPossibleState) { - Map<String, Map<String, String>> result = new HashMap<String, Map<String, String>>(); - for (Partition partition : bestPossibleState.keySet()) { - result.put(partition.getPartitionName(), bestPossibleState.get(partition)); - } - return result; - } - - @Override - public boolean verify() throws Exception { - ClusterDataCache cache = new ClusterDataCache(); - cache.refresh(_accessor); - - List<String> liveInstances = new ArrayList<String>(); - liveInstances.addAll(cache.getLiveInstances().keySet()); - boolean success = verifyLiveNodes(liveInstances); - if (!success) { - LOG.info("liveNodes not match, expect: " + _expectSortedLiveNodes + ", actual: " - + liveInstances); - return false; - } - - BestPossibleStateOutput bestPossbileStates = calculateBestPossibleState(cache); - Map<String, ExternalView> externalViews = - _accessor.getChildValuesMap(_keyBuilder.externalViews()); - - // TODO all ideal-states should be included in external-views - - for (String resourceName : externalViews.keySet()) { - ExternalView externalView = externalViews.get(resourceName); - Map<Partition, Map<String, String>> bestPossbileState = - bestPossbileStates.getResourceMap(resourceName); - success = verifyExternalView(externalView, bestPossbileState); - if (!success) { - LOG.info("external-view for resource: " + resourceName + " not match"); - return false; - } - } - - return true; - } - -} http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterStateVerifier.java ---------------------------------------------------------------------- diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterStateVerifier.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterStateVerifier.java deleted file mode 100644 index eace66f..0000000 --- a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterStateVerifier.java +++ /dev/null @@ -1,739 +0,0 @@ -package org.apache.helix.tools.ClusterVerifiers; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.I0Itec.zkclient.IZkChildListener; -import org.I0Itec.zkclient.IZkDataListener; -import org.I0Itec.zkclient.exception.ZkNodeExistsException; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.helix.HelixDataAccessor; -import org.apache.helix.HelixDefinedState; -import org.apache.helix.PropertyKey; -import org.apache.helix.PropertyKey.Builder; -import org.apache.helix.PropertyPathBuilder; -import org.apache.helix.PropertyType; -import org.apache.helix.ZNRecord; -import org.apache.helix.controller.pipeline.Stage; -import org.apache.helix.controller.pipeline.StageContext; -import org.apache.helix.controller.stages.AttributeName; -import org.apache.helix.controller.stages.BestPossibleStateCalcStage; -import org.apache.helix.controller.stages.BestPossibleStateOutput; -import org.apache.helix.controller.stages.ClusterDataCache; -import org.apache.helix.controller.stages.ClusterEvent; -import org.apache.helix.controller.stages.CurrentStateComputationStage; -import org.apache.helix.controller.stages.ResourceComputationStage; -import org.apache.helix.manager.zk.ZKHelixDataAccessor; -import org.apache.helix.manager.zk.ZkBaseDataAccessor; -import org.apache.helix.manager.zk.ZkClient; -import org.apache.helix.model.ExternalView; -import org.apache.helix.model.IdealState; -import org.apache.helix.model.Partition; -import org.apache.helix.model.Resource; -import org.apache.helix.task.TaskConstants; -import org.apache.helix.tools.ClusterSetup; -import org.apache.helix.util.ZKClientPool; -import org.apache.log4j.Logger; - -import com.google.common.collect.Sets; - -/** - * This class is deprecated, please use dedicated verifier classes, such as BestPossibleExternViewVerifier, etc. - */ -@Deprecated -public class ClusterStateVerifier { - public static String cluster = "cluster"; - public static String zkServerAddress = "zkSvr"; - public static String help = "help"; - public static String timeout = "timeout"; - public static String period = "period"; - public static String resources = "resources"; - - private static Logger LOG = Logger.getLogger(ClusterStateVerifier.class); - - public interface Verifier { - boolean verify(); - } - - public interface ZkVerifier extends Verifier { - ZkClient getZkClient(); - - String getClusterName(); - } - - /** Use BestPossibleExternViewVerifier instead */ - @Deprecated - static class ExtViewVeriferZkListener implements IZkChildListener, IZkDataListener { - final CountDownLatch _countDown; - final ZkClient _zkClient; - final Verifier _verifier; - - public ExtViewVeriferZkListener(CountDownLatch countDown, ZkClient zkClient, ZkVerifier verifier) { - _countDown = countDown; - _zkClient = zkClient; - _verifier = verifier; - } - - @Override - public void handleDataChange(String dataPath, Object data) throws Exception { - boolean result = _verifier.verify(); - if (result == true) { - _countDown.countDown(); - } - } - - @Override - public void handleDataDeleted(String dataPath) throws Exception { - // TODO Auto-generated method stub - - } - - @Override - public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception { - for (String child : currentChilds) { - String childPath = parentPath.equals("/") ? parentPath + child : parentPath + "/" + child; - _zkClient.subscribeDataChanges(childPath, this); - } - - boolean result = _verifier.verify(); - if (result == true) { - _countDown.countDown(); - } - } - } - - private static ZkClient validateAndGetClient(String zkAddr, String clusterName) { - if (zkAddr == null || clusterName == null) { - throw new IllegalArgumentException("requires zkAddr|clusterName"); - } - return ZKClientPool.getZkClient(zkAddr); - } - - public static class BestPossAndExtViewZkVerifier implements ZkVerifier { - private final String clusterName; - private final Map<String, Map<String, String>> errStates; - private final ZkClient zkClient; - private final Set<String> resources; - - public BestPossAndExtViewZkVerifier(String zkAddr, String clusterName) { - this(zkAddr, clusterName, null); - } - - public BestPossAndExtViewZkVerifier(String zkAddr, String clusterName, - Map<String, Map<String, String>> errStates) { - this(zkAddr, clusterName, errStates, null); - } - - public BestPossAndExtViewZkVerifier(String zkAddr, String clusterName, - Map<String, Map<String, String>> errStates, Set<String> resources) { - this(validateAndGetClient(zkAddr, clusterName), clusterName, errStates, resources); - } - - public BestPossAndExtViewZkVerifier(ZkClient zkClient, String clusterName, - Map<String, Map<String, String>> errStates, Set<String> resources) { - if (zkClient == null || clusterName == null) { - throw new IllegalArgumentException("requires zkClient|clusterName"); - } - this.clusterName = clusterName; - this.errStates = errStates; - this.zkClient = zkClient; - this.resources = resources; - } - - @Override - public boolean verify() { - try { - HelixDataAccessor accessor = - new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(zkClient)); - - return verifyBestPossAndExtView(accessor, errStates, clusterName, resources); - } catch (Exception e) { - LOG.error("exception in verification", e); - } - return false; - } - - private boolean verifyBestPossAndExtView(HelixDataAccessor accessor, - Map<String, Map<String, String>> errStates, String clusterName, Set<String> resources) { - try { - PropertyKey.Builder keyBuilder = accessor.keyBuilder(); - // read cluster once and do verification - ClusterDataCache cache = new ClusterDataCache(); - cache.refresh(accessor); - - Map<String, IdealState> idealStates = cache.getIdealStates(); - if (idealStates == null) { - // ideal state is null because ideal state is dropped - idealStates = Collections.emptyMap(); - } - - // filter out all resources that use Task state model - Iterator<Map.Entry<String, IdealState>> it = idealStates.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry<String, IdealState> pair = it.next(); - if (pair.getValue().getStateModelDefRef().equals(TaskConstants.STATE_MODEL_NAME)) { - it.remove(); - } - } - - Map<String, ExternalView> extViews = accessor.getChildValuesMap(keyBuilder.externalViews()); - if (extViews == null) { - extViews = Collections.emptyMap(); - } - - // Filter resources if requested - if (resources != null && !resources.isEmpty()) { - idealStates.keySet().retainAll(resources); - extViews.keySet().retainAll(resources); - } - - // if externalView is not empty and idealState doesn't exist - // add empty idealState for the resource - for (String resource : extViews.keySet()) { - if (!idealStates.containsKey(resource)) { - idealStates.put(resource, new IdealState(resource)); - } - } - - // calculate best possible state - BestPossibleStateOutput bestPossOutput = calcBestPossState(cache, resources); - Map<String, Map<Partition, Map<String, String>>> bestPossStateMap = - bestPossOutput.getStateMap(); - - // set error states - if (errStates != null) { - for (String resourceName : errStates.keySet()) { - Map<String, String> partErrStates = errStates.get(resourceName); - for (String partitionName : partErrStates.keySet()) { - String instanceName = partErrStates.get(partitionName); - - if (!bestPossStateMap.containsKey(resourceName)) { - bestPossStateMap.put(resourceName, new HashMap<Partition, Map<String, String>>()); - } - Partition partition = new Partition(partitionName); - if (!bestPossStateMap.get(resourceName).containsKey(partition)) { - bestPossStateMap.get(resourceName).put(partition, new HashMap<String, String>()); - } - bestPossStateMap.get(resourceName).get(partition) - .put(instanceName, HelixDefinedState.ERROR.name()); - } - } - } - - // System.out.println("stateMap: " + bestPossStateMap); - - for (String resourceName : idealStates.keySet()) { - ExternalView extView = extViews.get(resourceName); - if (extView == null) { - IdealState is = idealStates.get(resourceName); - if (is.isExternalViewDisabled()) { - continue; - } else { - LOG.info("externalView for " + resourceName + " is not available"); - return false; - } - } - - // step 0: remove empty map and DROPPED state from best possible state - Map<Partition, Map<String, String>> bpStateMap = - bestPossOutput.getResourceMap(resourceName); - Iterator<Map.Entry<Partition, Map<String, String>>> iter = bpStateMap.entrySet().iterator(); - while (iter.hasNext()) { - Map.Entry<Partition, Map<String, String>> entry = iter.next(); - Map<String, String> instanceStateMap = entry.getValue(); - if (instanceStateMap.isEmpty()) { - iter.remove(); - } else { - // remove instances with DROPPED state - Iterator<Map.Entry<String, String>> insIter = instanceStateMap.entrySet().iterator(); - while (insIter.hasNext()) { - Map.Entry<String, String> insEntry = insIter.next(); - String state = insEntry.getValue(); - if (state.equalsIgnoreCase(HelixDefinedState.DROPPED.name())) { - insIter.remove(); - } - } - } - } - - // System.err.println("resource: " + resourceName + ", bpStateMap: " + bpStateMap); - - // step 1: externalView and bestPossibleState has equal size - int extViewSize = extView.getRecord().getMapFields().size(); - int bestPossStateSize = bestPossOutput.getResourceMap(resourceName).size(); - if (extViewSize != bestPossStateSize) { - LOG.info("exterView size (" + extViewSize + ") is different from bestPossState size (" - + bestPossStateSize + ") for resource: " + resourceName); - - // System.err.println("exterView size (" + extViewSize - // + ") is different from bestPossState size (" + bestPossStateSize - // + ") for resource: " + resourceName); - // System.out.println("extView: " + extView.getRecord().getMapFields()); - // System.out.println("bestPossState: " + - // bestPossOutput.getResourceMap(resourceName)); - return false; - } - - // step 2: every entry in external view is contained in best possible state - for (String partition : extView.getRecord().getMapFields().keySet()) { - Map<String, String> evInstanceStateMap = extView.getRecord().getMapField(partition); - Map<String, String> bpInstanceStateMap = - bestPossOutput.getInstanceStateMap(resourceName, new Partition(partition)); - - boolean result = compareMap(evInstanceStateMap, bpInstanceStateMap); - if (result == false) { - LOG.info("externalView is different from bestPossibleState for partition:" + partition); - - // System.err.println("externalView is different from bestPossibleState for partition: " - // + partition + ", actual: " + evInstanceStateMap + ", bestPoss: " + - // bpInstanceStateMap); - return false; - } - } - } - return true; - } catch (Exception e) { - LOG.error("exception in verification", e); - return false; - } - } - - /** - * calculate the best possible state note that DROPPED states are not checked since when - * kick off the BestPossibleStateCalcStage we are providing an empty current state map - * - * @param cache - * @return - * @throws Exception - */ - private BestPossibleStateOutput calcBestPossState(ClusterDataCache cache, Set<String> resources) - throws Exception { - ClusterEvent event = new ClusterEvent("sampleEvent"); - event.addAttribute("ClusterDataCache", cache); - - ResourceComputationStage rcState = new ResourceComputationStage(); - CurrentStateComputationStage csStage = new CurrentStateComputationStage(); - BestPossibleStateCalcStage bpStage = new BestPossibleStateCalcStage(); - - runStage(event, rcState); - - // Filter resources if specified - if (resources != null) { - Map<String, Resource> resourceMap = event.getAttribute(AttributeName.RESOURCES.name()); - resourceMap.keySet().retainAll(resources); - } - - runStage(event, csStage); - runStage(event, bpStage); - - BestPossibleStateOutput output = - event.getAttribute(AttributeName.BEST_POSSIBLE_STATE.name()); - - // System.out.println("output:" + output); - return output; - } - - private void runStage(ClusterEvent event, Stage stage) throws Exception { - StageContext context = new StageContext(); - stage.init(context); - stage.preProcess(); - stage.process(event); - stage.postProcess(); - } - - private <K, V> boolean compareMap(Map<K, V> map1, Map<K, V> map2) { - boolean isEqual = true; - if (map1 == null && map2 == null) { - // OK - } else if (map1 == null && map2 != null) { - if (!map2.isEmpty()) { - isEqual = false; - } - } else if (map1 != null && map2 == null) { - if (!map1.isEmpty()) { - isEqual = false; - } - } else { - // verify size - if (map1.size() != map2.size()) { - isEqual = false; - } - // verify each <key, value> in map1 is contained in map2 - for (K key : map1.keySet()) { - if (!map1.get(key).equals(map2.get(key))) { - LOG.debug( - "different value for key: " + key + "(map1: " + map1.get(key) + ", map2: " + map2 - .get(key) + ")"); - isEqual = false; - break; - } - } - } - return isEqual; - } - - @Override - public ZkClient getZkClient() { - return zkClient; - } - - @Override - public String getClusterName() { - return clusterName; - } - - @Override - public String toString() { - String verifierName = getClass().getName(); - verifierName = verifierName.substring(verifierName.lastIndexOf('.') + 1, verifierName.length()); - return verifierName + "(" + clusterName + "@" + zkClient.getServers() + ")"; - } - } - - - public static class MasterNbInExtViewVerifier implements ZkVerifier { - private final String clusterName; - private final ZkClient zkClient; - - public MasterNbInExtViewVerifier(String zkAddr, String clusterName) { - this(validateAndGetClient(zkAddr, clusterName), clusterName); - } - - public MasterNbInExtViewVerifier(ZkClient zkClient, String clusterName) { - if (zkClient == null || clusterName == null) { - throw new IllegalArgumentException("requires zkClient|clusterName"); - } - this.clusterName = clusterName; - this.zkClient = zkClient; - } - - @Override - public boolean verify() { - try { - ZKHelixDataAccessor accessor = - new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(zkClient)); - - return verifyMasterNbInExtView(accessor); - } catch (Exception e) { - LOG.error("exception in verification", e); - } - return false; - } - - @Override - public ZkClient getZkClient() { - return zkClient; - } - - @Override - public String getClusterName() { - return clusterName; - } - - private boolean verifyMasterNbInExtView(HelixDataAccessor accessor) { - Builder keyBuilder = accessor.keyBuilder(); - - Map<String, IdealState> idealStates = accessor.getChildValuesMap(keyBuilder.idealStates()); - if (idealStates == null || idealStates.size() == 0) { - LOG.info("No resource idealState"); - return true; - } - - Map<String, ExternalView> extViews = accessor.getChildValuesMap(keyBuilder.externalViews()); - if (extViews == null || extViews.size() < idealStates.size()) { - LOG.info("No externalViews | externalView.size() < idealState.size()"); - return false; - } - - for (String resource : extViews.keySet()) { - int partitions = idealStates.get(resource).getNumPartitions(); - Map<String, Map<String, String>> instanceStateMap = - extViews.get(resource).getRecord().getMapFields(); - if (instanceStateMap.size() < partitions) { - LOG.info("Number of externalViews (" + instanceStateMap.size() + ") < partitions (" - + partitions + ")"); - return false; - } - - for (String partition : instanceStateMap.keySet()) { - boolean foundMaster = false; - for (String instance : instanceStateMap.get(partition).keySet()) { - if (instanceStateMap.get(partition).get(instance).equalsIgnoreCase("MASTER")) { - foundMaster = true; - break; - } - } - if (!foundMaster) { - LOG.info("No MASTER for partition: " + partition); - return false; - } - } - } - return true; - } - } - - public static boolean verifyByPolling(Verifier verifier) { - return verifyByPolling(verifier, 30 * 1000); - } - - public static boolean verifyByPolling(Verifier verifier, long timeout) { - return verifyByPolling(verifier, timeout, 1000); - } - - public static boolean verifyByPolling(Verifier verifier, long timeout, long period) { - long startTime = System.currentTimeMillis(); - boolean result = false; - try { - long curTime; - do { - Thread.sleep(period); - result = verifier.verify(); - if (result == true) { - break; - } - curTime = System.currentTimeMillis(); - } while (curTime <= startTime + timeout); - return result; - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } finally { - long endTime = System.currentTimeMillis(); - - // debug - System.err.println(result + ": " + verifier + ": wait " + (endTime - startTime) - + "ms to verify"); - - } - return false; - } - - public static boolean verifyByZkCallback(ZkVerifier verifier) { - return verifyByZkCallback(verifier, 30000); - } - - /** - * This function should be always single threaded - * - * @param verifier - * @param timeout - * @return - */ - public static boolean verifyByZkCallback(ZkVerifier verifier, long timeout) { - long startTime = System.currentTimeMillis(); - CountDownLatch countDown = new CountDownLatch(1); - ZkClient zkClient = verifier.getZkClient(); - String clusterName = verifier.getClusterName(); - - // add an ephemeral node to /{clusterName}/CONFIGS/CLUSTER/verify - // so when analyze zk log, we know when a test ends - try { - zkClient.createEphemeral("/" + clusterName + "/CONFIGS/CLUSTER/verify"); - } catch (ZkNodeExistsException ex) { - LOG.error("There is already a verification in progress", ex); - throw ex; - } - - ExtViewVeriferZkListener listener = new ExtViewVeriferZkListener(countDown, zkClient, verifier); - - String extViewPath = PropertyPathBuilder.getPath(PropertyType.EXTERNALVIEW, clusterName); - zkClient.subscribeChildChanges(extViewPath, listener); - for (String child : zkClient.getChildren(extViewPath)) { - String childPath = extViewPath.equals("/") ? extViewPath + child : extViewPath + "/" + child; - zkClient.subscribeDataChanges(childPath, listener); - } - - // do initial verify - boolean result = verifier.verify(); - if (result == false) { - try { - result = countDown.await(timeout, TimeUnit.MILLISECONDS); - if (result == false) { - // make a final try if timeout - result = verifier.verify(); - } - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - // clean up - zkClient.unsubscribeChildChanges(extViewPath, listener); - for (String child : zkClient.getChildren(extViewPath)) { - String childPath = extViewPath.equals("/") ? extViewPath + child : extViewPath + "/" + child; - zkClient.unsubscribeDataChanges(childPath, listener); - } - - long endTime = System.currentTimeMillis(); - - zkClient.delete("/" + clusterName + "/CONFIGS/CLUSTER/verify"); - // debug - System.err.println(result + ": wait " + (endTime - startTime) + "ms, " + verifier); - - return result; - } - - @SuppressWarnings("static-access") - private static Options constructCommandLineOptions() { - Option helpOption = - OptionBuilder.withLongOpt(help).withDescription("Prints command-line options info") - .create(); - - Option zkServerOption = - OptionBuilder.withLongOpt(zkServerAddress).withDescription("Provide zookeeper address") - .create(); - zkServerOption.setArgs(1); - zkServerOption.setRequired(true); - zkServerOption.setArgName("ZookeeperServerAddress(Required)"); - - Option clusterOption = - OptionBuilder.withLongOpt(cluster).withDescription("Provide cluster name").create(); - clusterOption.setArgs(1); - clusterOption.setRequired(true); - clusterOption.setArgName("Cluster name (Required)"); - - Option timeoutOption = - OptionBuilder.withLongOpt(timeout).withDescription("Timeout value for verification") - .create(); - timeoutOption.setArgs(1); - timeoutOption.setArgName("Timeout value (Optional), default=30s"); - - Option sleepIntervalOption = - OptionBuilder.withLongOpt(period).withDescription("Polling period for verification") - .create(); - sleepIntervalOption.setArgs(1); - sleepIntervalOption.setArgName("Polling period value (Optional), default=1s"); - - Option resourcesOption = - OptionBuilder.withLongOpt(resources).withDescription("Specific set of resources to verify") - .create(); - resourcesOption.setArgs(1); - resourcesOption.setArgName("Comma-separated resource names, default is all resources"); - - Options options = new Options(); - options.addOption(helpOption); - options.addOption(zkServerOption); - options.addOption(clusterOption); - options.addOption(timeoutOption); - options.addOption(sleepIntervalOption); - options.addOption(resourcesOption); - - return options; - } - - public static void printUsage(Options cliOptions) { - HelpFormatter helpFormatter = new HelpFormatter(); - helpFormatter.setWidth(1000); - helpFormatter.printHelp("java " + ClusterSetup.class.getName(), cliOptions); - } - - public static CommandLine processCommandLineArgs(String[] cliArgs) { - CommandLineParser cliParser = new GnuParser(); - Options cliOptions = constructCommandLineOptions(); - // CommandLine cmd = null; - - try { - return cliParser.parse(cliOptions, cliArgs); - } catch (ParseException pe) { - System.err.println("CommandLineClient: failed to parse command-line options: " - + pe.toString()); - printUsage(cliOptions); - System.exit(1); - } - return null; - } - - public static boolean verifyState(String[] args) { - // TODO Auto-generated method stub - String clusterName = "storage-cluster"; - String zkServer = "localhost:2181"; - long timeoutValue = 0; - long periodValue = 1000; - - Set<String> resourceSet = null; - if (args.length > 0) { - CommandLine cmd = processCommandLineArgs(args); - zkServer = cmd.getOptionValue(zkServerAddress); - clusterName = cmd.getOptionValue(cluster); - String timeoutStr = cmd.getOptionValue(timeout); - String periodStr = cmd.getOptionValue(period); - String resourceStr = cmd.getOptionValue(resources); - - if (timeoutStr != null) { - try { - timeoutValue = Long.parseLong(timeoutStr); - } catch (Exception e) { - System.err.println("Exception in converting " + timeoutStr + " to long. Use default (0)"); - } - } - - if (periodStr != null) { - try { - periodValue = Long.parseLong(periodStr); - } catch (Exception e) { - System.err.println("Exception in converting " + periodStr - + " to long. Use default (1000)"); - } - } - - // Allow specifying resources explicitly - if (resourceStr != null) { - String[] resources = resourceStr.split("[\\s,]"); - resourceSet = Sets.newHashSet(resources); - } - - } - // return verifyByPolling(new BestPossAndExtViewZkVerifier(zkServer, clusterName), - // timeoutValue, - // periodValue); - - ZkVerifier verifier; - if (resourceSet == null) { - verifier = new BestPossAndExtViewZkVerifier(zkServer, clusterName); - } else { - verifier = new BestPossAndExtViewZkVerifier(zkServer, clusterName, null, resourceSet); - } - return verifyByZkCallback(verifier, timeoutValue); - } - - public static void main(String[] args) { - boolean result = verifyState(args); - System.out.println(result ? "Successful" : "failed"); - System.exit(1); - } - -} http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterVerifier.java ---------------------------------------------------------------------- diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterVerifier.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterVerifier.java deleted file mode 100644 index 13d98f4..0000000 --- a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterVerifier.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.apache.helix.tools.ClusterVerifiers; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.I0Itec.zkclient.IZkChildListener; -import org.I0Itec.zkclient.IZkDataListener; -import org.apache.helix.HelixDataAccessor; -import org.apache.helix.PropertyKey; -import org.apache.helix.ZNRecord; -import org.apache.helix.manager.zk.ZKHelixDataAccessor; -import org.apache.helix.manager.zk.ZkBaseDataAccessor; -import org.apache.helix.manager.zk.ZkClient; -import org.apache.log4j.Logger; - -@Deprecated -public abstract class ClusterVerifier implements IZkChildListener, IZkDataListener { - private static Logger LOG = Logger.getLogger(ClusterVerifier.class); - - protected final ZkClient _zkclient; - protected final String _clusterName; - protected final HelixDataAccessor _accessor; - protected final PropertyKey.Builder _keyBuilder; - private CountDownLatch _countdown; - - static class ClusterVerifyTrigger { - final PropertyKey _triggerKey; - final boolean _triggerOnChildDataChange; - - public ClusterVerifyTrigger(PropertyKey triggerKey, boolean triggerOnChildDataChange) { - _triggerKey = triggerKey; - _triggerOnChildDataChange = triggerOnChildDataChange; - } - } - - public ClusterVerifier(ZkClient zkclient, String clusterName) { - _zkclient = zkclient; - _clusterName = clusterName; - _accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(zkclient)); - _keyBuilder = _accessor.keyBuilder(); - } - - public boolean verifyByCallback(long timeout, List<ClusterVerifyTrigger> triggers) { - _countdown = new CountDownLatch(1); - - for (ClusterVerifyTrigger trigger : triggers) { - String path = trigger._triggerKey.getPath(); - _zkclient.subscribeChildChanges(path, this); - if (trigger._triggerOnChildDataChange) { - List<String> childs = _zkclient.getChildren(path); - for (String child : childs) { - String childPath = String.format("%s/%s", path, child); - _zkclient.subscribeDataChanges(childPath, this); - } - } - } - - boolean success = false; - try { - success = verify(); - if (!success) { - - success = _countdown.await(timeout, TimeUnit.MILLISECONDS); - if (!success) { - // make a final try if timeout - success = verify(); - } - } - } catch (Exception e) { - LOG.error("Exception in verifier", e); - } - - // clean up - _zkclient.unsubscribeAll(); - - return success; - } - - @Override - public void handleDataChange(String dataPath, Object data) throws Exception { - boolean success = verify(); - if (success) { - _countdown.countDown(); - } - } - - @Override - public void handleDataDeleted(String dataPath) throws Exception { - _zkclient.unsubscribeDataChanges(dataPath, this); - } - - @Override - public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception { - for (String child : currentChilds) { - String childPath = String.format("%s/%s", parentPath, child); - _zkclient.subscribeDataChanges(childPath, this); - } - - boolean success = verify(); - if (success) { - _countdown.countDown(); - } - } - - public boolean verifyByPolling(long timeout) { - try { - long start = System.currentTimeMillis(); - boolean success; - do { - success = verify(); - if (success) { - return true; - } - TimeUnit.MILLISECONDS.sleep(500); - } while ((System.currentTimeMillis() - start) <= timeout); - } catch (Exception e) { - LOG.error("Exception in verifier", e); - } - return false; - } - - /** - * verify - * @return - * @throws Exception - */ - public abstract boolean verify() throws Exception; -} http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/test/java/org/apache/helix/integration/TestAddClusterV2.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestAddClusterV2.java b/helix-core/src/test/java/org/apache/helix/integration/TestAddClusterV2.java index 1cc462b..6b27085 100644 --- a/helix-core/src/test/java/org/apache/helix/integration/TestAddClusterV2.java +++ b/helix-core/src/test/java/org/apache/helix/integration/TestAddClusterV2.java @@ -25,7 +25,7 @@ import org.apache.helix.integration.common.ZkIntegrationTestBase; import org.apache.helix.integration.manager.ClusterDistributedController; import org.apache.helix.integration.manager.MockParticipantManager; import org.apache.helix.tools.ClusterSetup; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; +import org.apache.helix.tools.ClusterStateVerifier; import org.apache.log4j.Logger; import org.testng.Assert; import org.testng.annotations.AfterClass; http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/test/java/org/apache/helix/integration/TestAddStateModelFactoryAfterConnect.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestAddStateModelFactoryAfterConnect.java b/helix-core/src/test/java/org/apache/helix/integration/TestAddStateModelFactoryAfterConnect.java index 5cbec64..b05b9fb 100644 --- a/helix-core/src/test/java/org/apache/helix/integration/TestAddStateModelFactoryAfterConnect.java +++ b/helix-core/src/test/java/org/apache/helix/integration/TestAddStateModelFactoryAfterConnect.java @@ -34,8 +34,8 @@ import org.apache.helix.mock.participant.MockMSModelFactory; import org.apache.helix.model.IdealState; import org.apache.helix.model.Message; import org.apache.helix.tools.ClusterSetup; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier.BestPossAndExtViewZkVerifier; +import org.apache.helix.tools.ClusterStateVerifier; +import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier; import org.testng.Assert; import org.testng.annotations.Test; http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalanceWithDisabledInstance.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalanceWithDisabledInstance.java b/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalanceWithDisabledInstance.java deleted file mode 100644 index dfc7b35..0000000 --- a/helix-core/src/test/java/org/apache/helix/integration/TestAutoRebalanceWithDisabledInstance.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.apache.helix.integration; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.apache.helix.HelixAdmin; -import org.apache.helix.integration.manager.MockParticipantManager; -import org.apache.helix.model.ExternalView; -import org.apache.helix.model.IdealState.RebalanceMode; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; -import org.testng.Assert; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -public class TestAutoRebalanceWithDisabledInstance extends ZkStandAloneCMTestBase { - private static String TEST_DB_2 = "TestDB2"; - - @BeforeClass - @Override - public void beforeClass() throws Exception { - super.beforeClass(); - _setupTool.addResourceToCluster(CLUSTER_NAME, TEST_DB_2, _PARTITIONS, STATE_MODEL, - RebalanceMode.FULL_AUTO + ""); - _setupTool.rebalanceResource(CLUSTER_NAME, TEST_DB_2, _replica); - - Thread.sleep(200); - - boolean result = - ClusterStateVerifier.verifyByZkCallback(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, - CLUSTER_NAME)); - Assert.assertTrue(result); - } - - @Test() - public void testDisableEnableInstanceAutoRebalance() throws Exception { - String disabledInstance = _participants[0].getInstanceName(); - - Set<String> currentPartitions = getCurrentPartitionsOnInstance(CLUSTER_NAME, TEST_DB_2, - disabledInstance); - Assert.assertFalse(currentPartitions.isEmpty()); - - // disable instance - _setupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, disabledInstance, false); - Thread.sleep(1000); - currentPartitions = getCurrentPartitionsOnInstance(CLUSTER_NAME, TEST_DB_2, disabledInstance); - Assert.assertTrue(currentPartitions.isEmpty()); - - //enable instance - _setupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, disabledInstance, true); - Thread.sleep(1000); - currentPartitions = getCurrentPartitionsOnInstance(CLUSTER_NAME, TEST_DB_2, disabledInstance); - Assert.assertFalse(currentPartitions.isEmpty()); - } - - @Test() - public void testAddDisabledInstanceAutoRebalance() throws Exception { - // add disabled instance. - String nodeName = PARTICIPANT_PREFIX + "_" + (START_PORT + NODE_NR); - _setupTool.addInstanceToCluster(CLUSTER_NAME, nodeName); - MockParticipantManager participant = - new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, nodeName); - _setupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, nodeName, false); - - participant.syncStart(); - - Thread.sleep(400); - Set<String> currentPartitions = getCurrentPartitionsOnInstance(CLUSTER_NAME, TEST_DB_2, - nodeName); - Assert.assertTrue(currentPartitions.isEmpty()); - - //enable instance - _setupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, nodeName, true); - Thread.sleep(400); - currentPartitions = getCurrentPartitionsOnInstance(CLUSTER_NAME, TEST_DB_2, nodeName); - Assert.assertFalse(currentPartitions.isEmpty()); - } - - - private Set<String> getCurrentPartitionsOnInstance(String cluster, String dbName, String instance) { - HelixAdmin admin = _setupTool.getClusterManagementTool(); - Set<String> partitionSet = new HashSet<String>(); - - ExternalView ev = admin.getResourceExternalView(cluster, dbName); - for (String partition : ev.getRecord().getMapFields().keySet()) { - Map<String, String> assignments = ev.getRecord().getMapField(partition); - for (String ins : assignments.keySet()) { - if (ins.equals(instance)) { - partitionSet.add(partition); - } - } - } - return partitionSet; - } -} http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/test/java/org/apache/helix/integration/TestBasicSpectator.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestBasicSpectator.java b/helix-core/src/test/java/org/apache/helix/integration/TestBasicSpectator.java index e621350..a24ded4 100644 --- a/helix-core/src/test/java/org/apache/helix/integration/TestBasicSpectator.java +++ b/helix-core/src/test/java/org/apache/helix/integration/TestBasicSpectator.java @@ -30,7 +30,7 @@ import org.apache.helix.InstanceType; import org.apache.helix.NotificationContext; import org.apache.helix.integration.common.ZkStandAloneCMTestBase; import org.apache.helix.model.ExternalView; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; +import org.apache.helix.tools.ClusterStateVerifier; import org.testng.Assert; import org.testng.annotations.Test; http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/test/java/org/apache/helix/integration/TestCleanupExternalView.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestCleanupExternalView.java b/helix-core/src/test/java/org/apache/helix/integration/TestCleanupExternalView.java index f042e66..e7a7ea0 100644 --- a/helix-core/src/test/java/org/apache/helix/integration/TestCleanupExternalView.java +++ b/helix-core/src/test/java/org/apache/helix/integration/TestCleanupExternalView.java @@ -27,7 +27,7 @@ import org.apache.helix.manager.zk.ZKHelixDataAccessor; import org.apache.helix.manager.zk.ZkBaseDataAccessor; import org.apache.helix.model.ExternalView; import org.apache.helix.model.LiveInstance; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; +import org.apache.helix.tools.ClusterStateVerifier; import org.testng.Assert; import org.testng.annotations.Test; http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/test/java/org/apache/helix/integration/TestControllerHistory.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestControllerHistory.java b/helix-core/src/test/java/org/apache/helix/integration/TestControllerHistory.java deleted file mode 100644 index b078c96..0000000 --- a/helix-core/src/test/java/org/apache/helix/integration/TestControllerHistory.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.apache.helix.integration; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import org.apache.helix.HelixManager; -import org.apache.helix.HelixManagerFactory; -import org.apache.helix.InstanceType; -import org.apache.helix.PropertyKey; -import org.apache.helix.integration.manager.ClusterControllerManager; -import org.apache.helix.model.LeaderHistory; -import org.testng.Assert; -import org.testng.annotations.Test; - -import java.util.List; - -public class TestControllerHistory extends ZkStandAloneCMTestBase { - - @Test() - public void testControllerLeaderHistory() throws Exception { - HelixManager manager = HelixManagerFactory - .getZKHelixManager(CLUSTER_NAME, "admin", InstanceType.ADMINISTRATOR, ZK_ADDR); - manager.connect(); - - PropertyKey.Builder keyBuilder = new PropertyKey.Builder(CLUSTER_NAME); - PropertyKey propertyKey = keyBuilder.controllerLeaderHistory(); - LeaderHistory leaderHistory = manager.getHelixDataAccessor().getProperty(propertyKey); - Assert.assertNotNull(leaderHistory); - List<String> list = leaderHistory.getRecord().getListField("HISTORY"); - Assert.assertEquals(list.size(), 1); - - for (int i = 0; i <= 12; i++) { - _controller.syncStop(); - _controller = new ClusterControllerManager(ZK_ADDR, CLUSTER_NAME, "Controller-" + i); - _controller.syncStart(); - } - - leaderHistory = manager.getHelixDataAccessor().getProperty(propertyKey); - Assert.assertNotNull(leaderHistory); - list = leaderHistory.getRecord().getListField("HISTORY"); - Assert.assertEquals(list.size(), 10); - manager.disconnect(); - } -} http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/test/java/org/apache/helix/integration/TestCorrectnessOnConnectivityLoss.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestCorrectnessOnConnectivityLoss.java b/helix-core/src/test/java/org/apache/helix/integration/TestCorrectnessOnConnectivityLoss.java index 753baf7..cbe231d 100644 --- a/helix-core/src/test/java/org/apache/helix/integration/TestCorrectnessOnConnectivityLoss.java +++ b/helix-core/src/test/java/org/apache/helix/integration/TestCorrectnessOnConnectivityLoss.java @@ -36,8 +36,8 @@ import org.apache.helix.participant.statemachine.StateModelInfo; import org.apache.helix.participant.statemachine.StateTransitionError; import org.apache.helix.participant.statemachine.Transition; import org.apache.helix.spectator.RoutingTableProvider; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier.BestPossAndExtViewZkVerifier; +import org.apache.helix.tools.ClusterStateVerifier; +import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; http://git-wip-us.apache.org/repos/asf/helix/blob/fa587f0b/helix-core/src/test/java/org/apache/helix/integration/TestCrushAutoRebalance.java ---------------------------------------------------------------------- diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestCrushAutoRebalance.java b/helix-core/src/test/java/org/apache/helix/integration/TestCrushAutoRebalance.java deleted file mode 100644 index 72df3eb..0000000 --- a/helix-core/src/test/java/org/apache/helix/integration/TestCrushAutoRebalance.java +++ /dev/null @@ -1,222 +0,0 @@ -package org.apache.helix.integration; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import org.apache.helix.controller.rebalancer.strategy.CrushRebalanceStrategy; -import org.apache.helix.controller.rebalancer.strategy.MultiRoundCrushRebalanceStrategy; -import org.apache.helix.integration.manager.ClusterControllerManager; -import org.apache.helix.integration.manager.MockParticipantManager; -import org.apache.helix.model.BuiltInStateModelDefinitions; -import org.apache.helix.model.ExternalView; -import org.apache.helix.model.IdealState; -import org.apache.helix.model.IdealState.RebalanceMode; -import org.apache.helix.model.InstanceConfig; -import org.apache.helix.tools.ClusterSetup; -import org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class TestCrushAutoRebalance extends ZkIntegrationTestBase { - final int NUM_NODE = 6; - protected static final int START_PORT = 12918; - protected static final int _PARTITIONS = 20; - - protected final String CLASS_NAME = getShortClassName(); - protected final String CLUSTER_NAME = CLUSTER_PREFIX + "_" + CLASS_NAME; - protected ClusterControllerManager _controller; - - protected ClusterSetup _setupTool = null; - List<MockParticipantManager> _participants = new ArrayList<MockParticipantManager>(); - Map<String, String> _nodeToZoneMap = new HashMap<String, String>(); - Map<String, String> _nodeToTagMap = new HashMap<String, String>(); - List<String> _nodes = new ArrayList<String>(); - List<String> _allDBs = new ArrayList<String>(); - int _replica = 3; - - @BeforeClass - public void beforeClass() throws Exception { - System.out.println("START " + CLASS_NAME + " at " + new Date(System.currentTimeMillis())); - - String namespace = "/" + CLUSTER_NAME; - if (_gZkClient.exists(namespace)) { - _gZkClient.deleteRecursive(namespace); - } - _setupTool = new ClusterSetup(_gZkClient); - _setupTool.addCluster(CLUSTER_NAME, true); - - for (int i = 0; i < NUM_NODE; i++) { - String storageNodeName = PARTICIPANT_PREFIX + "_" + (START_PORT + i); - _setupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName); - String zone = "zone-" + i % 3; - String tag = "tag-" + i % 2; - _setupTool.getClusterManagementTool().setInstanceZoneId(CLUSTER_NAME, storageNodeName, zone); - _setupTool.getClusterManagementTool().addInstanceTag(CLUSTER_NAME, storageNodeName, tag); - _nodeToZoneMap.put(storageNodeName, zone); - _nodeToTagMap.put(storageNodeName, tag); - _nodes.add(storageNodeName); - } - - // start dummy participants - for (String node : _nodes) { - MockParticipantManager participant = - new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, node); - participant.syncStart(); - _participants.add(participant); - } - - // start controller - String controllerName = CONTROLLER_PREFIX + "_0"; - _controller = new ClusterControllerManager(ZK_ADDR, CLUSTER_NAME, controllerName); - _controller.syncStart(); - } - - @DataProvider(name = "rebalanceStrategies") - public static String [][] rebalanceStrategies() { - return new String[][] { {"CrushRebalanceStrategy", CrushRebalanceStrategy.class.getName()}, - {"MultiRoundCrushRebalanceStrategy", MultiRoundCrushRebalanceStrategy.class.getName()} - }; - } - - @Test(dataProvider = "rebalanceStrategies") - public void testZoneIsolation(String rebalanceStrategyName, String rebalanceStrategyClass) - throws Exception { - System.out.println("Test " + rebalanceStrategyName); - List<String> testDBs = new ArrayList<String>(); - String[] testModels = { BuiltInStateModelDefinitions.OnlineOffline.name(), - BuiltInStateModelDefinitions.MasterSlave.name(), - BuiltInStateModelDefinitions.LeaderStandby.name() - }; - int i = 0; - for (String stateModel : testModels) { - String db = "Test-DB-" + rebalanceStrategyName + "-" + i++; - _setupTool.addResourceToCluster(CLUSTER_NAME, db, _PARTITIONS, stateModel, - RebalanceMode.FULL_AUTO + "", rebalanceStrategyClass); - _setupTool.rebalanceStorageCluster(CLUSTER_NAME, db, _replica); - testDBs.add(db); - _allDBs.add(db); - } - Thread.sleep(300); - - boolean result = ClusterStateVerifier.verifyByZkCallback( - new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME)); - Assert.assertTrue(result); - - for (String db : testDBs) { - IdealState is = _setupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db); - ExternalView ev = - _setupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, db); - validateZoneAndTagIsolation(is, ev); - } - } - - @Test(dataProvider = "rebalanceStrategies") - public void testZoneIsolationWithInstanceTag( - String rebalanceStrategyName, String rebalanceStrategyClass) throws Exception { - List<String> testDBs = new ArrayList<String>(); - Set<String> tags = new HashSet<String>(_nodeToTagMap.values()); - int i = 0; - for (String tag : tags) { - String db = "Test-DB-Tag-" + rebalanceStrategyName + "-" + i++; - _setupTool.addResourceToCluster(CLUSTER_NAME, db, _PARTITIONS, - BuiltInStateModelDefinitions.MasterSlave.name(), RebalanceMode.FULL_AUTO + "", - rebalanceStrategyClass); - IdealState is = _setupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db); - is.setInstanceGroupTag(tag); - _setupTool.getClusterManagementTool().setResourceIdealState(CLUSTER_NAME, db, is); - _setupTool.rebalanceStorageCluster(CLUSTER_NAME, db, _replica); - testDBs.add(db); - _allDBs.add(db); - } - Thread.sleep(300); - - boolean result = ClusterStateVerifier.verifyByZkCallback( - new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME)); - Assert.assertTrue(result); - - for (String db : testDBs) { - IdealState is = _setupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db); - ExternalView ev = - _setupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, db); - validateZoneAndTagIsolation(is, ev); - } - } - - /** - * Validate instances for each partition is on different zone and with necessary tagged instances. - */ - private void validateZoneAndTagIsolation(IdealState is, ExternalView ev) { - int replica = Integer.valueOf(is.getReplicas()); - String tag = is.getInstanceGroupTag(); - - for (String partition : is.getPartitionSet()) { - Set<String> assignedZones = new HashSet<String>(); - - Map<String, String> assignmentMap = ev.getRecord().getMapField(partition); - Set<String> instancesInEV = assignmentMap.keySet(); - for (String instance : instancesInEV) { - assignedZones.add(_nodeToZoneMap.get(instance)); - if (tag != null) { - InstanceConfig config = - _setupTool.getClusterManagementTool().getInstanceConfig(CLUSTER_NAME, instance); - Assert.assertTrue(config.containsTag(tag)); - } - } - Assert.assertEquals(assignedZones.size(), replica); - } - } - - @Test() - public void testAddZone() throws Exception { - //TODO - } - - @Test() - public void testAddNodes() throws Exception { - //TODO - } - - @Test() - public void testNodeFailure() throws Exception { - //TODO - } - - @AfterClass - public void afterClass() throws Exception { - /** - * shutdown order: 1) disconnect the controller 2) disconnect participants - */ - _controller.syncStop(); - for (MockParticipantManager participant : _participants) { - participant.syncStop(); - } - _setupTool.deleteCluster(CLUSTER_NAME); - System.out.println("END " + CLASS_NAME + " at " + new Date(System.currentTimeMillis())); - } -}
