Move all options from IdealState to ResourceConfig, add Bulder for building ResourceConfig, and a new RebalanceConfig to hold all rebalance options for a resource.
Project: http://git-wip-us.apache.org/repos/asf/helix/repo Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/65cb3165 Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/65cb3165 Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/65cb3165 Branch: refs/heads/helix-0.6.x Commit: 65cb3165dd244747fed8a29f9a741486f269ad8f Parents: 998a7bd Author: Lei Xia <l...@linkedin.com> Authored: Mon Oct 31 11:06:00 2016 -0700 Committer: Lei Xia <l...@linkedin.com> Committed: Wed Feb 8 09:57:00 2017 -0800 ---------------------------------------------------------------------- .../helix/api/config/RebalanceConfig.java | 221 +++++++++++ .../java/org/apache/helix/model/IdealState.java | 6 + .../org/apache/helix/model/ResourceConfig.java | 397 ++++++++++++++++++- 3 files changed, 610 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/helix/blob/65cb3165/helix-core/src/main/java/org/apache/helix/api/config/RebalanceConfig.java ---------------------------------------------------------------------- diff --git a/helix-core/src/main/java/org/apache/helix/api/config/RebalanceConfig.java b/helix-core/src/main/java/org/apache/helix/api/config/RebalanceConfig.java new file mode 100644 index 0000000..31f6d3b --- /dev/null +++ b/helix-core/src/main/java/org/apache/helix/api/config/RebalanceConfig.java @@ -0,0 +1,221 @@ +package org.apache.helix.api.config; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.helix.ZNRecord; +import org.apache.helix.controller.rebalancer.Rebalancer; +import org.apache.helix.task.TaskRebalancer; +import org.apache.log4j.Logger; + +import java.util.HashMap; +import java.util.Map; + +/** + * Resource's rebalance configurations + */ +public class RebalanceConfig { + /** + * Configurable rebalance options of a resource + */ + public enum RebalanceConfigProperty { + REBALANCE_DELAY, + DELAY_REBALANCE_DISABLED, + REBALANCE_MODE, + REBALANCER_CLASS_NAME, + REBALANCE_TIMER_PERIOD, + REBALANCE_STRATEGY + } + + /** + * The mode used for rebalance. FULL_AUTO does both node location calculation and state + * assignment, SEMI_AUTO only does the latter, and CUSTOMIZED does neither. USER_DEFINED + * uses a Rebalancer implementation plugged in by the user. TASK designates that a + * {@link TaskRebalancer} instance should be used to rebalance this resource. + */ + public enum RebalanceMode { + FULL_AUTO, + SEMI_AUTO, + CUSTOMIZED, + USER_DEFINED, + TASK, + NONE + } + + private static final int DEFAULT_REBALANCE_DELAY = -1; + + private long _rebalanceDelay = DEFAULT_REBALANCE_DELAY; + private RebalanceMode _rebalanceMode; + private String _rebalancerClassName; + private String _rebalanceStrategy; + private Boolean _delayRebalanceDisabled; + private long _rebalanceTimerPeriod = -1; /* in seconds */ + + private static final Logger _logger = Logger.getLogger(RebalanceConfig.class.getName()); + + /** + * Instantiate from an znRecord + * + * @param znRecord + */ + public RebalanceConfig(ZNRecord znRecord) { + _rebalanceDelay = znRecord.getLongField(RebalanceConfigProperty.REBALANCE_DELAY.name(), -1); + _rebalanceMode = znRecord + .getEnumField(RebalanceConfigProperty.REBALANCE_MODE.name(), RebalanceMode.class, + RebalanceMode.NONE); + _rebalancerClassName = + znRecord.getSimpleField(RebalanceConfigProperty.REBALANCER_CLASS_NAME.name()); + _rebalanceStrategy = znRecord.getSimpleField(RebalanceConfigProperty.REBALANCE_STRATEGY.name()); + _delayRebalanceDisabled = + znRecord.getBooleanField(RebalanceConfigProperty.DELAY_REBALANCE_DISABLED.name(), false); + _rebalanceTimerPeriod = + znRecord.getLongField(RebalanceConfigProperty.REBALANCE_TIMER_PERIOD.name(), -1); + } + + /** + * Get rebalance delay (in milliseconds), default is -1 is not set. + * @return + */ + public long getRebalanceDelay() { + return _rebalanceDelay; + } + + /** + * Set the delay time (in ms) that Helix should move the partition after an instance goes offline. + * This option only takes effects when delay rebalance is enabled. + * @param rebalanceDelay + */ + public void setRebalanceDelay(long rebalanceDelay) { + this._rebalanceDelay = rebalanceDelay; + } + + public RebalanceMode getRebalanceMode() { + return _rebalanceMode; + } + + public void setRebalanceMode(RebalanceMode rebalanceMode) { + this._rebalanceMode = rebalanceMode; + } + + /** + * Get the name of the user-defined rebalancer associated with this resource + * @return the rebalancer class name, or null if none is being used + */ + public String getRebalanceClassName() { + return _rebalancerClassName; + } + + /** + * Define a custom rebalancer that implements {@link Rebalancer} + * @param rebalancerClassName the name of the custom rebalancing class + */ + public void setRebalanceClassName(String rebalancerClassName) { + this._rebalancerClassName = rebalancerClassName; + } + + /** + * Get the rebalance strategy for this resource. + * + * @return rebalance strategy, or null if not specified. + */ + public String getRebalanceStrategy() { + return _rebalanceStrategy; + } + + /** + * Specify the strategy for Helix to use to compute the partition-instance assignment, + * i,e, the custom rebalance strategy that implements {@link org.apache.helix.controller.rebalancer.strategy.RebalanceStrategy} + * + * @param rebalanceStrategy + * @return + */ + public void setRebalanceStrategy(String rebalanceStrategy) { + this._rebalanceStrategy = rebalanceStrategy; + } + + /** + * Whether the delay rebalance is disabled. By default, it is false. + * @return + */ + public Boolean isDelayRebalanceDisabled() { + return _delayRebalanceDisabled; + } + + /** + * If disabled is true, the delayed rebalance time will be ignored. + * @param delayRebalanceDisabled + */ + public void setDelayRebalanceDisabled(Boolean delayRebalanceDisabled) { + this._delayRebalanceDisabled = delayRebalanceDisabled; + } + + /** + * Get the frequency with which to rebalance + * @return the rebalancing timer period + */ + public long getRebalanceTimerPeriod() { + return _rebalanceTimerPeriod; + } + + /** + * Set the frequency with which to rebalance + * @param rebalanceTimerPeriod + */ + public void setRebalanceTimerPeriod(long rebalanceTimerPeriod) { + this._rebalanceTimerPeriod = rebalanceTimerPeriod; + } + + /** + * Generate the simple field map for RebalanceConfig. + * + * @return + */ + public Map<String, String> getSimpleFieldsMap() { + Map<String, String> simpleFieldMap = new HashMap<String, String>(); + + if (_rebalanceDelay >= 0) { + simpleFieldMap + .put(RebalanceConfigProperty.REBALANCE_DELAY.name(), String.valueOf(_rebalanceDelay)); + } + if (_rebalanceMode != null) { + simpleFieldMap.put(RebalanceConfigProperty.REBALANCE_MODE.name(), _rebalanceMode.name()); + } + if (_rebalancerClassName != null) { + simpleFieldMap.put(RebalanceConfigProperty.REBALANCER_CLASS_NAME.name(), _rebalancerClassName); + } + if (_rebalanceStrategy != null) { + simpleFieldMap.put(RebalanceConfigProperty.REBALANCE_STRATEGY.name(), _rebalanceStrategy); + } + if (_delayRebalanceDisabled != null) { + simpleFieldMap.put(RebalanceConfigProperty.DELAY_REBALANCE_DISABLED.name(), + String.valueOf(_delayRebalanceDisabled)); + } + if (_rebalanceTimerPeriod > 0) { + simpleFieldMap.put(RebalanceConfigProperty.REBALANCE_TIMER_PERIOD.name(), + String.valueOf(_rebalanceTimerPeriod)); + } + + return simpleFieldMap; + } + + public boolean isValid() { + return true; + } +} + http://git-wip-us.apache.org/repos/asf/helix/blob/65cb3165/helix-core/src/main/java/org/apache/helix/model/IdealState.java ---------------------------------------------------------------------- diff --git a/helix-core/src/main/java/org/apache/helix/model/IdealState.java b/helix-core/src/main/java/org/apache/helix/model/IdealState.java index 5ced7a6..907bd27 100644 --- a/helix-core/src/main/java/org/apache/helix/model/IdealState.java +++ b/helix-core/src/main/java/org/apache/helix/model/IdealState.java @@ -44,7 +44,9 @@ import org.apache.log4j.Logger; public class IdealState extends HelixProperty { /** * Properties that are persisted and are queryable for an ideal state + * Deprecated, use ResourceConfig.ResourceConfigProperty instead. */ + @Deprecated public enum IdealStateProperty { NUM_PARTITIONS, STATE_MODEL_DEF_REF, @@ -69,6 +71,10 @@ public class IdealState extends HelixProperty { public static final String QUERY_LIST = "PREFERENCE_LIST_QUERYS"; + /** + * Deprecated, use ResourceConfig.ResourceConfigConstants instead + */ + @Deprecated public enum IdealStateConstants { ANY_LIVEINSTANCE } http://git-wip-us.apache.org/repos/asf/helix/blob/65cb3165/helix-core/src/main/java/org/apache/helix/model/ResourceConfig.java ---------------------------------------------------------------------- diff --git a/helix-core/src/main/java/org/apache/helix/model/ResourceConfig.java b/helix-core/src/main/java/org/apache/helix/model/ResourceConfig.java index c8c7b72..616d8a2 100644 --- a/helix-core/src/main/java/org/apache/helix/model/ResourceConfig.java +++ b/helix-core/src/main/java/org/apache/helix/model/ResourceConfig.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.helix.HelixProperty; import org.apache.helix.ZNRecord; import org.apache.helix.api.config.StateTransitionTimeoutConfig; +import org.apache.helix.api.config.RebalanceConfig; import org.apache.log4j.Logger; /** @@ -36,6 +37,22 @@ public class ResourceConfig extends HelixProperty { */ public enum ResourceConfigProperty { MONITORING_DISABLED, // Resource-level config, do not create Mbean and report any status for the resource. + NUM_PARTITIONS, + STATE_MODEL_DEF_REF, + STATE_MODEL_FACTORY_NAME, + REPLICAS, + MIN_ACTIVE_REPLICAS, + MAX_PARTITIONS_PER_INSTANCE, + INSTANCE_GROUP_TAG, + HELIX_ENABLED, + RESOURCE_GROUP_NAME, + RESOURCE_TYPE, + GROUP_ROUTING_ENABLED, + EXTERNAL_VIEW_DISABLED + } + + public enum ResourceConfigConstants { + ANY_LIVEINSTANCE } private static final Logger _logger = Logger.getLogger(ResourceConfig.class.getName()); @@ -66,6 +83,64 @@ public class ResourceConfig extends HelixProperty { public ResourceConfig(ZNRecord record, String id) { super(record, id); } + + public ResourceConfig(String resourceId, Boolean monitorDisabled, int numPartitions, + String stateModelDefRef, String stateModelFactoryName, String numReplica, + int minActiveReplica, int maxPartitionsPerInstance, String instanceGroupTag, + Boolean helixEnabled, String resourceGroupName, String resourceType, + Boolean groupRoutingEnabled, Boolean externalViewDisabled, + RebalanceConfig rebalanceConfig) { + super(resourceId); + + if (monitorDisabled != null) { + _record.setBooleanField(ResourceConfigProperty.MONITORING_DISABLED.name(), monitorDisabled); + } + _record.setIntField(ResourceConfigProperty.NUM_PARTITIONS.name(), numPartitions); + _record.setSimpleField(ResourceConfigProperty.STATE_MODEL_DEF_REF.name(), stateModelDefRef); + if (stateModelFactoryName != null) { + _record.setSimpleField(ResourceConfigProperty.STATE_MODEL_FACTORY_NAME.name(), stateModelFactoryName); + } + _record.setSimpleField(ResourceConfigProperty.REPLICAS.name(), numReplica); + + if (minActiveReplica >= 0) { + _record.setIntField(ResourceConfigProperty.MIN_ACTIVE_REPLICAS.name(), minActiveReplica); + } + + if (maxPartitionsPerInstance >= 0) { + _record.setIntField(ResourceConfigProperty.MAX_PARTITIONS_PER_INSTANCE.name(), maxPartitionsPerInstance); + } + + if (instanceGroupTag != null) { + _record.setSimpleField(ResourceConfigProperty.INSTANCE_GROUP_TAG.name(), instanceGroupTag); + } + + if (helixEnabled != null) { + _record.setBooleanField(ResourceConfigProperty.HELIX_ENABLED.name(), helixEnabled); + } + + if (resourceGroupName != null) { + _record.setSimpleField(ResourceConfigProperty.RESOURCE_GROUP_NAME.name(), resourceGroupName); + } + + if (resourceType != null) { + _record.setSimpleField(ResourceConfigProperty.RESOURCE_TYPE.name(), resourceType); + } + + if (groupRoutingEnabled != null) { + _record.setBooleanField(ResourceConfigProperty.GROUP_ROUTING_ENABLED.name(), + groupRoutingEnabled); + } + + if (externalViewDisabled != null) { + _record.setBooleanField(ResourceConfigProperty.EXTERNAL_VIEW_DISABLED.name(), externalViewDisabled); + } + + if (rebalanceConfig != null) { + putSimpleConfigs(rebalanceConfig.getSimpleFieldsMap()); + } + } + + /** * Get the value of DisableMonitoring set. * @@ -75,14 +150,122 @@ public class ResourceConfig extends HelixProperty { return _record.getBooleanField(ResourceConfigProperty.MONITORING_DISABLED.toString(), false); } + + /** + * Get the associated resource + * @return the name of the resource + */ + public String getResourceName() { + return _record.getId(); + } + + /** + * Get the number of partitions of this resource + * @return the number of partitions + */ + public int getNumPartitions() { + return _record.getIntField(ResourceConfigProperty.NUM_PARTITIONS.name(), 0); + } + + /** + * Get the state model associated with this resource + * @return an identifier of the state model + */ + public String getStateModelDefRef() { + return _record.getSimpleField(ResourceConfigProperty.STATE_MODEL_DEF_REF.name()); + } + + /** + * Get the state model factory associated with this resource + * @return state model factory name + */ + public String getStateModelFactoryName() { + return _record.getSimpleField(ResourceConfigProperty.STATE_MODEL_FACTORY_NAME.name()); + } + + /** + * Get the number of replicas for each partition of this resource + * @return number of replicas (as a string) + */ + public String getNumReplica() { + // TODO: use IdealState.getNumbReplica()? + return _record.getSimpleField(ResourceConfigProperty.REPLICAS.name()); + } + + /** + * Get the number of minimal active partitions for this resource. + * + * @return + */ + public int getMinActiveReplica() { + return _record.getIntField(ResourceConfigProperty.MIN_ACTIVE_REPLICAS.name(), -1); + } + + public int getMaxPartitionsPerInstance() { + return _record.getIntField(ResourceConfigProperty.MAX_PARTITIONS_PER_INSTANCE.toString(), + Integer.MAX_VALUE); + } + + /** + * Check for a tag that will restrict assignment to instances with a matching tag + * @return the group tag, or null if none is present + */ + public String getInstanceGroupTag() { + return _record.getSimpleField(ResourceConfigProperty.INSTANCE_GROUP_TAG.toString()); + } + + /** + * Get if the resource is enabled or not + * By default, it's enabled + * @return true if enabled; false otherwise + */ + public Boolean isEnabled() { + return _record.getBooleanField(ResourceConfigProperty.HELIX_ENABLED.name(), true); + } + /** - * Set whether to disable monitoring for this resource. + * Get the resource type + * @return the resource type, or null if none is being set + */ + public String getResourceType() { + return _record.getSimpleField(ResourceConfigProperty.RESOURCE_TYPE.name()); + } + + /** + * Get the resource group name * - * @param monitoringDisabled whether to disable monitoring for this resource. + * @return + */ + public String getResourceGroupName() { + return _record.getSimpleField(ResourceConfigProperty.RESOURCE_GROUP_NAME.name()); + } + + /** + * Get if the resource group routing feature is enabled or not + * By default, it's disabled + * + * @return true if enabled; false otherwise + */ + public Boolean isGroupRoutingEnabled() { + return _record.getBooleanField(ResourceConfigProperty.GROUP_ROUTING_ENABLED.name(), false); + } + + /** + * If the external view for this resource is disabled. by default, it is false. + * + * @return true if the external view should be disabled for this resource. + */ + public Boolean isExternalViewDisabled() { + return _record.getBooleanField(ResourceConfigProperty.EXTERNAL_VIEW_DISABLED.name(), false); + } + + /** + * Get rebalance config for this resource. + * @return */ - public void setMonitoringDisabled(boolean monitoringDisabled) { - _record - .setBooleanField(ResourceConfigProperty.MONITORING_DISABLED.toString(), monitoringDisabled); + public RebalanceConfig getRebalanceConfig() { + RebalanceConfig rebalanceConfig = new RebalanceConfig(_record); + return rebalanceConfig; } // TODO: Move it to constructor and Builder when the logic merged in @@ -197,17 +380,203 @@ public class ResourceConfig extends HelixProperty { return getId().hashCode(); } - /** - * Get the name of this resource - * - * @return the instance name - */ - public String getResourceName() { - return _record.getId(); - } - @Override public boolean isValid() { return true; } + + + public class Builder { + private String _resourceId; + private Boolean _monitorDisabled; + private int _numPartitions; + private String _stateModelDefRef; + private String _stateModelFactoryName; + private String _numReplica; + private int _minActiveReplica = -1; + private int _maxPartitionsPerInstance = -1; + private String _instanceGroupTag; + private Boolean _helixEnabled; + private String _resourceGroupName; + private String _resourceType; + private Boolean _groupRoutingEnabled; + private Boolean _externalViewDisabled; + private RebalanceConfig _rebalanceConfig; + + public Builder(String resourceId) { + _resourceId = resourceId; + } + + public Builder setMonitorDisabled(boolean monitorDisabled) { + _monitorDisabled = monitorDisabled; + return this; + } + + public Boolean isMonitorDisabled() { + return _monitorDisabled; + } + + public String getResourceId() { + return _resourceId; + } + + public int getNumPartitions() { + return _numPartitions; + } + + public Builder setNumPartitions(int numPartitions) { + _numPartitions = numPartitions; + return this; + } + + public String getStateModelDefRef() { + return _stateModelDefRef; + } + + public Builder setStateModelDefRef(String stateModelDefRef) { + _stateModelDefRef = stateModelDefRef; + return this; + } + + public String getStateModelFactoryName() { + return _stateModelFactoryName; + } + + public Builder setStateModelFactoryName(String stateModelFactoryName) { + _stateModelFactoryName = stateModelFactoryName; + return this; + } + + public String getNumReplica() { + return _numReplica; + } + + public Builder setNumReplica(String numReplica) { + _numReplica = numReplica; + return this; + } + + public Builder setNumReplica(int numReplica) { + return setNumReplica(String.valueOf(numReplica)); + } + + public int getMinActiveReplica() { + return _minActiveReplica; + } + + public Builder setMinActiveReplica(int minActiveReplica) { + _minActiveReplica = minActiveReplica; + return this; + } + + public int getMaxPartitionsPerInstance() { + return _maxPartitionsPerInstance; + } + + public Builder setMaxPartitionsPerInstance(int maxPartitionsPerInstance) { + _maxPartitionsPerInstance = maxPartitionsPerInstance; + return this; + } + + public String getInstanceGroupTag() { + return _instanceGroupTag; + } + + public Builder setInstanceGroupTag(String instanceGroupTag) { + _instanceGroupTag = instanceGroupTag; + return this; + } + + public Boolean isHelixEnabled() { + return _helixEnabled; + } + + public Builder setHelixEnabled(boolean helixEnabled) { + _helixEnabled = helixEnabled; + return this; + } + + public String getResourceType() { + return _resourceType; + } + + public Builder setResourceType(String resourceType) { + _resourceType = resourceType; + return this; + } + + public String getResourceGroupName() { + return _resourceGroupName; + } + + public Builder setResourceGroupName(String resourceGroupName) { + _resourceGroupName = resourceGroupName; + return this; + } + + public Boolean isGroupRoutingEnabled() { + return _groupRoutingEnabled; + } + + public Builder setGroupRoutingEnabled(boolean groupRoutingEnabled) { + _groupRoutingEnabled = groupRoutingEnabled; + return this; + } + + public Boolean isExternalViewDisabled() { + return _externalViewDisabled; + } + + public Builder setExternalViewDisabled(boolean externalViewDisabled) { + _externalViewDisabled = externalViewDisabled; + return this; + } + + public Builder setRebalanceConfig(RebalanceConfig rebalanceConfig) { + _rebalanceConfig = rebalanceConfig; + return this; + } + + public RebalanceConfig getRebalanceConfig() { + return _rebalanceConfig; + } + + private void validate() { + if (_rebalanceConfig == null) { + throw new IllegalArgumentException("RebalanceConfig not set!"); + } else { + if (_rebalanceConfig.isValid()) { + throw new IllegalArgumentException("Invalid RebalanceConfig!"); + } + } + if (_numPartitions <= 0) { + throw new IllegalArgumentException("Invalid number of partitions!"); + } + + if (_stateModelDefRef == null) { + throw new IllegalArgumentException("State Model Definition Reference is not set!"); + } + + if (_numReplica == null) { + throw new IllegalArgumentException("Number of replica is not set!"); + } else { + if (!_numReplica.equals(ResourceConfigConstants.ANY_LIVEINSTANCE.name())) { + try { + Integer.parseInt(_numReplica); + } catch (NumberFormatException ex) { + throw new IllegalArgumentException("Invalid number of replica!"); + } + } + } + } + + public ResourceConfig build() { + validate(); + + return new ResourceConfig(_resourceId, _monitorDisabled, _numPartitions, _stateModelDefRef, + _stateModelFactoryName, _numReplica, _minActiveReplica, _maxPartitionsPerInstance, + _instanceGroupTag, _helixEnabled, _resourceGroupName, _resourceType, _groupRoutingEnabled, + _externalViewDisabled, _rebalanceConfig); + } + } }
