This is an automated email from the ASF dual-hosted git repository. jiajunwang pushed a commit to branch helix-0.9.x in repository https://gitbox.apache.org/repos/asf/helix.git
commit bc9e33f37f0a52f9916ac4330ae923393032203b Author: Ali Reza Zamani Zadeh Najari <[email protected]> AuthorDate: Tue Nov 26 15:26:10 2019 -0800 Add java API to create cluster with CloudConfig In this commit the below APIs have been added. 1- AddCluster with CloudCOnfig API. 2- addCloudConfig to existing cluster. 3- Update CloudConfig to update existing Cloud Config. Several tests have been added to test these APIs. --- .../main/java/org/apache/helix/ConfigAccessor.java | 4 +- .../src/main/java/org/apache/helix/HelixAdmin.java | 14 ++ .../org/apache/helix/manager/zk/ZKHelixAdmin.java | 28 ++++ .../java/org/apache/helix/model/CloudConfig.java | 149 +++++++-------------- .../java/org/apache/helix/tools/ClusterSetup.java | 17 ++- .../java/org/apache/helix/TestConfigAccessor.java | 66 +++++++-- .../apache/helix/manager/zk/TestZkHelixAdmin.java | 69 ++++++++++ .../java/org/apache/helix/mock/MockHelixAdmin.java | 11 ++ .../apache/helix/model/cloud/TestCloudConfig.java | 60 ++++++--- .../org/apache/helix/tools/TestClusterSetup.java | 88 +++++++++++- 10 files changed, 368 insertions(+), 138 deletions(-) diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java index 9bd84d1..d61b4ee 100644 --- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java +++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java @@ -569,7 +569,7 @@ public class ConfigAccessor { * @return The instance of {@link CloudConfig} */ public CloudConfig getCloudConfig(String clusterName) { - if (!ZKUtil.isClusterSetup(clusterName, zkClient)) { + if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { throw new HelixException( String.format("Failed to get config. cluster: %s is not setup.", clusterName)); } @@ -582,7 +582,7 @@ public class ConfigAccessor { return null; } - return new CloudConfig(record); + return new CloudConfig.Builder(record).build(); } /** diff --git a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java index 7402c19..bb5f3bf 100644 --- a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java +++ b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java @@ -22,6 +22,7 @@ package org.apache.helix; import java.io.IOException; import java.util.List; import java.util.Map; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.ClusterConstraints; import org.apache.helix.model.ClusterConstraints.ConstraintType; import org.apache.helix.model.ConstraintItem; @@ -380,6 +381,19 @@ public interface HelixAdmin { void dropResource(String clusterName, String resourceName); /** + * Add cloud config to the cluster. + * @param clusterName + * @param cloudConfig + */ + void addCloudConfig(String clusterName, CloudConfig cloudConfig); + + /** + * Remove the Cloud Config for specific cluster + * @param clusterName + */ + void removeCloudConfig(String clusterName); + + /** * Get a list of state model definitions in a cluster * @param clusterName * @return diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java index af0035a..8a0d72c 100644 --- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java +++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java @@ -58,6 +58,7 @@ import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy; import org.apache.helix.controller.rebalancer.strategy.RebalanceStrategy; import org.apache.helix.manager.zk.client.HelixZkClient; import org.apache.helix.manager.zk.client.SharedZkClientFactory; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.ClusterConfig; import org.apache.helix.model.ClusterConstraints; import org.apache.helix.model.ClusterConstraints.ConstraintType; @@ -1019,6 +1020,33 @@ public class ZKHelixAdmin implements HelixAdmin { } @Override + public void addCloudConfig(String clusterName, CloudConfig cloudConfig) { + logger.info("Add CloudConfig to cluster {}, CloudConfig is {}.", clusterName, + cloudConfig.toString()); + + if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { + throw new HelixException("cluster " + clusterName + " is not setup yet"); + } + + CloudConfig.Builder builder = new CloudConfig.Builder(cloudConfig); + CloudConfig cloudConfigBuilder = builder.build(); + + ZKHelixDataAccessor accessor = + new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); + Builder keyBuilder = accessor.keyBuilder(); + accessor.setProperty(keyBuilder.cloudConfig(), cloudConfigBuilder); + } + + @Override + public void removeCloudConfig(String clusterName) { + logger.info("Remove Cloud Config for cluster {}.", clusterName); + HelixDataAccessor accessor = + new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); + Builder keyBuilder = accessor.keyBuilder(); + accessor.removeProperty(keyBuilder.cloudConfig()); + } + + @Override public List<String> getStateModelDefs(String clusterName) { return _zkClient.getChildren(PropertyPathBuilder.stateModelDef(clusterName)); } diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java index c8ab6eb..f6c279c 100644 --- a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java +++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java @@ -30,6 +30,9 @@ import org.apache.helix.cloud.constants.CloudProvider; * Cloud configurations */ public class CloudConfig extends HelixProperty { + + public static final String CLOUD_CONFIG_KW = "CloudConfig"; + /** * Configurable characteristics of a cloud. * NOTE: Do NOT use this field name directly, use its corresponding getter/setter in the @@ -52,39 +55,22 @@ public class CloudConfig extends HelixProperty { /** * Instantiate the CloudConfig for the cloud - * @param cluster */ - public CloudConfig(String cluster) { - super(cluster); + private CloudConfig() { + super(CLOUD_CONFIG_KW); } /** - * Instantiate with a pre-populated record - * @param record a ZNRecord corresponding to a cloud configuration + * The constructor from the ZNRecord. + * @param record */ - public CloudConfig(ZNRecord record) { - super(record); + private CloudConfig(ZNRecord record) { + super(CLOUD_CONFIG_KW); + _record.setSimpleFields(record.getSimpleFields()); + _record.setListFields(record.getListFields()); + _record.setMapFields(record.getMapFields()); } - /** - * Instantiate the config using each field individually. - * Users should use CloudConfig.Builder to create CloudConfig. - * @param cluster - * @param enabled - * @param cloudID - */ - public CloudConfig(String cluster, boolean enabled, CloudProvider cloudProvider, String cloudID, - List<String> cloudInfoSource, String cloudProcessorName) { - super(cluster); - _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), enabled); - _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name()); - _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID); - if (cloudProvider.equals(CloudProvider.CUSTOMIZED)) { - _record - .setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), cloudProcessorName); - _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSource); - } - } /** * Enable/Disable the CLOUD_ENABLED field. @@ -135,15 +121,6 @@ public class CloudConfig extends HelixProperty { } /** - * Set the CLOUD_INFO_PROCESSOR_NAME field. - * @param cloudInfoProcessorName - */ - public void setCloudInfoFProcessorName(String cloudInfoProcessorName) { - _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), - cloudInfoProcessorName); - } - - /** * Get the CLOUD_INFO_PROCESSOR_NAME field. * @return CLOUD_INFO_PROCESSOR_NAME field. */ @@ -152,14 +129,6 @@ public class CloudConfig extends HelixProperty { } /** - * Set the CLOUD_PROVIDER field. - * @param cloudProvider - */ - public void setCloudProvider(CloudProvider cloudProvider) { - _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name()); - } - - /** * Get the CLOUD_PROVIDER field. * @return CLOUD_PROVIDER field. */ @@ -167,32 +136,28 @@ public class CloudConfig extends HelixProperty { return _record.getSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name()); } + public static class Builder { - private String _clusterName = null; - private CloudProvider _cloudProvider; - private boolean _cloudEnabled = DEFAULT_CLOUD_ENABLED; - private String _cloudID; - private List<String> _cloudInfoSources; - private String _cloudInfoProcessorName; + private ZNRecord _record; public CloudConfig build() { validate(); - return new CloudConfig(_clusterName, _cloudEnabled, _cloudProvider, _cloudID, - _cloudInfoSources, _cloudInfoProcessorName); + return new CloudConfig(_record); } /** * Default constructor */ public Builder() { + _record = new ZNRecord(CLOUD_CONFIG_KW); } /** - * Constructor with Cluster Name as input - * @param clusterName + * Instantiate with a pre-populated record + * @param record a ZNRecord corresponding to a cloud configuration */ - public Builder(String clusterName) { - _clusterName = clusterName; + public Builder(ZNRecord record) { + _record = record; } /** @@ -200,89 +165,75 @@ public class CloudConfig extends HelixProperty { * @param cloudConfig */ public Builder(CloudConfig cloudConfig) { - _cloudEnabled = cloudConfig.isCloudEnabled(); - _cloudProvider = CloudProvider.valueOf(cloudConfig.getCloudProvider()); - _cloudID = cloudConfig.getCloudID(); - _cloudInfoSources = cloudConfig.getCloudInfoSources(); - _cloudInfoProcessorName = cloudConfig.getCloudInfoProcessorName(); - } - - public Builder setClusterName(String v) { - _clusterName = v; - return this; + _record = cloudConfig.getRecord(); } public Builder setCloudEnabled(boolean isEnabled) { - _cloudEnabled = isEnabled; + _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), isEnabled); return this; } public Builder setCloudProvider(CloudProvider cloudProvider) { - _cloudProvider = cloudProvider; + _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name()); return this; } - public Builder setCloudID(String v) { - _cloudID = v; + public Builder setCloudID(String cloudID) { + _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID); return this; } - public Builder setCloudInfoSources(List<String> v) { - _cloudInfoSources = v; + public Builder setCloudInfoSources(List<String> cloudInfoSources) { + _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSources); return this; } - public Builder addCloudInfoSource(String v) { - if (_cloudInfoSources == null) { - _cloudInfoSources = new ArrayList<String>(); + public Builder addCloudInfoSource(String cloudInfoSource) { + if (_record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name()) == null) { + _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), new ArrayList<String>()); } - _cloudInfoSources.add(v); + List<String> cloudInfoSourcesList = _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name()); + cloudInfoSourcesList.add(cloudInfoSource); + _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSourcesList); return this; } - public Builder setCloudInfoProcessorName(String v) { - _cloudInfoProcessorName = v; + public Builder setCloudInfoProcessorName(String cloudInfoProcessorName) { + _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), + cloudInfoProcessorName); return this; } - public String getClusterName() { - return _clusterName; - } - - public CloudProvider getCloudProvider() { - return _cloudProvider; + public String getCloudProvider() { + return _record.getSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name()); } public boolean getCloudEnabled() { - return _cloudEnabled; + return _record.getBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), + DEFAULT_CLOUD_ENABLED); } public String getCloudID() { - return _cloudID; + return _record.getSimpleField(CloudConfigProperty.CLOUD_ID.name()); } public List<String> getCloudInfoSources() { - return _cloudInfoSources; + return _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name()); } public String getCloudInfoProcessorName() { - return _cloudInfoProcessorName; + return _record.getSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name()); } private void validate() { - if (_cloudEnabled) { - if (_cloudID == null) { - throw new HelixException( - "This Cloud Configuration is Invalid. The CloudID is missing from the config."); - } - if (_cloudProvider == null) { + if (this.getCloudProvider() == null) { + throw new HelixException( + "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config."); + } else if (this.getCloudProvider().equals(CloudProvider.CUSTOMIZED.name())) { + if (this.getCloudInfoProcessorName() == null || this.getCloudInfoSources() == null + || this.getCloudInfoSources().size() == 0) { throw new HelixException( - "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config."); - } else if (_cloudProvider == CloudProvider.CUSTOMIZED) { - if (_cloudInfoProcessorName == null || _cloudInfoSources == null || _cloudInfoSources.size() == 0) { - throw new HelixException( - "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources"); - } + "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources"); } } } diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java index 677c1b4..27d8c88 100644 --- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java +++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java @@ -40,7 +40,9 @@ import org.apache.helix.HelixAdmin; import org.apache.helix.HelixConstants; import org.apache.helix.HelixException; import org.apache.helix.PropertyKey.Builder; +import org.apache.helix.SystemPropertyKeys; import org.apache.helix.ZNRecord; +import org.apache.helix.cloud.constants.CloudProvider; import org.apache.helix.manager.zk.ZKHelixAdmin; import org.apache.helix.manager.zk.ZKHelixDataAccessor; import org.apache.helix.manager.zk.ZNRecordSerializer; @@ -48,6 +50,7 @@ import org.apache.helix.manager.zk.ZkBaseDataAccessor; import org.apache.helix.manager.zk.client.HelixZkClient; import org.apache.helix.manager.zk.client.SharedZkClientFactory; import org.apache.helix.model.BuiltInStateModelDefinitions; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.ClusterConfig; import org.apache.helix.model.ClusterConstraints; import org.apache.helix.model.ClusterConstraints.ConstraintType; @@ -158,15 +161,23 @@ public class ClusterSetup { _admin = zkHelixAdmin; } - public void addCluster(String clusterName, boolean overwritePrevious) { + public void addCluster(String clusterName, boolean overwritePrevious, CloudConfig cloudConfig) + throws HelixException { _admin.addCluster(clusterName, overwritePrevious); - for (BuiltInStateModelDefinitions def : BuiltInStateModelDefinitions.values()) { addStateModelDef(clusterName, def.getStateModelDefinition().getId(), - def.getStateModelDefinition(), overwritePrevious); + def.getStateModelDefinition(), overwritePrevious); + } + + if (cloudConfig != null) { + _admin.addCloudConfig(clusterName, cloudConfig); } } + public void addCluster(String clusterName, boolean overwritePrevious) { + addCluster(clusterName, overwritePrevious, null); + } + public void activateCluster(String clusterName, String grandCluster, boolean enable) { if (enable) { _admin.addClusterToGrandCluster(clusterName, grandCluster); diff --git a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java index f4377d6..d59beb6 100644 --- a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java +++ b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java @@ -19,13 +19,17 @@ package org.apache.helix; * under the License. */ +import java.util.ArrayList; import java.util.Date; import java.util.List; +import org.apache.helix.cloud.constants.CloudProvider; import org.apache.helix.manager.zk.ZKHelixAdmin; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.ConfigScope; import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty; import org.apache.helix.model.InstanceConfig; import org.apache.helix.model.builder.ConfigScopeBuilder; +import org.apache.helix.tools.ClusterSetup; import org.testng.Assert; import org.testng.annotations.Test; @@ -60,9 +64,8 @@ public class TestConfigAccessor extends ZkUnitTestBase { Assert.assertEquals(resourceConfigValue, "resourceConfigValue"); // partition scope config - ConfigScope partitionScope = - new ConfigScopeBuilder().forCluster(clusterName).forResource("testResource") - .forPartition("testPartition").build(); + ConfigScope partitionScope = new ConfigScopeBuilder().forCluster(clusterName) + .forResource("testResource").forPartition("testPartition").build(); configAccessor.set(partitionScope, "partitionConfigKey", "partitionConfigValue"); String partitionConfigValue = configAccessor.get(partitionScope, "partitionConfigKey"); Assert.assertEquals(partitionConfigValue, "partitionConfigValue"); @@ -105,9 +108,8 @@ public class TestConfigAccessor extends ZkUnitTestBase { "should be [HELIX_ENABLED, HELIX_ENABLED_TIMESTAMP, HELIX_HOST, HELIX_PORT, participantConfigKey]"); Assert.assertEquals(keys.get(4), "participantConfigKey"); - keys = - configAccessor.getKeys(ConfigScopeProperty.PARTITION, clusterName, "testResource", - "testPartition"); + keys = configAccessor.getKeys(ConfigScopeProperty.PARTITION, clusterName, "testResource", + "testPartition"); Assert.assertEquals(keys.size(), 1, "should be [partitionConfigKey]"); Assert.assertEquals(keys.get(0), "partitionConfigKey"); @@ -173,8 +175,8 @@ public class TestConfigAccessor extends ZkUnitTestBase { try { configAccessor.set(participantScope, "participantConfigKey", "participantConfigValue"); - Assert - .fail("Except fail to set participant-config because participant: localhost_12918 is not added to cluster yet"); + Assert.fail( + "Except fail to set participant-config because participant: localhost_12918 is not added to cluster yet"); } catch (HelixException e) { // OK } @@ -183,8 +185,8 @@ public class TestConfigAccessor extends ZkUnitTestBase { try { configAccessor.set(participantScope, "participantConfigKey", "participantConfigValue"); } catch (Exception e) { - Assert - .fail("Except succeed to set participant-config because participant: localhost_12918 has been added to cluster"); + Assert.fail( + "Except succeed to set participant-config because participant: localhost_12918 has been added to cluster"); } String participantConfigValue = configAccessor.get(participantScope, "participantConfigKey"); @@ -193,4 +195,48 @@ public class TestConfigAccessor extends ZkUnitTestBase { admin.dropCluster(clusterName); System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis())); } + + @Test + public void testUpdateCloudConfig() throws Exception { + ClusterSetup _clusterSetup = new ClusterSetup(ZK_ADDR); + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String clusterName = className + "_" + methodName; + + CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder(); + cloudConfigInitBuilder.setCloudEnabled(true); + cloudConfigInitBuilder.setCloudID("TestCloudID"); + List<String> sourceList = new ArrayList<String>(); + sourceList.add("TestURL"); + cloudConfigInitBuilder.setCloudInfoSources(sourceList); + cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor"); + cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED); + CloudConfig cloudConfigInit = cloudConfigInitBuilder.build(); + + _clusterSetup.addCluster(clusterName, false, cloudConfigInit); + + // Read CloudConfig from Zookeeper and check the content + ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient); + CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID"); + List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + Assert.assertEquals(listUrlFromZk.get(0), "TestURL"); + Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor"); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name()); + + // Change the processor name and check if processor name has been changed in Zookeeper. + cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor2"); + cloudConfigInit = cloudConfigInitBuilder.build(); + ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient); + admin.addCloudConfig(clusterName, cloudConfigInit); + + cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID"); + listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + Assert.assertEquals(listUrlFromZk.get(0), "TestURL"); + Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor2"); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name()); + } } diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java index d372d67..f7fe23c 100644 --- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java +++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.helix.BaseDataAccessor; +import org.apache.helix.ConfigAccessor; import org.apache.helix.HelixAdmin; import org.apache.helix.HelixDataAccessor; import org.apache.helix.HelixException; @@ -38,7 +39,9 @@ import org.apache.helix.PropertyType; import org.apache.helix.TestHelper; import org.apache.helix.ZNRecord; import org.apache.helix.ZkUnitTestBase; +import org.apache.helix.cloud.constants.CloudProvider; import org.apache.helix.examples.MasterSlaveStateModelFactory; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.ClusterConstraints; import org.apache.helix.model.ClusterConstraints.ConstraintAttribute; import org.apache.helix.model.ClusterConstraints.ConstraintType; @@ -505,4 +508,70 @@ public class TestZkHelixAdmin extends ZkUnitTestBase { .getListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name()).size(), 2); } + + @Test + public void testAddCloudConfig() { + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String clusterName = className + "_" + methodName; + + HelixAdmin admin = new ZKHelixAdmin(_gZkClient); + admin.addCluster(clusterName, true); + + CloudConfig.Builder builder = new CloudConfig.Builder(); + builder.setCloudEnabled(true); + builder.setCloudID("TestID"); + builder.addCloudInfoSource("TestURL"); + builder.setCloudProvider(CloudProvider.CUSTOMIZED); + builder.setCloudInfoProcessorName("TestProcessor"); + CloudConfig cloudConfig = builder.build(); + + admin.addCloudConfig(clusterName, cloudConfig); + + // Read CloudConfig from Zookeeper and check the content + ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient); + CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID"); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name()); + List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + Assert.assertEquals(listUrlFromZk.get(0), "TestURL"); + Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor"); + } + + + @Test + public void testRemoveCloudConfig() throws Exception { + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String clusterName = className + "_" + methodName; + + HelixAdmin admin = new ZKHelixAdmin(_gZkClient); + admin.addCluster(clusterName, true); + + CloudConfig.Builder builder = new CloudConfig.Builder(); + builder.setCloudEnabled(true); + builder.setCloudID("TestID"); + builder.addCloudInfoSource("TestURL"); + builder.setCloudProvider(CloudProvider.CUSTOMIZED); + builder.setCloudInfoProcessorName("TestProcessor"); + CloudConfig cloudConfig = builder.build(); + + admin.addCloudConfig(clusterName, cloudConfig); + + // Read CloudConfig from Zookeeper and check the content + ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient); + CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID"); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name()); + List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + Assert.assertEquals(listUrlFromZk.get(0), "TestURL"); + Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor"); + + // Remove Cloud Config and make sure it has been removed from Zookeeper + admin.removeCloudConfig(clusterName); + cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertNull(cloudConfigFromZk); + } } diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java index 6cb7790..e06c902 100644 --- a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java +++ b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java @@ -30,6 +30,7 @@ import org.apache.helix.HelixManager; import org.apache.helix.PropertyPathBuilder; import org.apache.helix.PropertyType; import org.apache.helix.ZNRecord; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.ClusterConfig; import org.apache.helix.model.ClusterConstraints; import org.apache.helix.model.ConstraintItem; @@ -312,6 +313,16 @@ public class MockHelixAdmin implements HelixAdmin { } + @Override + public void addCloudConfig(String clusterName, CloudConfig cloudConfig) { + + } + + @Override + public void removeCloudConfig(String clusterName) { + + } + @Override public List<String> getStateModelDefs(String clusterName) { return null; } diff --git a/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java index ee83011..01945de 100644 --- a/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java +++ b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java @@ -57,20 +57,22 @@ public class TestCloudConfig extends ZkUnitTestBase { } @Test(dependsOnMethods = "testCloudConfigNull") - public void testCloudConfig() { + public void testCloudConfig() throws Exception { String className = getShortClassName(); String clusterName = "CLUSTER_" + className; TestHelper.setupEmptyCluster(_gZkClient, clusterName); // Create dummy CloudConfig object - CloudConfig cloudConfig = new CloudConfig(clusterName); - cloudConfig.setCloudEnabled(true); - cloudConfig.setCloudProvider(CloudProvider.AZURE); - cloudConfig.setCloudID("TestID"); + CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder(); + cloudConfigBuilder.setCloudEnabled(true); + cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE); + cloudConfigBuilder.setCloudID("TestID"); List<String> infoURL = new ArrayList<String>(); infoURL.add("TestURL"); - cloudConfig.setCloudInfoSource(infoURL); - cloudConfig.setCloudInfoFProcessorName("TestProcessor"); + cloudConfigBuilder.setCloudInfoSources(infoURL); + cloudConfigBuilder.setCloudInfoProcessorName("TestProcessor"); + + CloudConfig cloudConfig = cloudConfigBuilder.build(); // Write the CloudConfig to Zookeeper ZKHelixDataAccessor accessor = @@ -84,8 +86,8 @@ public class TestCloudConfig extends ZkUnitTestBase { Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name()); Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID"); + Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().size(), 1); - Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().get(0), "TestURL"); Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor"); } @@ -93,7 +95,7 @@ public class TestCloudConfig extends ZkUnitTestBase { public void testUnverifiedCloudConfigBuilder() { String className = getShortClassName(); String clusterName = "CLUSTER_" + className; - CloudConfig.Builder builder = new CloudConfig.Builder(clusterName); + CloudConfig.Builder builder = new CloudConfig.Builder(); builder.setCloudEnabled(true); // Verify will fail because cloudID has net been defined. CloudConfig cloudConfig = builder.build(); @@ -103,7 +105,7 @@ public class TestCloudConfig extends ZkUnitTestBase { public void testUnverifiedCloudConfigBuilderEmptySources() { String className = getShortClassName(); String clusterName = "CLUSTER_" + className; - CloudConfig.Builder builder = new CloudConfig.Builder(clusterName); + CloudConfig.Builder builder = new CloudConfig.Builder(); builder.setCloudEnabled(true); builder.setCloudProvider(CloudProvider.CUSTOMIZED); builder.setCloudID("TestID"); @@ -115,9 +117,7 @@ public class TestCloudConfig extends ZkUnitTestBase { @Test(expectedExceptions = HelixException.class) public void testUnverifiedCloudConfigBuilderWithoutProcessor() { - String className = getShortClassName(); - String clusterName = "CLUSTER_" + className; - CloudConfig.Builder builder = new CloudConfig.Builder(clusterName); + CloudConfig.Builder builder = new CloudConfig.Builder(); builder.setCloudEnabled(true); builder.setCloudProvider(CloudProvider.CUSTOMIZED); builder.setCloudID("TestID"); @@ -132,7 +132,7 @@ public class TestCloudConfig extends ZkUnitTestBase { String className = getShortClassName(); String clusterName = "CLUSTER_" + className; TestHelper.setupEmptyCluster(_gZkClient, clusterName); - CloudConfig.Builder builder = new CloudConfig.Builder(clusterName); + CloudConfig.Builder builder = new CloudConfig.Builder(); builder.setCloudEnabled(true); builder.setCloudProvider(CloudProvider.CUSTOMIZED); builder.setCloudID("TestID"); @@ -142,8 +142,7 @@ public class TestCloudConfig extends ZkUnitTestBase { // Check builder getter methods Assert.assertTrue(builder.getCloudEnabled()); - Assert.assertEquals(builder.getCloudProvider(), CloudProvider.CUSTOMIZED); - Assert.assertEquals(builder.getClusterName(), clusterName); + Assert.assertEquals(builder.getCloudProvider(), CloudProvider.CUSTOMIZED.name()); Assert.assertEquals(builder.getCloudID(), "TestID"); List<String> listUrlFromBuilder = builder.getCloudInfoSources(); Assert.assertEquals(listUrlFromBuilder.size(), 2); @@ -175,15 +174,14 @@ public class TestCloudConfig extends ZkUnitTestBase { String className = getShortClassName(); String clusterName = "CLUSTER_" + className; TestHelper.setupEmptyCluster(_gZkClient, clusterName); - CloudConfig.Builder builder = new CloudConfig.Builder(clusterName); + CloudConfig.Builder builder = new CloudConfig.Builder(); builder.setCloudEnabled(true); builder.setCloudProvider(CloudProvider.AZURE); builder.setCloudID("TestID"); - builder.setCloudInfoProcessorName("TestProcessor"); // Check builder getter methods Assert.assertTrue(builder.getCloudEnabled()); - Assert.assertEquals(builder.getCloudProvider(), CloudProvider.AZURE); + Assert.assertEquals(builder.getCloudProvider(), CloudProvider.AZURE.name()); CloudConfig cloudConfig = builder.build(); @@ -198,7 +196,29 @@ public class TestCloudConfig extends ZkUnitTestBase { Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name()); - // Since CloudProvider is not CUSTOMIZED, CloudInfoProcessor will be null. + // Since user does not set the CloudInfoProcessorName, this field will be null. Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName()); + + // Checking the set method in CloudConfig + cloudConfig.setCloudEnabled(false); + accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig); + cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertFalse(cloudConfigFromZk.isCloudEnabled()); + + cloudConfig.setCloudEnabled(true); + cloudConfig.setCloudID("TestID2"); + List<String> sourceList = new ArrayList<String>(); + sourceList.add("TestURL0"); + sourceList.add("TestURL1"); + cloudConfig.setCloudInfoSource(sourceList); + accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig); + + cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID2"); + List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + Assert.assertEquals(listUrlFromZk.get(0), "TestURL0"); + Assert.assertEquals(listUrlFromZk.get(1), "TestURL1"); } } diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java index 84d5f51..6a6d48d 100644 --- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java +++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java @@ -19,9 +19,12 @@ package org.apache.helix.tools; * under the License. */ +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.List; import org.apache.helix.BaseDataAccessor; +import org.apache.helix.ConfigAccessor; import org.apache.helix.HelixDataAccessor; import org.apache.helix.HelixException; import org.apache.helix.PropertyKey; @@ -30,15 +33,15 @@ import org.apache.helix.PropertyPathBuilder; import org.apache.helix.TestHelper; import org.apache.helix.ZNRecord; import org.apache.helix.ZkUnitTestBase; +import org.apache.helix.cloud.constants.CloudProvider; import org.apache.helix.manager.zk.ZKHelixAdmin; import org.apache.helix.manager.zk.ZKHelixDataAccessor; import org.apache.helix.manager.zk.ZNRecordSerializer; import org.apache.helix.manager.zk.ZkBaseDataAccessor; +import org.apache.helix.model.CloudConfig; import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty; import org.apache.helix.model.IdealState; import org.apache.helix.model.LiveInstance; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.AssertJUnit; import org.testng.annotations.AfterClass; @@ -47,8 +50,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; public class TestClusterSetup extends ZkUnitTestBase { - private static Logger LOG = LoggerFactory.getLogger(TestClusterSetup.class); - protected static final String CLUSTER_NAME = "TestClusterSetup"; protected static final String TEST_DB = "TestDB"; protected static final String INSTANCE_PREFIX = "instance_"; @@ -435,4 +436,83 @@ public class TestClusterSetup extends ZkUnitTestBase { System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis())); } + + @Test(expectedExceptions = HelixException.class) + public void testAddClusterWithInvalidCloudConfig() throws Exception { + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String clusterName = className + "_" + methodName; + + CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder(); + cloudConfigInitBuilder.setCloudEnabled(true); + List<String> sourceList = new ArrayList<String>(); + sourceList.add("TestURL"); + cloudConfigInitBuilder.setCloudInfoSources(sourceList); + cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED); + + CloudConfig cloudConfigInit = cloudConfigInitBuilder.build(); + + + // Since setCloudInfoProcessorName is missing, this add cluster call will throw an exception + _clusterSetup.addCluster(clusterName, false, cloudConfigInit); + } + + @Test(dependsOnMethods = "testAddClusterWithInvalidCloudConfig") + public void testAddClusterWithValidCloudConfig() throws Exception { + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String clusterName = className + "_" + methodName; + + CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder(); + cloudConfigInitBuilder.setCloudEnabled(true); + cloudConfigInitBuilder.setCloudID("TestID"); + List<String> sourceList = new ArrayList<String>(); + sourceList.add("TestURL"); + cloudConfigInitBuilder.setCloudInfoSources(sourceList); + cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessorName"); + cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED); + + CloudConfig cloudConfigInit = cloudConfigInitBuilder.build(); + + _clusterSetup.addCluster(clusterName, false, cloudConfigInit); + + // Read CloudConfig from Zookeeper and check the content + ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient); + CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID"); + List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + Assert.assertEquals(listUrlFromZk.get(0), "TestURL"); + Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessorName"); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name()); + } + + + @Test(dependsOnMethods = "testAddClusterWithValidCloudConfig") + public void testAddClusterAzureProvider() throws Exception { + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String clusterName = className + "_" + methodName; + + CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder(); + cloudConfigInitBuilder.setCloudEnabled(true); + cloudConfigInitBuilder.setCloudID("TestID"); + cloudConfigInitBuilder.setCloudProvider(CloudProvider.AZURE); + + CloudConfig cloudConfigInit = cloudConfigInitBuilder.build(); + + _clusterSetup.addCluster(clusterName, false, cloudConfigInit); + + // Read CloudConfig from Zookeeper and check the content + ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient); + CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName); + Assert.assertTrue(cloudConfigFromZk.isCloudEnabled()); + Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID"); + List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources(); + + // Since provider is not customized, CloudInfoSources and CloudInfoProcessorName will be null. + Assert.assertNull(listUrlFromZk); + Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName()); + Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name()); + } }
