zhangmeng916 commented on a change in pull request #1935: URL: https://github.com/apache/helix/pull/1935#discussion_r799911831
########## File path: helix-rest/src/main/java/org/apache/helix/rest/server/service/VirtualTopologyGroupService.java ########## @@ -0,0 +1,183 @@ +package org.apache.helix.rest.server.service; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.helix.AccessOption; +import org.apache.helix.ConfigAccessor; +import org.apache.helix.HelixAdmin; +import org.apache.helix.HelixDataAccessor; +import org.apache.helix.HelixException; +import org.apache.helix.PropertyPathBuilder; +import org.apache.helix.cloud.constants.VirtualTopologyGroupConstants; +import org.apache.helix.cloud.topology.FifoVirtualGroupAssignmentAlgorithm; +import org.apache.helix.cloud.topology.VirtualGroupAssignmentAlgorithm; +import org.apache.helix.model.CloudConfig; +import org.apache.helix.model.ClusterConfig; +import org.apache.helix.model.ClusterTopologyConfig; +import org.apache.helix.model.InstanceConfig; +import org.apache.helix.rest.server.json.cluster.ClusterTopology; +import org.apache.helix.zookeeper.datamodel.ZNRecord; +import org.apache.helix.zookeeper.zkclient.DataUpdater; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Service for virtual topology group. + */ +public class VirtualTopologyGroupService { + private static final Logger LOG = LoggerFactory.getLogger(VirtualTopologyGroupService.class); + + private final HelixAdmin _helixAdmin; + private final ClusterService _clusterService; + private final ConfigAccessor _configAccessor; + private final HelixDataAccessor _dataAccessor; + private final VirtualGroupAssignmentAlgorithm _assignmentAlgorithm; + + public VirtualTopologyGroupService(HelixAdmin helixAdmin, ClusterService clusterService, + ConfigAccessor configAccessor, HelixDataAccessor dataAccessor) { + _helixAdmin = helixAdmin; + _clusterService = clusterService; + _configAccessor = configAccessor; + _dataAccessor = dataAccessor; + _assignmentAlgorithm = FifoVirtualGroupAssignmentAlgorithm.getInstance(); + } + + /** + * Add virtual topology group for a cluster. + * This includes calculating the virtual group assignment for all instances in the cluster then update instance config + * and cluster config. Cluster is expected to enter maintenanceMode during config update, this is either enabled/disabled + * in place this method or handled by client side code. Review comment: Please make sure the java doc grammatically correct. ########## File path: helix-rest/src/test/java/org/apache/helix/rest/server/service/TestVirtualTopologyGroupService.java ########## @@ -0,0 +1,183 @@ +package org.apache.helix.rest.server.service; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.helix.ConfigAccessor; +import org.apache.helix.HelixAdmin; +import org.apache.helix.HelixDataAccessor; +import org.apache.helix.HelixException; +import org.apache.helix.cloud.azure.AzureConstants; +import org.apache.helix.cloud.constants.CloudProvider; +import org.apache.helix.model.CloudConfig; +import org.apache.helix.model.ClusterConfig; +import org.apache.helix.model.HelixConfigScope; +import org.apache.helix.model.InstanceConfig; +import org.apache.helix.model.builder.HelixConfigScopeBuilder; +import org.apache.helix.rest.server.json.cluster.ClusterTopology; +import org.apache.helix.zookeeper.datamodel.ZNRecord; +import org.apache.helix.zookeeper.zkclient.DataUpdater; +import org.testng.Assert; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static org.apache.helix.cloud.constants.VirtualTopologyGroupConstants.*; +import static org.mockito.Mockito.*; + + +public class TestVirtualTopologyGroupService { + private static final String TEST_CLUSTER = "Test_Cluster"; + private static final String TEST_CLUSTER0 = "TestCluster_0"; + private static final String TEST_CLUSTER1 = "TestCluster_1"; + private static final String TEST_CLUSTER2 = "TestCluster_2"; + + private final ConfigAccessor _configAccessor = mock(ConfigAccessor.class); + private final HelixDataAccessor _dataAccessor = mock(HelixDataAccessor.class); + private InstanceConfig _instanceConfig0; + private InstanceConfig _instanceConfig1; + private InstanceConfig _instanceConfig2; + private Map<String, DataUpdater<ZNRecord>> _updaterMap; + private VirtualTopologyGroupService _service; + + @BeforeTest + public void prepare() { + Map<String, Set<String>> assignment = new HashMap<>(); + _instanceConfig0 = new InstanceConfig("instance_0"); + _instanceConfig0.setDomain("helixZoneId=zone0"); + _instanceConfig1 = new InstanceConfig("instance_1"); + _instanceConfig1.setDomain("helixZoneId=zone0"); + _instanceConfig2 = new InstanceConfig("instance_2"); + _instanceConfig2.setDomain("helixZoneId=zone1"); + + assignment.put("virtual_group_0", ImmutableSet.of("instance_0", "instance_1")); + assignment.put("virtual_group_1", ImmutableSet.of("instance_2")); + _updaterMap = VirtualTopologyGroupService.createInstanceConfigUpdater(TEST_CLUSTER, assignment); + + ClusterConfig clusterConfig = new ClusterConfig(TEST_CLUSTER0); + clusterConfig.setFaultZoneType(AzureConstants.AZURE_FAULT_ZONE_TYPE); + clusterConfig.setTopology(AzureConstants.AZURE_TOPOLOGY); + clusterConfig.setTopologyAwareEnabled(true); + when(_configAccessor.getClusterConfig(TEST_CLUSTER0)).thenReturn(clusterConfig); + + CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder(); + cloudConfigBuilder.setCloudEnabled(true); + cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE); + cloudConfigBuilder.setCloudID("TestID"); + CloudConfig cloudConfig = cloudConfigBuilder.build(); + when(_configAccessor.getCloudConfig(TEST_CLUSTER0)).thenReturn(cloudConfig); + when(_configAccessor.getCloudConfig(TEST_CLUSTER2)).thenReturn(cloudConfig); + + HelixAdmin helixAdmin = mock(HelixAdmin.class); + when(helixAdmin.isInMaintenanceMode(anyString())).thenReturn(true); + + boolean[] results = new boolean[2]; + results[0] = results[1] = true; + when(_dataAccessor.updateChildren(anyList(), anyList(), anyInt())).thenReturn(results); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterTopology(anyString())).thenReturn(prepareClusterTopology()); + _service = new VirtualTopologyGroupService(helixAdmin, clusterService, _configAccessor, _dataAccessor); + } + + @Test(expectedExceptions = HelixException.class, expectedExceptionsMessageRegExp = "Cloud is not enabled.*") + public void testClusterCloudConfigSetup() { + ClusterConfig clusterConfig1 = new ClusterConfig(TEST_CLUSTER1); + when(_configAccessor.getClusterConfig(TEST_CLUSTER1)).thenReturn(clusterConfig1); + _service.addVirtualTopologyGroup( + TEST_CLUSTER1, ImmutableMap.of(GROUP_NAME, "test-group", GROUP_NUMBER, "2")); + } + + @Test + public void testVirtualTopologyGroupService() { + _service.addVirtualTopologyGroup(TEST_CLUSTER0, ImmutableMap.of( + GROUP_NAME, "test-group", GROUP_NUMBER, "2", ENTER_MAINTENANCE_MODE, "false")); + verify(_dataAccessor, times(1)).updateChildren(anyList(), anyList(), anyInt()); + verify(_configAccessor, times(1)).updateClusterConfig(anyString(), any()); + } + + @Test(expectedExceptions = IllegalStateException.class, + expectedExceptionsMessageRegExp = "This operation is not allowed while cluster is in maintenance mode.*") Review comment: Is the exception correct? Should be "the operation is not allowed while cluster is not in maintenance mode"? ########## File path: helix-rest/src/main/java/org/apache/helix/rest/server/service/VirtualTopologyGroupService.java ########## @@ -0,0 +1,183 @@ +package org.apache.helix.rest.server.service; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.helix.AccessOption; +import org.apache.helix.ConfigAccessor; +import org.apache.helix.HelixAdmin; +import org.apache.helix.HelixDataAccessor; +import org.apache.helix.HelixException; +import org.apache.helix.PropertyPathBuilder; +import org.apache.helix.cloud.constants.VirtualTopologyGroupConstants; +import org.apache.helix.cloud.topology.FifoVirtualGroupAssignmentAlgorithm; +import org.apache.helix.cloud.topology.VirtualGroupAssignmentAlgorithm; +import org.apache.helix.model.CloudConfig; +import org.apache.helix.model.ClusterConfig; +import org.apache.helix.model.ClusterTopologyConfig; +import org.apache.helix.model.InstanceConfig; +import org.apache.helix.rest.server.json.cluster.ClusterTopology; +import org.apache.helix.zookeeper.datamodel.ZNRecord; +import org.apache.helix.zookeeper.zkclient.DataUpdater; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Service for virtual topology group. + */ +public class VirtualTopologyGroupService { + private static final Logger LOG = LoggerFactory.getLogger(VirtualTopologyGroupService.class); + + private final HelixAdmin _helixAdmin; + private final ClusterService _clusterService; + private final ConfigAccessor _configAccessor; + private final HelixDataAccessor _dataAccessor; + private final VirtualGroupAssignmentAlgorithm _assignmentAlgorithm; + + public VirtualTopologyGroupService(HelixAdmin helixAdmin, ClusterService clusterService, + ConfigAccessor configAccessor, HelixDataAccessor dataAccessor) { + _helixAdmin = helixAdmin; + _clusterService = clusterService; + _configAccessor = configAccessor; + _dataAccessor = dataAccessor; + _assignmentAlgorithm = FifoVirtualGroupAssignmentAlgorithm.getInstance(); + } + + /** + * Add virtual topology group for a cluster. + * This includes calculating the virtual group assignment for all instances in the cluster then update instance config + * and cluster config. Cluster is expected to enter maintenanceMode during config update, this is either enabled/disabled + * in place this method or handled by client side code. + * @param clusterName the cluster name. + * @param customFields custom fields, {@link VirtualTopologyGroupConstants#GROUP_NAME} + * and {@link VirtualTopologyGroupConstants#GROUP_NUMBER} are required, + * {@link VirtualTopologyGroupConstants#ENTER_MAINTENANCE_MODE} is optional. + * if enabled, the cluster will enter maintenance mode during the setup and exit once it + * completes. Otherwise, it's expected the maintenanceMode is controlled by client side. + */ + public void addVirtualTopologyGroup(String clusterName, Map<String, String> customFields) { + // validation + CloudConfig cloudConfig = _configAccessor.getCloudConfig(clusterName); + if (cloudConfig == null || !cloudConfig.isCloudEnabled()) { + throw new HelixException( + "Cloud is not enabled, addVirtualTopologyGroup is not allowed to run in non-cloud environment."); + } + ClusterConfig clusterConfig = _configAccessor.getClusterConfig(clusterName); + Preconditions.checkState(clusterConfig.isTopologyAwareEnabled(), + "Topology-aware rebalance is not enabled in cluster " + clusterName); + String groupName = customFields.get(VirtualTopologyGroupConstants.GROUP_NAME); + String groupNumberStr = Preconditions.checkNotNull( + customFields.get(VirtualTopologyGroupConstants.GROUP_NUMBER), + "virtualTopologyGroupNumber cannot be empty!"); + Preconditions.checkArgument(!StringUtils.isEmpty(groupName), "virtualTopologyGroupName cannot be empty!"); + int numGroups = Integer.parseInt(groupNumberStr); + LOG.info("Computing virtual topology group for cluster {} with param {}", clusterName, customFields); + + // compute group assignment + ClusterTopology clusterTopology = _clusterService.getClusterTopology(clusterName); + Preconditions.checkArgument(numGroups <= clusterTopology.getAllInstances().size(), + "Number of virtual groups cannot be greater than the number of instances."); + Map<String, Set<String>> assignment = + _assignmentAlgorithm.computeAssignment(numGroups, groupName, clusterTopology.toZoneMapping()); + + boolean enterMaintenanceMode = Boolean.parseBoolean( + customFields.getOrDefault(VirtualTopologyGroupConstants.ENTER_MAINTENANCE_MODE, "true")); Review comment: What are we trying to achieve with this extra config `ENTER_MAINTENANCE_MODE`? Are we trying to avoid the case where customers call this API but they do not want to enter maintenance mode? Then the call will fail, right? This means customers use this API in a wrong way. Please let me know if you have other intention with this config. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
