This is an automated email from the ASF dual-hosted git repository. xyuanlu pushed a commit to branch ApplicationClusterManager in repository https://gitbox.apache.org/repos/asf/helix.git
commit 58cccf5ce43989c8bc83bc521bef80fd1f18c1e7 Author: Xiaxuan Gao <[email protected]> AuthorDate: Fri Nov 3 17:29:20 2023 -0700 Enhanced stoppable checks with node evacuation filtering and introduced blacklisting capabilities (#2687) Enhanced stoppable checks with node evacuation filtering and introduced blacklisting capabilities --- .../apache/helix/util/InstanceValidationUtil.java | 9 +- .../helix/util/TestInstanceValidationUtil.java | 2 +- .../MaintenanceManagementService.java | 115 ++++++++++++++++++++- .../StoppableInstancesSelector.java | 42 +++++++- .../server/resources/helix/InstancesAccessor.java | 44 +++++++- .../helix/rest/server/AbstractTestClass.java | 2 - .../helix/rest/server/TestInstancesAccessor.java | 99 +++++++++++++++++- 7 files changed, 291 insertions(+), 22 deletions(-) diff --git a/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java b/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java index 5f179e784..2542ecf7f 100644 --- a/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java +++ b/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java @@ -295,7 +295,7 @@ public class InstanceValidationUtil { if (stateMap.containsKey(instanceToBeStop) && stateMap.get(instanceToBeStop).equals(stateModelDefinition.getTopState())) { for (String siblingInstance : stateMap.keySet()) { - // Skip this self check + // Skip this self check and instances we assume to be already stopped if (siblingInstance.equals(instanceToBeStop) || (toBeStoppedInstances != null && toBeStoppedInstances.contains(siblingInstance))) { continue; @@ -451,9 +451,10 @@ public class InstanceValidationUtil { if (stateByInstanceMap.containsKey(instanceName)) { int numHealthySiblings = 0; for (Map.Entry<String, String> entry : stateByInstanceMap.entrySet()) { - if (!entry.getKey().equals(instanceName) && (toBeStoppedInstances == null - || !toBeStoppedInstances.contains(entry.getKey())) && !unhealthyStates.contains( - entry.getValue())) { + String siblingInstanceName = entry.getKey(); + if (!siblingInstanceName.equals(instanceName) && (toBeStoppedInstances == null + || !toBeStoppedInstances.contains(siblingInstanceName)) + && !unhealthyStates.contains(entry.getValue())) { numHealthySiblings++; } } diff --git a/helix-core/src/test/java/org/apache/helix/util/TestInstanceValidationUtil.java b/helix-core/src/test/java/org/apache/helix/util/TestInstanceValidationUtil.java index aa1ba3229..79b0fdce8 100644 --- a/helix-core/src/test/java/org/apache/helix/util/TestInstanceValidationUtil.java +++ b/helix-core/src/test/java/org/apache/helix/util/TestInstanceValidationUtil.java @@ -375,7 +375,7 @@ public class TestInstanceValidationUtil { String resource = "resource"; Mock mock = new Mock(); doReturn(ImmutableList.of(resource)).when(mock.dataAccessor) - .getChildNames(argThat(new PropertyKeyArgument(PropertyType.EXTERNALVIEW))); + .getChildNames(argThat(new PropertyKeyArgument(PropertyType.IDEALSTATES))); // set ideal state IdealState idealState = mock(IdealState.class); when(idealState.isEnabled()).thenReturn(true); diff --git a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java index c3fa04966..52377e612 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java @@ -93,6 +93,10 @@ public class MaintenanceManagementService { private final HelixDataAccessorWrapper _dataAccessor; private final Set<String> _nonBlockingHealthChecks; private final Set<StoppableCheck.Category> _skipHealthCheckCategories; + // Set the default value of _skipStoppableHealthCheckList to be an empty list to + // maintain the backward compatibility with users who don't use MaintenanceManagementServiceBuilder + // to create the MaintenanceManagementService object. + private List<HealthCheck> _skipStoppableHealthCheckList = Collections.emptyList(); public MaintenanceManagementService(ZKHelixDataAccessor dataAccessor, ConfigAccessor configAccessor, boolean skipZKRead, String namespace) { @@ -144,6 +148,25 @@ public class MaintenanceManagementService { _namespace = namespace; } + private MaintenanceManagementService(ZKHelixDataAccessor dataAccessor, + ConfigAccessor configAccessor, CustomRestClient customRestClient, boolean skipZKRead, + Set<String> nonBlockingHealthChecks, Set<StoppableCheck.Category> skipHealthCheckCategories, + List<HealthCheck> skipStoppableHealthCheckList, String namespace) { + _dataAccessor = + new HelixDataAccessorWrapper(dataAccessor, customRestClient, + namespace); + _configAccessor = configAccessor; + _customRestClient = customRestClient; + _skipZKRead = skipZKRead; + _nonBlockingHealthChecks = + nonBlockingHealthChecks == null ? Collections.emptySet() : nonBlockingHealthChecks; + _skipHealthCheckCategories = + skipHealthCheckCategories == null ? Collections.emptySet() : skipHealthCheckCategories; + _skipStoppableHealthCheckList = skipStoppableHealthCheckList == null ? Collections.emptyList() + : skipStoppableHealthCheckList; + _namespace = namespace; + } + /** * Perform health check and maintenance operation check and execution for a instance in * one cluster. @@ -463,7 +486,10 @@ public class MaintenanceManagementService { return instances; } RESTConfig restConfig = _configAccessor.getRESTConfig(clusterId); - if (restConfig == null) { + if (restConfig == null && ( + !_skipHealthCheckCategories.contains(StoppableCheck.Category.CUSTOM_INSTANCE_CHECK) + || !_skipHealthCheckCategories.contains( + StoppableCheck.Category.CUSTOM_PARTITION_CHECK))) { String errorMessage = String.format( "The cluster %s hasn't enabled client side health checks yet, " + "thus the stoppable check result is inaccurate", clusterId); @@ -612,8 +638,10 @@ public class MaintenanceManagementService { private StoppableCheck performHelixOwnInstanceCheck(String clusterId, String instanceName, Set<String> toBeStoppedInstances) { LOG.info("Perform helix own custom health checks for {}/{}", clusterId, instanceName); + List<HealthCheck> healthChecksToExecute = new ArrayList<>(HealthCheck.STOPPABLE_CHECK_LIST); + healthChecksToExecute.removeAll(_skipStoppableHealthCheckList); Map<String, Boolean> helixStoppableCheck = - getInstanceHealthStatus(clusterId, instanceName, HealthCheck.STOPPABLE_CHECK_LIST, + getInstanceHealthStatus(clusterId, instanceName, healthChecksToExecute, toBeStoppedInstances); return new StoppableCheck(helixStoppableCheck, StoppableCheck.Category.HELIX_OWN_CHECK); @@ -771,4 +799,87 @@ public class MaintenanceManagementService { return healthStatus; } + + public static class MaintenanceManagementServiceBuilder { + private ConfigAccessor _configAccessor; + private boolean _skipZKRead; + private String _namespace; + private ZKHelixDataAccessor _dataAccessor; + private CustomRestClient _customRestClient; + private Set<String> _nonBlockingHealthChecks; + private Set<StoppableCheck.Category> _skipHealthCheckCategories = Collections.emptySet(); + private List<HealthCheck> _skipStoppableHealthCheckList = Collections.emptyList(); + + public MaintenanceManagementServiceBuilder setConfigAccessor(ConfigAccessor configAccessor) { + _configAccessor = configAccessor; + return this; + } + + public MaintenanceManagementServiceBuilder setSkipZKRead(boolean skipZKRead) { + _skipZKRead = skipZKRead; + return this; + } + + public MaintenanceManagementServiceBuilder setNamespace(String namespace) { + _namespace = namespace; + return this; + } + + public MaintenanceManagementServiceBuilder setDataAccessor( + ZKHelixDataAccessor dataAccessor) { + _dataAccessor = dataAccessor; + return this; + } + + public MaintenanceManagementServiceBuilder setCustomRestClient( + CustomRestClient customRestClient) { + _customRestClient = customRestClient; + return this; + } + + public MaintenanceManagementServiceBuilder setNonBlockingHealthChecks( + Set<String> nonBlockingHealthChecks) { + _nonBlockingHealthChecks = nonBlockingHealthChecks; + return this; + } + + public MaintenanceManagementServiceBuilder setSkipHealthCheckCategories( + Set<StoppableCheck.Category> skipHealthCheckCategories) { + _skipHealthCheckCategories = skipHealthCheckCategories; + return this; + } + + public MaintenanceManagementServiceBuilder setSkipStoppableHealthCheckList( + List<HealthCheck> skipStoppableHealthCheckList) { + _skipStoppableHealthCheckList = skipStoppableHealthCheckList; + return this; + } + + public MaintenanceManagementService build() { + validate(); + return new MaintenanceManagementService(_dataAccessor, _configAccessor, _customRestClient, + _skipZKRead, _nonBlockingHealthChecks, _skipHealthCheckCategories, + _skipStoppableHealthCheckList, _namespace); + } + + private void validate() throws IllegalArgumentException { + List<String> msg = new ArrayList<>(); + if (_configAccessor == null) { + msg.add("'configAccessor' can't be null."); + } + if (_namespace == null) { + msg.add("'namespace' can't be null."); + } + if (_dataAccessor == null) { + msg.add("'_dataAccessor' can't be null."); + } + if (_customRestClient == null) { + msg.add("'customRestClient' can't be null."); + } + if (msg.size() != 0) { + throw new IllegalArgumentException( + "One or more mandatory arguments are not set " + msg); + } + } + } } diff --git a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/StoppableInstancesSelector.java b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/StoppableInstancesSelector.java index 8cf8bc83c..877aaa9c8 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/StoppableInstancesSelector.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/StoppableInstancesSelector.java @@ -34,6 +34,10 @@ import java.util.stream.Collectors; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.helix.PropertyKey; +import org.apache.helix.constants.InstanceConstants; +import org.apache.helix.manager.zk.ZKHelixDataAccessor; +import org.apache.helix.model.InstanceConfig; import org.apache.helix.rest.server.json.cluster.ClusterTopology; import org.apache.helix.rest.server.json.instance.StoppableCheck; import org.apache.helix.rest.server.resources.helix.InstancesAccessor; @@ -48,15 +52,17 @@ public class StoppableInstancesSelector { private final String _customizedInput; private final MaintenanceManagementService _maintenanceService; private final ClusterTopology _clusterTopology; + private final ZKHelixDataAccessor _dataAccessor; - public StoppableInstancesSelector(String clusterId, List<String> orderOfZone, + private StoppableInstancesSelector(String clusterId, List<String> orderOfZone, String customizedInput, MaintenanceManagementService maintenanceService, - ClusterTopology clusterTopology) { + ClusterTopology clusterTopology, ZKHelixDataAccessor dataAccessor) { _clusterId = clusterId; _orderOfZone = orderOfZone; _customizedInput = customizedInput; _maintenanceService = maintenanceService; _clusterTopology = clusterTopology; + _dataAccessor = dataAccessor; } /** @@ -66,7 +72,7 @@ public class StoppableInstancesSelector { * reasons for non-stoppability. * * @param instances A list of instance to be evaluated. - * @param toBeStoppedInstances A list of instances presumed to be are already stopped + * @param toBeStoppedInstances A list of instances presumed to be already stopped * @return An ObjectNode containing: * - 'stoppableNode': List of instances that can be stopped. * - 'instance_not_stoppable_with_reasons': A map with the instance name as the key and @@ -81,6 +87,7 @@ public class StoppableInstancesSelector { ObjectNode failedStoppableInstances = result.putObject( InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name()); Set<String> toBeStoppedInstancesSet = new HashSet<>(toBeStoppedInstances); + collectEvacuatingInstances(toBeStoppedInstancesSet); List<String> zoneBasedInstance = getZoneBasedInstances(instances, _clusterTopology.toZoneMapping()); @@ -97,7 +104,7 @@ public class StoppableInstancesSelector { * non-stoppability. * * @param instances A list of instance to be evaluated. - * @param toBeStoppedInstances A list of instances presumed to be are already stopped + * @param toBeStoppedInstances A list of instances presumed to be already stopped * @return An ObjectNode containing: * - 'stoppableNode': List of instances that can be stopped. * - 'instance_not_stoppable_with_reasons': A map with the instance name as the key and @@ -112,6 +119,7 @@ public class StoppableInstancesSelector { ObjectNode failedStoppableInstances = result.putObject( InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name()); Set<String> toBeStoppedInstancesSet = new HashSet<>(toBeStoppedInstances); + collectEvacuatingInstances(toBeStoppedInstancesSet); Map<String, Set<String>> zoneMapping = _clusterTopology.toZoneMapping(); for (String zone : _orderOfZone) { @@ -249,12 +257,31 @@ public class StoppableInstancesSelector { (existing, replacement) -> existing, LinkedHashMap::new)); } + /** + * Collect instances marked for evacuation in the current topology and add them into the given set + * + * @param toBeStoppedInstances A set of instances we presume to be stopped. + */ + private void collectEvacuatingInstances(Set<String> toBeStoppedInstances) { + Set<String> allInstances = _clusterTopology.getAllInstances(); + for (String instance : allInstances) { + PropertyKey.Builder propertyKeyBuilder = _dataAccessor.keyBuilder(); + InstanceConfig instanceConfig = + _dataAccessor.getProperty(propertyKeyBuilder.instanceConfig(instance)); + if (InstanceConstants.InstanceOperation.EVACUATE.name() + .equals(instanceConfig.getInstanceOperation())) { + toBeStoppedInstances.add(instance); + } + } + } + public static class StoppableInstancesSelectorBuilder { private String _clusterId; private List<String> _orderOfZone; private String _customizedInput; private MaintenanceManagementService _maintenanceService; private ClusterTopology _clusterTopology; + private ZKHelixDataAccessor _dataAccessor; public StoppableInstancesSelectorBuilder setClusterId(String clusterId) { _clusterId = clusterId; @@ -282,9 +309,14 @@ public class StoppableInstancesSelector { return this; } + public StoppableInstancesSelectorBuilder setDataAccessor(ZKHelixDataAccessor dataAccessor) { + _dataAccessor = dataAccessor; + return this; + } + public StoppableInstancesSelector build() { return new StoppableInstancesSelector(_clusterId, _orderOfZone, _customizedInput, - _maintenanceService, _clusterTopology); + _maintenanceService, _clusterTopology, _dataAccessor); } } } diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java index 785195ebe..fcad387dc 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java @@ -20,12 +20,12 @@ package org.apache.helix.rest.server.resources.helix; */ import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -46,6 +46,8 @@ import org.apache.helix.HelixException; import org.apache.helix.manager.zk.ZKHelixDataAccessor; import org.apache.helix.model.ClusterConfig; import org.apache.helix.model.InstanceConfig; +import org.apache.helix.rest.client.CustomRestClientFactory; +import org.apache.helix.rest.clusterMaintenanceService.HealthCheck; import org.apache.helix.rest.clusterMaintenanceService.MaintenanceManagementService; import org.apache.helix.rest.common.HttpConstants; import org.apache.helix.rest.clusterMaintenanceService.StoppableInstancesSelector; @@ -59,10 +61,13 @@ import org.apache.helix.util.InstanceValidationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.helix.rest.clusterMaintenanceService.MaintenanceManagementService.ALL_HEALTH_CHECK_NONBLOCK; + @ClusterAuth @Path("/clusters/{clusterId}/instances") public class InstancesAccessor extends AbstractHelixResource { private final static Logger _logger = LoggerFactory.getLogger(InstancesAccessor.class); + public enum InstancesProperties { instances, online, @@ -70,6 +75,7 @@ public class InstancesAccessor extends AbstractHelixResource { selection_base, zone_order, to_be_stopped_instances, + skip_stoppable_check_list, customized_values, instance_stoppable_parallel, instance_not_stoppable_with_reasons @@ -228,6 +234,9 @@ public class InstancesAccessor extends AbstractHelixResource { List<String> orderOfZone = null; String customizedInput = null; List<String> toBeStoppedInstances = Collections.emptyList(); + // By default, if skip_stoppable_check_list is unset, all checks are performed to maintain + // backward compatibility with existing clients. + List<HealthCheck> skipStoppableCheckList = Collections.emptyList(); if (node.get(InstancesAccessor.InstancesProperties.customized_values.name()) != null) { customizedInput = node.get(InstancesAccessor.InstancesProperties.customized_values.name()).toString(); @@ -260,10 +269,36 @@ public class InstancesAccessor extends AbstractHelixResource { } } + if (node.get(InstancesProperties.skip_stoppable_check_list.name()) != null) { + List<String> list = OBJECT_MAPPER.readValue( + node.get(InstancesProperties.skip_stoppable_check_list.name()).toString(), + OBJECT_MAPPER.getTypeFactory().constructCollectionType(List.class, String.class)); + try { + skipStoppableCheckList = + list.stream().map(HealthCheck::valueOf).collect(Collectors.toList()); + } catch (IllegalArgumentException e) { + String message = + "'skip_stoppable_check_list' has invalid check names: " + list + + ". Supported checks: " + HealthCheck.STOPPABLE_CHECK_LIST; + _logger.error(message, e); + return badRequest(message); + } + } + + String namespace = getNamespace(); MaintenanceManagementService maintenanceService = - new MaintenanceManagementService((ZKHelixDataAccessor) getDataAccssor(clusterId), - getConfigAccessor(), skipZKRead, continueOnFailures, skipHealthCheckCategories, - getNamespace()); + new MaintenanceManagementService.MaintenanceManagementServiceBuilder() + .setDataAccessor((ZKHelixDataAccessor) getDataAccssor(clusterId)) + .setConfigAccessor(getConfigAccessor()) + .setSkipZKRead(skipZKRead) + .setNonBlockingHealthChecks( + continueOnFailures ? Collections.singleton(ALL_HEALTH_CHECK_NONBLOCK) : null) + .setCustomRestClient(CustomRestClientFactory.get()) + .setSkipHealthCheckCategories(skipHealthCheckCategories) + .setNamespace(namespace) + .setSkipStoppableHealthCheckList(skipStoppableCheckList) + .build(); + ClusterService clusterService = new ClusterServiceImpl(getDataAccssor(clusterId), getConfigAccessor()); ClusterTopology clusterTopology = clusterService.getClusterTopology(clusterId); @@ -274,6 +309,7 @@ public class InstancesAccessor extends AbstractHelixResource { .setCustomizedInput(customizedInput) .setMaintenanceService(maintenanceService) .setClusterTopology(clusterTopology) + .setDataAccessor((ZKHelixDataAccessor) getDataAccssor(clusterId)) .build(); stoppableInstancesSelector.calculateOrderOfZone(instances, random); ObjectNode result; diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java b/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java index 68561ce83..6b357a384 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java @@ -621,8 +621,6 @@ public class AbstractTestClass extends JerseyTestNg.ContainerPerClassTest { clusterConfig.setFaultZoneType("helixZoneId"); clusterConfig.setPersistIntermediateAssignment(true); _configAccessor.setClusterConfig(clusterName, clusterConfig); - RESTConfig emptyRestConfig = new RESTConfig(clusterName); - _configAccessor.setRESTConfig(clusterName, emptyRestConfig); // Create instance configs List<InstanceConfig> instanceConfigs = new ArrayList<>(); int perZoneInstancesCount = 3; diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java index 2bc539a4d..92dfff002 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java @@ -34,6 +34,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.helix.TestHelper; +import org.apache.helix.constants.InstanceConstants; import org.apache.helix.model.ClusterConfig; import org.apache.helix.model.InstanceConfig; import org.apache.helix.rest.server.resources.helix.InstancesAccessor; @@ -113,7 +114,7 @@ public class TestInstancesAccessor extends AbstractTestClass { System.out.println("End test :" + TestHelper.getTestMethodName()); } - @Test + @Test(dependsOnMethods = "testInstanceStoppableZoneBasedWithToBeStoppedInstances") public void testInstanceStoppableZoneBasedWithoutZoneOrder() throws IOException { System.out.println("Start test :" + TestHelper.getTestMethodName()); String content = String.format( @@ -144,7 +145,8 @@ public class TestInstancesAccessor extends AbstractTestClass { System.out.println("End test :" + TestHelper.getTestMethodName()); } - @Test(dataProvider = "generatePayloadCrossZoneStoppableCheckWithZoneOrder") + @Test(dataProvider = "generatePayloadCrossZoneStoppableCheckWithZoneOrder", + dependsOnMethods = "testInstanceStoppableZoneBasedWithoutZoneOrder") public void testCrossZoneStoppableWithZoneOrder(String content) throws IOException { System.out.println("Start test :" + TestHelper.getTestMethodName()); Response response = new JerseyUriRequestBuilder( @@ -166,7 +168,7 @@ public class TestInstancesAccessor extends AbstractTestClass { System.out.println("End test :" + TestHelper.getTestMethodName()); } - @Test + @Test(dependsOnMethods = "testCrossZoneStoppableWithZoneOrder") public void testCrossZoneStoppableWithoutZoneOrder() throws IOException { System.out.println("Start test :" + TestHelper.getTestMethodName()); String content = String.format( @@ -199,8 +201,97 @@ public class TestInstancesAccessor extends AbstractTestClass { System.out.println("End test :" + TestHelper.getTestMethodName()); } + @Test(dependsOnMethods = "testCrossZoneStoppableWithoutZoneOrder") + public void testInstanceStoppableCrossZoneBasedWithSelectedCheckList() throws IOException { + System.out.println("Start test :" + TestHelper.getTestMethodName()); + // Select instances with cross zone based and perform all checks + String content = + String.format("{\"%s\":\"%s\",\"%s\":[\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\", \"%s\"], \"%s\":[\"%s\"]}", + InstancesAccessor.InstancesProperties.selection_base.name(), + InstancesAccessor.InstanceHealthSelectionBase.cross_zone_based.name(), + InstancesAccessor.InstancesProperties.instances.name(), "instance0", "instance1", + "instance2", "instance3", "instance4", "instance5", "invalidInstance", + InstancesAccessor.InstancesProperties.skip_stoppable_check_list.name(), "DUMMY_TEST_NO_EXISTS"); - @Test(dependsOnMethods = "testInstanceStoppableZoneBasedWithToBeStoppedInstances") + new JerseyUriRequestBuilder("clusters/{}/instances?command=stoppable").format(STOPPABLE_CLUSTER) + .isBodyReturnExpected(true) + .expectedReturnStatusCode(Response.Status.BAD_REQUEST.getStatusCode()) + .post(this, Entity.entity(content, MediaType.APPLICATION_JSON_TYPE)); + + // Select instances with cross zone based and perform a subset of checks + content = String.format( + "{\"%s\":\"%s\",\"%s\":[\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\", \"%s\"], \"%s\":[\"%s\",\"%s\"], \"%s\":[\"%s\", \"%s\"]}", + InstancesAccessor.InstancesProperties.selection_base.name(), + InstancesAccessor.InstanceHealthSelectionBase.cross_zone_based.name(), + InstancesAccessor.InstancesProperties.instances.name(), "instance0", "instance1", + "instance2", "instance3", "instance4", "instance5", "invalidInstance", + InstancesAccessor.InstancesProperties.zone_order.name(), "zone2", "zone1", + InstancesAccessor.InstancesProperties.skip_stoppable_check_list.name(), "INSTANCE_NOT_ENABLED", "INSTANCE_NOT_STABLE"); + Response response = new JerseyUriRequestBuilder( + "clusters/{}/instances?command=stoppable&skipHealthCheckCategories=CUSTOM_INSTANCE_CHECK,CUSTOM_PARTITION_CHECK").format( + STOPPABLE_CLUSTER).post(this, Entity.entity(content, MediaType.APPLICATION_JSON_TYPE)); + JsonNode jsonNode = OBJECT_MAPPER.readTree(response.readEntity(String.class)); + JsonNode nonStoppableInstances = jsonNode.get( + InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name()); + Assert.assertEquals(getStringSet(nonStoppableInstances, "instance5"), + ImmutableSet.of("HELIX:EMPTY_RESOURCE_ASSIGNMENT", "HELIX:INSTANCE_NOT_ALIVE")); + Assert.assertEquals(getStringSet(nonStoppableInstances, "instance4"), + ImmutableSet.of("HELIX:EMPTY_RESOURCE_ASSIGNMENT", "HELIX:INSTANCE_NOT_ALIVE")); + Assert.assertEquals(getStringSet(nonStoppableInstances, "instance1"), + ImmutableSet.of("HELIX:EMPTY_RESOURCE_ASSIGNMENT")); + Assert.assertEquals(getStringSet(nonStoppableInstances, "invalidInstance"), + ImmutableSet.of("HELIX:INSTANCE_NOT_EXIST")); + + System.out.println("End test :" + TestHelper.getTestMethodName()); + } + + @Test(dependsOnMethods = "testInstanceStoppableCrossZoneBasedWithSelectedCheckList") + public void testInstanceStoppableCrossZoneBasedWithEvacuatingInstances() throws IOException { + System.out.println("Start test :" + TestHelper.getTestMethodName()); + String content = String.format( + "{\"%s\":\"%s\",\"%s\":[\"%s\",\"%s\",\"%s\",\"%s\", \"%s\", \"%s\", \"%s\",\"%s\", \"%s\", \"%s\"]}", + InstancesAccessor.InstancesProperties.selection_base.name(), + InstancesAccessor.InstanceHealthSelectionBase.cross_zone_based.name(), + InstancesAccessor.InstancesProperties.instances.name(), "instance1", "instance3", + "instance6", "instance9", "instance10", "instance11", "instance12", "instance13", + "instance14", "invalidInstance"); + + // Change instance config of instance1 & instance0 to be evacuating + String instance0 = "instance0"; + InstanceConfig instanceConfig = _configAccessor.getInstanceConfig(STOPPABLE_CLUSTER2, instance0); + instanceConfig.setInstanceOperation(InstanceConstants.InstanceOperation.EVACUATE); + _configAccessor.setInstanceConfig(STOPPABLE_CLUSTER2, instance0, instanceConfig); + String instance1 = "instance1"; + InstanceConfig instanceConfig1 = _configAccessor.getInstanceConfig(STOPPABLE_CLUSTER2, instance1); + instanceConfig1.setInstanceOperation(InstanceConstants.InstanceOperation.EVACUATE); + _configAccessor.setInstanceConfig(STOPPABLE_CLUSTER2, instance1, instanceConfig1); + // It takes time to reflect the changes. + BestPossibleExternalViewVerifier verifier = + new BestPossibleExternalViewVerifier.Builder(STOPPABLE_CLUSTER2).setZkAddr(ZK_ADDR).build(); + Assert.assertTrue(verifier.verifyByPolling()); + + Response response = new JerseyUriRequestBuilder( + "clusters/{}/instances?command=stoppable&skipHealthCheckCategories=CUSTOM_INSTANCE_CHECK,CUSTOM_PARTITION_CHECK").format( + STOPPABLE_CLUSTER2).post(this, Entity.entity(content, MediaType.APPLICATION_JSON_TYPE)); + JsonNode jsonNode = OBJECT_MAPPER.readTree(response.readEntity(String.class)); + + Set<String> stoppableSet = getStringSet(jsonNode, + InstancesAccessor.InstancesProperties.instance_stoppable_parallel.name()); + Assert.assertTrue(stoppableSet.contains("instance12") + && stoppableSet.contains("instance11") && stoppableSet.contains("instance10")); + + JsonNode nonStoppableInstances = jsonNode.get( + InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name()); + Assert.assertEquals(getStringSet(nonStoppableInstances, "instance13"), + ImmutableSet.of("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED")); + Assert.assertEquals(getStringSet(nonStoppableInstances, "instance14"), + ImmutableSet.of("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED")); + Assert.assertEquals(getStringSet(nonStoppableInstances, "invalidInstance"), + ImmutableSet.of("HELIX:INSTANCE_NOT_EXIST")); + System.out.println("End test :" + TestHelper.getTestMethodName()); + } + + @Test(dependsOnMethods = "testInstanceStoppableCrossZoneBasedWithEvacuatingInstances") public void testInstanceStoppable_zoneBased_zoneOrder() throws IOException { System.out.println("Start test :" + TestHelper.getTestMethodName()); // Select instances with zone based
