This is an automated email from the ASF dual-hosted git repository.
xyuanlu pushed a commit to branch ApplicationClusterManager
in repository https://gitbox.apache.org/repos/asf/helix.git
The following commit(s) were added to refs/heads/ApplicationClusterManager by
this push:
new 809657a07 Enhanced stoppable checks with node evacuation filtering and
introduced blacklisting capabilities (#2687)
809657a07 is described below
commit 809657a07e940516779665af150f3e4fdf28f74a
Author: Xiaxuan Gao <[email protected]>
AuthorDate: Fri Nov 3 17:29:20 2023 -0700
Enhanced stoppable checks with node evacuation filtering and introduced
blacklisting capabilities (#2687)
Enhanced stoppable checks with node evacuation filtering and introduced
blacklisting capabilities
---
.../apache/helix/util/InstanceValidationUtil.java | 9 +-
.../helix/util/TestInstanceValidationUtil.java | 2 +-
.../MaintenanceManagementService.java | 115 ++++++++++++++++++++-
.../StoppableInstancesSelector.java | 42 +++++++-
.../server/resources/helix/InstancesAccessor.java | 44 +++++++-
.../helix/rest/server/AbstractTestClass.java | 2 -
.../helix/rest/server/TestInstancesAccessor.java | 99 +++++++++++++++++-
7 files changed, 291 insertions(+), 22 deletions(-)
diff --git
a/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java
b/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java
index 5f179e784..2542ecf7f 100644
--- a/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java
+++ b/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java
@@ -295,7 +295,7 @@ public class InstanceValidationUtil {
if (stateMap.containsKey(instanceToBeStop)
&&
stateMap.get(instanceToBeStop).equals(stateModelDefinition.getTopState())) {
for (String siblingInstance : stateMap.keySet()) {
- // Skip this self check
+ // Skip this self check and instances we assume to be already
stopped
if (siblingInstance.equals(instanceToBeStop) ||
(toBeStoppedInstances != null
&& toBeStoppedInstances.contains(siblingInstance))) {
continue;
@@ -451,9 +451,10 @@ public class InstanceValidationUtil {
if (stateByInstanceMap.containsKey(instanceName)) {
int numHealthySiblings = 0;
for (Map.Entry<String, String> entry :
stateByInstanceMap.entrySet()) {
- if (!entry.getKey().equals(instanceName) && (toBeStoppedInstances
== null
- || !toBeStoppedInstances.contains(entry.getKey())) &&
!unhealthyStates.contains(
- entry.getValue())) {
+ String siblingInstanceName = entry.getKey();
+ if (!siblingInstanceName.equals(instanceName) &&
(toBeStoppedInstances == null
+ || !toBeStoppedInstances.contains(siblingInstanceName))
+ && !unhealthyStates.contains(entry.getValue())) {
numHealthySiblings++;
}
}
diff --git
a/helix-core/src/test/java/org/apache/helix/util/TestInstanceValidationUtil.java
b/helix-core/src/test/java/org/apache/helix/util/TestInstanceValidationUtil.java
index aa1ba3229..79b0fdce8 100644
---
a/helix-core/src/test/java/org/apache/helix/util/TestInstanceValidationUtil.java
+++
b/helix-core/src/test/java/org/apache/helix/util/TestInstanceValidationUtil.java
@@ -375,7 +375,7 @@ public class TestInstanceValidationUtil {
String resource = "resource";
Mock mock = new Mock();
doReturn(ImmutableList.of(resource)).when(mock.dataAccessor)
- .getChildNames(argThat(new
PropertyKeyArgument(PropertyType.EXTERNALVIEW)));
+ .getChildNames(argThat(new
PropertyKeyArgument(PropertyType.IDEALSTATES)));
// set ideal state
IdealState idealState = mock(IdealState.class);
when(idealState.isEnabled()).thenReturn(true);
diff --git
a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java
b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java
index c3fa04966..52377e612 100644
---
a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java
+++
b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java
@@ -93,6 +93,10 @@ public class MaintenanceManagementService {
private final HelixDataAccessorWrapper _dataAccessor;
private final Set<String> _nonBlockingHealthChecks;
private final Set<StoppableCheck.Category> _skipHealthCheckCategories;
+ // Set the default value of _skipStoppableHealthCheckList to be an empty
list to
+ // maintain the backward compatibility with users who don't use
MaintenanceManagementServiceBuilder
+ // to create the MaintenanceManagementService object.
+ private List<HealthCheck> _skipStoppableHealthCheckList =
Collections.emptyList();
public MaintenanceManagementService(ZKHelixDataAccessor dataAccessor,
ConfigAccessor configAccessor, boolean skipZKRead, String namespace) {
@@ -144,6 +148,25 @@ public class MaintenanceManagementService {
_namespace = namespace;
}
+ private MaintenanceManagementService(ZKHelixDataAccessor dataAccessor,
+ ConfigAccessor configAccessor, CustomRestClient customRestClient,
boolean skipZKRead,
+ Set<String> nonBlockingHealthChecks, Set<StoppableCheck.Category>
skipHealthCheckCategories,
+ List<HealthCheck> skipStoppableHealthCheckList, String namespace) {
+ _dataAccessor =
+ new HelixDataAccessorWrapper(dataAccessor, customRestClient,
+ namespace);
+ _configAccessor = configAccessor;
+ _customRestClient = customRestClient;
+ _skipZKRead = skipZKRead;
+ _nonBlockingHealthChecks =
+ nonBlockingHealthChecks == null ? Collections.emptySet() :
nonBlockingHealthChecks;
+ _skipHealthCheckCategories =
+ skipHealthCheckCategories == null ? Collections.emptySet() :
skipHealthCheckCategories;
+ _skipStoppableHealthCheckList = skipStoppableHealthCheckList == null ?
Collections.emptyList()
+ : skipStoppableHealthCheckList;
+ _namespace = namespace;
+ }
+
/**
* Perform health check and maintenance operation check and execution for a
instance in
* one cluster.
@@ -463,7 +486,10 @@ public class MaintenanceManagementService {
return instances;
}
RESTConfig restConfig = _configAccessor.getRESTConfig(clusterId);
- if (restConfig == null) {
+ if (restConfig == null && (
+
!_skipHealthCheckCategories.contains(StoppableCheck.Category.CUSTOM_INSTANCE_CHECK)
+ || !_skipHealthCheckCategories.contains(
+ StoppableCheck.Category.CUSTOM_PARTITION_CHECK))) {
String errorMessage = String.format(
"The cluster %s hasn't enabled client side health checks yet, "
+ "thus the stoppable check result is inaccurate", clusterId);
@@ -612,8 +638,10 @@ public class MaintenanceManagementService {
private StoppableCheck performHelixOwnInstanceCheck(String clusterId, String
instanceName,
Set<String> toBeStoppedInstances) {
LOG.info("Perform helix own custom health checks for {}/{}", clusterId,
instanceName);
+ List<HealthCheck> healthChecksToExecute = new
ArrayList<>(HealthCheck.STOPPABLE_CHECK_LIST);
+ healthChecksToExecute.removeAll(_skipStoppableHealthCheckList);
Map<String, Boolean> helixStoppableCheck =
- getInstanceHealthStatus(clusterId, instanceName,
HealthCheck.STOPPABLE_CHECK_LIST,
+ getInstanceHealthStatus(clusterId, instanceName, healthChecksToExecute,
toBeStoppedInstances);
return new StoppableCheck(helixStoppableCheck,
StoppableCheck.Category.HELIX_OWN_CHECK);
@@ -771,4 +799,87 @@ public class MaintenanceManagementService {
return healthStatus;
}
+
+ public static class MaintenanceManagementServiceBuilder {
+ private ConfigAccessor _configAccessor;
+ private boolean _skipZKRead;
+ private String _namespace;
+ private ZKHelixDataAccessor _dataAccessor;
+ private CustomRestClient _customRestClient;
+ private Set<String> _nonBlockingHealthChecks;
+ private Set<StoppableCheck.Category> _skipHealthCheckCategories =
Collections.emptySet();
+ private List<HealthCheck> _skipStoppableHealthCheckList =
Collections.emptyList();
+
+ public MaintenanceManagementServiceBuilder
setConfigAccessor(ConfigAccessor configAccessor) {
+ _configAccessor = configAccessor;
+ return this;
+ }
+
+ public MaintenanceManagementServiceBuilder setSkipZKRead(boolean
skipZKRead) {
+ _skipZKRead = skipZKRead;
+ return this;
+ }
+
+ public MaintenanceManagementServiceBuilder setNamespace(String namespace) {
+ _namespace = namespace;
+ return this;
+ }
+
+ public MaintenanceManagementServiceBuilder setDataAccessor(
+ ZKHelixDataAccessor dataAccessor) {
+ _dataAccessor = dataAccessor;
+ return this;
+ }
+
+ public MaintenanceManagementServiceBuilder setCustomRestClient(
+ CustomRestClient customRestClient) {
+ _customRestClient = customRestClient;
+ return this;
+ }
+
+ public MaintenanceManagementServiceBuilder setNonBlockingHealthChecks(
+ Set<String> nonBlockingHealthChecks) {
+ _nonBlockingHealthChecks = nonBlockingHealthChecks;
+ return this;
+ }
+
+ public MaintenanceManagementServiceBuilder setSkipHealthCheckCategories(
+ Set<StoppableCheck.Category> skipHealthCheckCategories) {
+ _skipHealthCheckCategories = skipHealthCheckCategories;
+ return this;
+ }
+
+ public MaintenanceManagementServiceBuilder setSkipStoppableHealthCheckList(
+ List<HealthCheck> skipStoppableHealthCheckList) {
+ _skipStoppableHealthCheckList = skipStoppableHealthCheckList;
+ return this;
+ }
+
+ public MaintenanceManagementService build() {
+ validate();
+ return new MaintenanceManagementService(_dataAccessor, _configAccessor,
_customRestClient,
+ _skipZKRead, _nonBlockingHealthChecks, _skipHealthCheckCategories,
+ _skipStoppableHealthCheckList, _namespace);
+ }
+
+ private void validate() throws IllegalArgumentException {
+ List<String> msg = new ArrayList<>();
+ if (_configAccessor == null) {
+ msg.add("'configAccessor' can't be null.");
+ }
+ if (_namespace == null) {
+ msg.add("'namespace' can't be null.");
+ }
+ if (_dataAccessor == null) {
+ msg.add("'_dataAccessor' can't be null.");
+ }
+ if (_customRestClient == null) {
+ msg.add("'customRestClient' can't be null.");
+ }
+ if (msg.size() != 0) {
+ throw new IllegalArgumentException(
+ "One or more mandatory arguments are not set " + msg);
+ }
+ }
+ }
}
diff --git
a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/StoppableInstancesSelector.java
b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/StoppableInstancesSelector.java
index 8cf8bc83c..877aaa9c8 100644
---
a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/StoppableInstancesSelector.java
+++
b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/StoppableInstancesSelector.java
@@ -34,6 +34,10 @@ import java.util.stream.Collectors;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.constants.InstanceConstants;
+import org.apache.helix.manager.zk.ZKHelixDataAccessor;
+import org.apache.helix.model.InstanceConfig;
import org.apache.helix.rest.server.json.cluster.ClusterTopology;
import org.apache.helix.rest.server.json.instance.StoppableCheck;
import org.apache.helix.rest.server.resources.helix.InstancesAccessor;
@@ -48,15 +52,17 @@ public class StoppableInstancesSelector {
private final String _customizedInput;
private final MaintenanceManagementService _maintenanceService;
private final ClusterTopology _clusterTopology;
+ private final ZKHelixDataAccessor _dataAccessor;
- public StoppableInstancesSelector(String clusterId, List<String> orderOfZone,
+ private StoppableInstancesSelector(String clusterId, List<String>
orderOfZone,
String customizedInput, MaintenanceManagementService maintenanceService,
- ClusterTopology clusterTopology) {
+ ClusterTopology clusterTopology, ZKHelixDataAccessor dataAccessor) {
_clusterId = clusterId;
_orderOfZone = orderOfZone;
_customizedInput = customizedInput;
_maintenanceService = maintenanceService;
_clusterTopology = clusterTopology;
+ _dataAccessor = dataAccessor;
}
/**
@@ -66,7 +72,7 @@ public class StoppableInstancesSelector {
* reasons for non-stoppability.
*
* @param instances A list of instance to be evaluated.
- * @param toBeStoppedInstances A list of instances presumed to be are
already stopped
+ * @param toBeStoppedInstances A list of instances presumed to be already
stopped
* @return An ObjectNode containing:
* - 'stoppableNode': List of instances that can be stopped.
* - 'instance_not_stoppable_with_reasons': A map with the instance
name as the key and
@@ -81,6 +87,7 @@ public class StoppableInstancesSelector {
ObjectNode failedStoppableInstances = result.putObject(
InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name());
Set<String> toBeStoppedInstancesSet = new HashSet<>(toBeStoppedInstances);
+ collectEvacuatingInstances(toBeStoppedInstancesSet);
List<String> zoneBasedInstance =
getZoneBasedInstances(instances, _clusterTopology.toZoneMapping());
@@ -97,7 +104,7 @@ public class StoppableInstancesSelector {
* non-stoppability.
*
* @param instances A list of instance to be evaluated.
- * @param toBeStoppedInstances A list of instances presumed to be are
already stopped
+ * @param toBeStoppedInstances A list of instances presumed to be already
stopped
* @return An ObjectNode containing:
* - 'stoppableNode': List of instances that can be stopped.
* - 'instance_not_stoppable_with_reasons': A map with the instance
name as the key and
@@ -112,6 +119,7 @@ public class StoppableInstancesSelector {
ObjectNode failedStoppableInstances = result.putObject(
InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name());
Set<String> toBeStoppedInstancesSet = new HashSet<>(toBeStoppedInstances);
+ collectEvacuatingInstances(toBeStoppedInstancesSet);
Map<String, Set<String>> zoneMapping = _clusterTopology.toZoneMapping();
for (String zone : _orderOfZone) {
@@ -249,12 +257,31 @@ public class StoppableInstancesSelector {
(existing, replacement) -> existing, LinkedHashMap::new));
}
+ /**
+ * Collect instances marked for evacuation in the current topology and add
them into the given set
+ *
+ * @param toBeStoppedInstances A set of instances we presume to be stopped.
+ */
+ private void collectEvacuatingInstances(Set<String> toBeStoppedInstances) {
+ Set<String> allInstances = _clusterTopology.getAllInstances();
+ for (String instance : allInstances) {
+ PropertyKey.Builder propertyKeyBuilder = _dataAccessor.keyBuilder();
+ InstanceConfig instanceConfig =
+
_dataAccessor.getProperty(propertyKeyBuilder.instanceConfig(instance));
+ if (InstanceConstants.InstanceOperation.EVACUATE.name()
+ .equals(instanceConfig.getInstanceOperation())) {
+ toBeStoppedInstances.add(instance);
+ }
+ }
+ }
+
public static class StoppableInstancesSelectorBuilder {
private String _clusterId;
private List<String> _orderOfZone;
private String _customizedInput;
private MaintenanceManagementService _maintenanceService;
private ClusterTopology _clusterTopology;
+ private ZKHelixDataAccessor _dataAccessor;
public StoppableInstancesSelectorBuilder setClusterId(String clusterId) {
_clusterId = clusterId;
@@ -282,9 +309,14 @@ public class StoppableInstancesSelector {
return this;
}
+ public StoppableInstancesSelectorBuilder
setDataAccessor(ZKHelixDataAccessor dataAccessor) {
+ _dataAccessor = dataAccessor;
+ return this;
+ }
+
public StoppableInstancesSelector build() {
return new StoppableInstancesSelector(_clusterId, _orderOfZone,
_customizedInput,
- _maintenanceService, _clusterTopology);
+ _maintenanceService, _clusterTopology, _dataAccessor);
}
}
}
diff --git
a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
index 785195ebe..fcad387dc 100644
---
a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
+++
b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java
@@ -20,12 +20,12 @@ package org.apache.helix.rest.server.resources.helix;
*/
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
@@ -46,6 +46,8 @@ import org.apache.helix.HelixException;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.InstanceConfig;
+import org.apache.helix.rest.client.CustomRestClientFactory;
+import org.apache.helix.rest.clusterMaintenanceService.HealthCheck;
import
org.apache.helix.rest.clusterMaintenanceService.MaintenanceManagementService;
import org.apache.helix.rest.common.HttpConstants;
import
org.apache.helix.rest.clusterMaintenanceService.StoppableInstancesSelector;
@@ -59,10 +61,13 @@ import org.apache.helix.util.InstanceValidationUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static
org.apache.helix.rest.clusterMaintenanceService.MaintenanceManagementService.ALL_HEALTH_CHECK_NONBLOCK;
+
@ClusterAuth
@Path("/clusters/{clusterId}/instances")
public class InstancesAccessor extends AbstractHelixResource {
private final static Logger _logger =
LoggerFactory.getLogger(InstancesAccessor.class);
+
public enum InstancesProperties {
instances,
online,
@@ -70,6 +75,7 @@ public class InstancesAccessor extends AbstractHelixResource {
selection_base,
zone_order,
to_be_stopped_instances,
+ skip_stoppable_check_list,
customized_values,
instance_stoppable_parallel,
instance_not_stoppable_with_reasons
@@ -228,6 +234,9 @@ public class InstancesAccessor extends
AbstractHelixResource {
List<String> orderOfZone = null;
String customizedInput = null;
List<String> toBeStoppedInstances = Collections.emptyList();
+ // By default, if skip_stoppable_check_list is unset, all checks are
performed to maintain
+ // backward compatibility with existing clients.
+ List<HealthCheck> skipStoppableCheckList = Collections.emptyList();
if
(node.get(InstancesAccessor.InstancesProperties.customized_values.name()) !=
null) {
customizedInput =
node.get(InstancesAccessor.InstancesProperties.customized_values.name()).toString();
@@ -260,10 +269,36 @@ public class InstancesAccessor extends
AbstractHelixResource {
}
}
+ if (node.get(InstancesProperties.skip_stoppable_check_list.name()) !=
null) {
+ List<String> list = OBJECT_MAPPER.readValue(
+
node.get(InstancesProperties.skip_stoppable_check_list.name()).toString(),
+ OBJECT_MAPPER.getTypeFactory().constructCollectionType(List.class,
String.class));
+ try {
+ skipStoppableCheckList =
+
list.stream().map(HealthCheck::valueOf).collect(Collectors.toList());
+ } catch (IllegalArgumentException e) {
+ String message =
+ "'skip_stoppable_check_list' has invalid check names: " + list
+ + ". Supported checks: " + HealthCheck.STOPPABLE_CHECK_LIST;
+ _logger.error(message, e);
+ return badRequest(message);
+ }
+ }
+
+ String namespace = getNamespace();
MaintenanceManagementService maintenanceService =
- new MaintenanceManagementService((ZKHelixDataAccessor)
getDataAccssor(clusterId),
- getConfigAccessor(), skipZKRead, continueOnFailures,
skipHealthCheckCategories,
- getNamespace());
+ new
MaintenanceManagementService.MaintenanceManagementServiceBuilder()
+ .setDataAccessor((ZKHelixDataAccessor) getDataAccssor(clusterId))
+ .setConfigAccessor(getConfigAccessor())
+ .setSkipZKRead(skipZKRead)
+ .setNonBlockingHealthChecks(
+ continueOnFailures ?
Collections.singleton(ALL_HEALTH_CHECK_NONBLOCK) : null)
+ .setCustomRestClient(CustomRestClientFactory.get())
+ .setSkipHealthCheckCategories(skipHealthCheckCategories)
+ .setNamespace(namespace)
+ .setSkipStoppableHealthCheckList(skipStoppableCheckList)
+ .build();
+
ClusterService clusterService =
new ClusterServiceImpl(getDataAccssor(clusterId),
getConfigAccessor());
ClusterTopology clusterTopology =
clusterService.getClusterTopology(clusterId);
@@ -274,6 +309,7 @@ public class InstancesAccessor extends
AbstractHelixResource {
.setCustomizedInput(customizedInput)
.setMaintenanceService(maintenanceService)
.setClusterTopology(clusterTopology)
+ .setDataAccessor((ZKHelixDataAccessor) getDataAccssor(clusterId))
.build();
stoppableInstancesSelector.calculateOrderOfZone(instances, random);
ObjectNode result;
diff --git
a/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
index 68561ce83..6b357a384 100644
---
a/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
+++
b/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
@@ -621,8 +621,6 @@ public class AbstractTestClass extends
JerseyTestNg.ContainerPerClassTest {
clusterConfig.setFaultZoneType("helixZoneId");
clusterConfig.setPersistIntermediateAssignment(true);
_configAccessor.setClusterConfig(clusterName, clusterConfig);
- RESTConfig emptyRestConfig = new RESTConfig(clusterName);
- _configAccessor.setRESTConfig(clusterName, emptyRestConfig);
// Create instance configs
List<InstanceConfig> instanceConfigs = new ArrayList<>();
int perZoneInstancesCount = 3;
diff --git
a/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java
b/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java
index 2bc539a4d..92dfff002 100644
---
a/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java
+++
b/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java
@@ -34,6 +34,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.helix.TestHelper;
+import org.apache.helix.constants.InstanceConstants;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.rest.server.resources.helix.InstancesAccessor;
@@ -113,7 +114,7 @@ public class TestInstancesAccessor extends
AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
- @Test
+ @Test(dependsOnMethods =
"testInstanceStoppableZoneBasedWithToBeStoppedInstances")
public void testInstanceStoppableZoneBasedWithoutZoneOrder() throws
IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
String content = String.format(
@@ -144,7 +145,8 @@ public class TestInstancesAccessor extends
AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
- @Test(dataProvider = "generatePayloadCrossZoneStoppableCheckWithZoneOrder")
+ @Test(dataProvider = "generatePayloadCrossZoneStoppableCheckWithZoneOrder",
+ dependsOnMethods = "testInstanceStoppableZoneBasedWithoutZoneOrder")
public void testCrossZoneStoppableWithZoneOrder(String content) throws
IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
Response response = new JerseyUriRequestBuilder(
@@ -166,7 +168,7 @@ public class TestInstancesAccessor extends
AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
- @Test
+ @Test(dependsOnMethods = "testCrossZoneStoppableWithZoneOrder")
public void testCrossZoneStoppableWithoutZoneOrder() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
String content = String.format(
@@ -199,8 +201,97 @@ public class TestInstancesAccessor extends
AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
+ @Test(dependsOnMethods = "testCrossZoneStoppableWithoutZoneOrder")
+ public void testInstanceStoppableCrossZoneBasedWithSelectedCheckList()
throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ // Select instances with cross zone based and perform all checks
+ String content =
+
String.format("{\"%s\":\"%s\",\"%s\":[\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",
\"%s\"], \"%s\":[\"%s\"]}",
+ InstancesAccessor.InstancesProperties.selection_base.name(),
+
InstancesAccessor.InstanceHealthSelectionBase.cross_zone_based.name(),
+ InstancesAccessor.InstancesProperties.instances.name(),
"instance0", "instance1",
+ "instance2", "instance3", "instance4", "instance5",
"invalidInstance",
+
InstancesAccessor.InstancesProperties.skip_stoppable_check_list.name(),
"DUMMY_TEST_NO_EXISTS");
- @Test(dependsOnMethods =
"testInstanceStoppableZoneBasedWithToBeStoppedInstances")
+ new
JerseyUriRequestBuilder("clusters/{}/instances?command=stoppable").format(STOPPABLE_CLUSTER)
+ .isBodyReturnExpected(true)
+ .expectedReturnStatusCode(Response.Status.BAD_REQUEST.getStatusCode())
+ .post(this, Entity.entity(content, MediaType.APPLICATION_JSON_TYPE));
+
+ // Select instances with cross zone based and perform a subset of checks
+ content = String.format(
+ "{\"%s\":\"%s\",\"%s\":[\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",
\"%s\"], \"%s\":[\"%s\",\"%s\"], \"%s\":[\"%s\", \"%s\"]}",
+ InstancesAccessor.InstancesProperties.selection_base.name(),
+ InstancesAccessor.InstanceHealthSelectionBase.cross_zone_based.name(),
+ InstancesAccessor.InstancesProperties.instances.name(), "instance0",
"instance1",
+ "instance2", "instance3", "instance4", "instance5", "invalidInstance",
+ InstancesAccessor.InstancesProperties.zone_order.name(), "zone2",
"zone1",
+
InstancesAccessor.InstancesProperties.skip_stoppable_check_list.name(),
"INSTANCE_NOT_ENABLED", "INSTANCE_NOT_STABLE");
+ Response response = new JerseyUriRequestBuilder(
+
"clusters/{}/instances?command=stoppable&skipHealthCheckCategories=CUSTOM_INSTANCE_CHECK,CUSTOM_PARTITION_CHECK").format(
+ STOPPABLE_CLUSTER).post(this, Entity.entity(content,
MediaType.APPLICATION_JSON_TYPE));
+ JsonNode jsonNode =
OBJECT_MAPPER.readTree(response.readEntity(String.class));
+ JsonNode nonStoppableInstances = jsonNode.get(
+
InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name());
+ Assert.assertEquals(getStringSet(nonStoppableInstances, "instance5"),
+ ImmutableSet.of("HELIX:EMPTY_RESOURCE_ASSIGNMENT",
"HELIX:INSTANCE_NOT_ALIVE"));
+ Assert.assertEquals(getStringSet(nonStoppableInstances, "instance4"),
+ ImmutableSet.of("HELIX:EMPTY_RESOURCE_ASSIGNMENT",
"HELIX:INSTANCE_NOT_ALIVE"));
+ Assert.assertEquals(getStringSet(nonStoppableInstances, "instance1"),
+ ImmutableSet.of("HELIX:EMPTY_RESOURCE_ASSIGNMENT"));
+ Assert.assertEquals(getStringSet(nonStoppableInstances, "invalidInstance"),
+ ImmutableSet.of("HELIX:INSTANCE_NOT_EXIST"));
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods =
"testInstanceStoppableCrossZoneBasedWithSelectedCheckList")
+ public void testInstanceStoppableCrossZoneBasedWithEvacuatingInstances()
throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ String content = String.format(
+ "{\"%s\":\"%s\",\"%s\":[\"%s\",\"%s\",\"%s\",\"%s\", \"%s\", \"%s\",
\"%s\",\"%s\", \"%s\", \"%s\"]}",
+ InstancesAccessor.InstancesProperties.selection_base.name(),
+ InstancesAccessor.InstanceHealthSelectionBase.cross_zone_based.name(),
+ InstancesAccessor.InstancesProperties.instances.name(), "instance1",
"instance3",
+ "instance6", "instance9", "instance10", "instance11", "instance12",
"instance13",
+ "instance14", "invalidInstance");
+
+ // Change instance config of instance1 & instance0 to be evacuating
+ String instance0 = "instance0";
+ InstanceConfig instanceConfig =
_configAccessor.getInstanceConfig(STOPPABLE_CLUSTER2, instance0);
+
instanceConfig.setInstanceOperation(InstanceConstants.InstanceOperation.EVACUATE);
+ _configAccessor.setInstanceConfig(STOPPABLE_CLUSTER2, instance0,
instanceConfig);
+ String instance1 = "instance1";
+ InstanceConfig instanceConfig1 =
_configAccessor.getInstanceConfig(STOPPABLE_CLUSTER2, instance1);
+
instanceConfig1.setInstanceOperation(InstanceConstants.InstanceOperation.EVACUATE);
+ _configAccessor.setInstanceConfig(STOPPABLE_CLUSTER2, instance1,
instanceConfig1);
+ // It takes time to reflect the changes.
+ BestPossibleExternalViewVerifier verifier =
+ new
BestPossibleExternalViewVerifier.Builder(STOPPABLE_CLUSTER2).setZkAddr(ZK_ADDR).build();
+ Assert.assertTrue(verifier.verifyByPolling());
+
+ Response response = new JerseyUriRequestBuilder(
+
"clusters/{}/instances?command=stoppable&skipHealthCheckCategories=CUSTOM_INSTANCE_CHECK,CUSTOM_PARTITION_CHECK").format(
+ STOPPABLE_CLUSTER2).post(this, Entity.entity(content,
MediaType.APPLICATION_JSON_TYPE));
+ JsonNode jsonNode =
OBJECT_MAPPER.readTree(response.readEntity(String.class));
+
+ Set<String> stoppableSet = getStringSet(jsonNode,
+
InstancesAccessor.InstancesProperties.instance_stoppable_parallel.name());
+ Assert.assertTrue(stoppableSet.contains("instance12")
+ && stoppableSet.contains("instance11") &&
stoppableSet.contains("instance10"));
+
+ JsonNode nonStoppableInstances = jsonNode.get(
+
InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name());
+ Assert.assertEquals(getStringSet(nonStoppableInstances, "instance13"),
+ ImmutableSet.of("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED"));
+ Assert.assertEquals(getStringSet(nonStoppableInstances, "instance14"),
+ ImmutableSet.of("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED"));
+ Assert.assertEquals(getStringSet(nonStoppableInstances, "invalidInstance"),
+ ImmutableSet.of("HELIX:INSTANCE_NOT_EXIST"));
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods =
"testInstanceStoppableCrossZoneBasedWithEvacuatingInstances")
public void testInstanceStoppable_zoneBased_zoneOrder() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
// Select instances with zone based
