This is an automated email from the ASF dual-hosted git repository. hulee pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/helix.git
commit 87de68624f692b58b41beb2232015ba981815236 Author: Junkai Xue <[email protected]> AuthorDate: Fri Mar 29 11:11:35 2019 -0700 Global instance stoppable API This API will be input list of instances and return stoppable instances. So checks performed here: 1. single stoppable for each instance. 2. shutdown instances will cause replicas drop less than min active number. For first phase, we do not implement instance based selection. Here we added an integration test: 1. test instances disabled, has disable partition, not same zone, not alive. 2. disable one stoppable instance, check failed 3. reeable the instance and remove the disabled partition, check for that instance passed again. Several places make set, map to be TreeSet and TreeMap is that we would like to guarantee the output result is consistent. We do see sorting different for Java 7 and Java 8. RB=1596424 BUG=HELIX-1680 G=helix-reviewers A=lxia Signed-off-by: Hunter Lee <[email protected]> --- .../apache/helix/util/InstanceValidationUtil.java | 7 + .../rest/server/json/cluster/ClusterTopology.java | 30 +++- .../rest/server/json/instance/StoppableCheck.java | 8 + .../server/resources/helix/InstanceAccessor.java | 161 ++++++++++++++++++--- .../rest/server/service/InstanceServiceImpl.java | 14 +- .../helix/rest/server/TestInstanceAccessor.java | 96 +++++++++++- 6 files changed, 285 insertions(+), 31 deletions(-) diff --git a/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java b/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java index 351c50d..112ae1b 100644 --- a/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java +++ b/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java @@ -270,7 +270,14 @@ public class InstanceValidationUtil { List<String> idealStateNames = dataAccessor.getChildNames(keyBuilder.idealStates()); for (String idealStateName : idealStateNames) { IdealState idealState = dataAccessor.getProperty(keyBuilder.idealStates(idealStateName)); + if (idealState == null || !idealState.isEnabled()) { + continue; + } ExternalView externalView = dataAccessor.getProperty(keyBuilder.externalView(idealStateName)); + if (externalView == null) { + throw new HelixException( + String.format("Resource %s does not have external view!", idealStateName)); + } for (String partition : idealState.getPartitionSet()) { Map<String, String> isPartitionMap = idealState.getInstanceStateMap(partition); Map<String, String> evPartitionMap = externalView.getStateMap(partition); diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java index c581c5d..8325822 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java @@ -1,9 +1,13 @@ package org.apache.helix.rest.server.json.cluster; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import com.fasterxml.jackson.annotation.JsonProperty; - +import java.util.Map; +import java.util.Set; /** * POJO class that can be easily convert to JSON object @@ -55,6 +59,10 @@ public class ClusterTopology { public void setInstances(List<Instance> instances) { this.instances = instances; } + + public String getId() { + return id; + } } public static final class Instance { @@ -64,5 +72,25 @@ public class ClusterTopology { public Instance(String id) { this.id = id; } + + public String getId() { + return id; + } + } + + public Map<String, Set<String>> toZoneMapping() { + Map<String, Set<String>> zoneMapping = new HashMap<>(); + if (zones == null) { + return Collections.emptyMap(); + } + for (ClusterTopology.Zone zone : zones) { + zoneMapping.put(zone.getId(), new HashSet<String>()); + if (zone.getInstances() != null) { + for (ClusterTopology.Instance instance : zone.getInstances()) { + zoneMapping.get(zone.getId()).add(instance.getId()); + } + } + } + return zoneMapping; } } diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/json/instance/StoppableCheck.java b/helix-rest/src/main/java/org/apache/helix/rest/server/json/instance/StoppableCheck.java index b8485b2..8567cdc 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/server/json/instance/StoppableCheck.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/json/instance/StoppableCheck.java @@ -65,4 +65,12 @@ public class StoppableCheck { return result; } + + public boolean isStoppable() { + return isStoppable; + } + + public List<String> getFailedChecks() { + return failedChecks; + } } diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstanceAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstanceAccessor.java index dc9da3e..d07fb4e 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstanceAccessor.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstanceAccessor.java @@ -19,11 +19,15 @@ package org.apache.helix.rest.server.resources.helix; * under the License. */ +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; - +import java.util.Set; +import java.util.TreeSet; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -34,7 +38,7 @@ import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; - +import org.apache.commons.lang3.NotImplementedException; import org.apache.helix.ConfigAccessor; import org.apache.helix.HelixAdmin; import org.apache.helix.HelixDataAccessor; @@ -55,6 +59,8 @@ import org.apache.helix.rest.client.CustomRestClientFactory; import org.apache.helix.rest.common.HelixDataAccessorWrapper; import org.apache.helix.rest.server.json.instance.InstanceInfo; import org.apache.helix.rest.server.json.instance.StoppableCheck; +import org.apache.helix.rest.server.service.ClusterService; +import org.apache.helix.rest.server.service.ClusterServiceImpl; import org.apache.helix.rest.server.service.InstanceService; import org.apache.helix.rest.server.service.InstanceServiceImpl; import org.codehaus.jackson.JsonNode; @@ -65,7 +71,6 @@ import org.eclipse.jetty.util.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.ObjectMapper; @Path("/clusters/{clusterId}/instances") public class InstanceAccessor extends AbstractHelixResource { @@ -86,7 +91,17 @@ public class InstanceAccessor extends AbstractHelixResource { total_message_count, read_message_count, healthreports, - instanceTags + instanceTags, + selection_base, + zone_order, + customized_values, + instance_stoppable_parallel, + instance_not_stoppable_with_reasons + } + + public enum InstanceHealthSelectionBase { + instance_based, + zone_based } @GET @@ -169,6 +184,79 @@ public class InstanceAccessor extends AbstractHelixResource { return OK(); } + @POST + @Path("stoppable") + @Consumes(MediaType.APPLICATION_JSON) + public Response getParallelStoppableInstances(@PathParam("clusterId") String clusterId, + String content) { + try { + JsonNode node = null; + if (content.length() != 0) { + node = OBJECT_MAPPER.readTree(content); + } + if (node == null) { + return badRequest("Invalid input for content : " + content); + } + + // TODO: Process input data from the content + InstanceHealthSelectionBase selectionBase = InstanceHealthSelectionBase + .valueOf(node.get(InstanceProperties.selection_base.name()).getValueAsText()); + List<String> instances = OBJECT_MAPPER + .readValue(node.get(InstanceProperties.instances.name()).toString(), + OBJECT_MAPPER.getTypeFactory().constructCollectionType(List.class, String.class)); + + List<String> orderOfZone = null; + String customizedInput = null; + if (node.get(InstanceProperties.customized_values.name()) != null) { + customizedInput = node.get(InstanceProperties.customized_values.name()).getTextValue(); + } + + if (node.get(InstanceProperties.zone_order.name()) != null) { + orderOfZone = OBJECT_MAPPER + .readValue(node.get(InstanceProperties.zone_order.name()).toString(), + OBJECT_MAPPER.getTypeFactory().constructCollectionType(List.class, String.class)); + } + + // Prepare output result + ObjectNode result = JsonNodeFactory.instance.objectNode(); + ArrayNode stoppableInstances = + result.putArray(InstanceProperties.instance_stoppable_parallel.name()); + ObjectNode failedStoppableInstances = + result.putObject(InstanceProperties.instance_not_stoppable_with_reasons.name()); + + switch (selectionBase) { + case zone_based: + List<String> zoneBasedInstance = getZoneBasedInstance(clusterId, instances, orderOfZone); + for (String instance : zoneBasedInstance) { + StoppableCheck stoppableCheck = + checkSingleInstanceStoppable(clusterId, instance, customizedInput); + if (!stoppableCheck.isStoppable()) { + ArrayNode failedReasonsNode = failedStoppableInstances.putArray(instance); + for (String failedReason : stoppableCheck.getFailedChecks()) { + failedReasonsNode.add(JsonNodeFactory.instance.textNode(failedReason)); + } + } else { + stoppableInstances.add(instance); + } + } + break; + case instance_based: + default: + throw new NotImplementedException("instance_based selection is not support now!"); + } + return JSONRepresentation(result); + } catch (HelixException e) { + _logger + .error(String.format("Current cluster %s has issue with health checks!", clusterId), e); + return serverError(e); + } catch (Exception e) { + _logger.error( + String.format("Failed to get parallel stoppable instances for cluster %s!", clusterId), + e); + return serverError(e); + } + } + @GET @Path("{instanceName}") public Response getInstanceById(@PathParam("clusterId") String clusterId, @@ -190,20 +278,14 @@ public class InstanceAccessor extends AbstractHelixResource { public Response isInstanceStoppable(String jsonContent, @PathParam("clusterId") String clusterId, @PathParam("instanceName") String instanceName) throws IOException { ObjectMapper objectMapper = new ObjectMapper(); - HelixDataAccessor dataAccessor = getDataAccssor(clusterId); - // TODO reduce GC by dependency injection - InstanceService instanceService = new InstanceServiceImpl( - new HelixDataAccessorWrapper((ZKHelixDataAccessor) dataAccessor), getConfigAccessor()); - - Map<String, Boolean> helixStoppableCheck = instanceService.getInstanceHealthStatus(clusterId, - instanceName, InstanceService.HealthCheck.STOPPABLE_CHECK_LIST); - CustomRestClient customClient = CustomRestClientFactory.get(jsonContent); - // TODO add the json content parse logic - Map<String, Boolean> customStoppableCheck = - customClient.getInstanceStoppableCheck(Collections.<String, String> emptyMap()); - StoppableCheck stoppableCheck = - StoppableCheck.mergeStoppableChecks(helixStoppableCheck, customStoppableCheck); - + StoppableCheck stoppableCheck = null; + try { + stoppableCheck = checkSingleInstanceStoppable(clusterId, instanceName, jsonContent); + } catch (HelixException e) { + _logger + .error(String.format("Current cluster %s has issue with health checks!", clusterId), e); + return serverError(e); + } return OK(objectMapper.writeValueAsString(stoppableCheck)); } @@ -606,4 +688,47 @@ public class InstanceAccessor extends AbstractHelixResource { private boolean validInstance(JsonNode node, String instanceName) { return instanceName.equals(node.get(Properties.id.name()).getValueAsText()); } + + private List<String> getZoneBasedInstance(String clusterId, List<String> instances, List<String> orderOfZone) { + ClusterService + clusterService = new ClusterServiceImpl(getDataAccssor(clusterId), getConfigAccessor()); + Map<String, Set<String>> zoneMapping = clusterService.getClusterTopology(clusterId).toZoneMapping(); + if (orderOfZone == null) { + orderOfZone = new ArrayList<>(zoneMapping.keySet()); + } + Collections.sort(orderOfZone); + if (orderOfZone.isEmpty()) { + return orderOfZone; + } + + Set<String> instanceSet = null; + for (String zone : orderOfZone) { + instanceSet = new TreeSet<>(instances); + Set<String> currentZoneInstanceSet = new HashSet<>(zoneMapping.get(zone)); + instanceSet.retainAll(currentZoneInstanceSet); + if (instanceSet.size() > 0) { + return new ArrayList<>(instanceSet); + } + } + + return Collections.EMPTY_LIST; + } + + private StoppableCheck checkSingleInstanceStoppable(String clusterId, String instanceName, + String jsonContent) { + HelixDataAccessor dataAccessor = getDataAccssor(clusterId); + // TODO reduce GC by dependency injection + InstanceService instanceService = new InstanceServiceImpl( + new HelixDataAccessorWrapper((ZKHelixDataAccessor) dataAccessor), getConfigAccessor()); + + Map<String, Boolean> helixStoppableCheck = instanceService.getInstanceHealthStatus(clusterId, + instanceName, InstanceService.HealthCheck.STOPPABLE_CHECK_LIST); + CustomRestClient customClient = CustomRestClientFactory.get(jsonContent); + // TODO add the json content parse logic + Map<String, Boolean> customStoppableCheck = + customClient.getInstanceStoppableCheck(Collections.<String, String> emptyMap()); + StoppableCheck stoppableCheck = + StoppableCheck.mergeStoppableChecks(helixStoppableCheck, customStoppableCheck); + return stoppableCheck; + } } diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/service/InstanceServiceImpl.java b/helix-rest/src/main/java/org/apache/helix/rest/server/service/InstanceServiceImpl.java index 3f8a7cb..c7ef015 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/server/service/InstanceServiceImpl.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/service/InstanceServiceImpl.java @@ -20,10 +20,9 @@ package org.apache.helix.rest.server.service; */ import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; - +import java.util.TreeMap; import org.apache.helix.ConfigAccessor; import org.apache.helix.HelixDataAccessor; import org.apache.helix.HelixException; @@ -49,7 +48,7 @@ public class InstanceServiceImpl implements InstanceService { @Override public Map<String, Boolean> getInstanceHealthStatus(String clusterId, String instanceName, List<HealthCheck> healthChecks) { - Map<String, Boolean> healthStatus = new HashMap<>(); + Map<String, Boolean> healthStatus = new TreeMap<>(); for (HealthCheck healthCheck : healthChecks) { switch (healthCheck) { case INVALID_CONFIG: @@ -68,13 +67,8 @@ public class InstanceServiceImpl implements InstanceService { InstanceValidationUtil.isAlive(_dataAccessor, clusterId, instanceName)); break; case INSTANCE_NOT_STABLE: - try { - boolean isStable = InstanceValidationUtil.isInstanceStable(_dataAccessor, instanceName); - healthStatus.put(HealthCheck.INSTANCE_NOT_STABLE.name(), isStable); - } catch (HelixException e) { - _logger.error("Failed to check instance is stable, message: {}", e.getMessage()); - // TODO action on the stable check exception - } + boolean isStable = InstanceValidationUtil.isInstanceStable(_dataAccessor, instanceName); + healthStatus.put(HealthCheck.INSTANCE_NOT_STABLE.name(), isStable); break; case HAS_ERROR_PARTITION: healthStatus.put(HealthCheck.HAS_ERROR_PARTITION.name(), diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstanceAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstanceAccessor.java index b92ce26..a4d0156 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstanceAccessor.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstanceAccessor.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import javax.ws.rs.client.Entity; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -37,11 +38,14 @@ import org.apache.helix.HelixException; import org.apache.helix.TestHelper; import org.apache.helix.ZNRecord; import org.apache.helix.manager.zk.ZKHelixDataAccessor; +import org.apache.helix.model.ClusterConfig; import org.apache.helix.model.InstanceConfig; import org.apache.helix.model.Message; import org.apache.helix.rest.server.resources.AbstractResource; import org.apache.helix.rest.server.resources.helix.InstanceAccessor; import org.apache.helix.rest.server.util.JerseyUriRequestBuilder; +import org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier; +import org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier; import org.apache.helix.util.InstanceValidationUtil; import org.codehaus.jackson.JsonNode; import org.testng.Assert; @@ -54,17 +58,75 @@ public class TestInstanceAccessor extends AbstractTestClass { private final static String CLUSTER_NAME = "TestCluster_0"; private final static String INSTANCE_NAME = CLUSTER_NAME + "localhost_12918"; + @Test + public void testEndToEndChecks() { + System.out.println("Start test :" + TestHelper.getTestMethodName()); + String clusterName = TestHelper.getTestMethodName(); + List<String> instances = + Arrays.asList("instance0", "instance1", "instance2", "instance3", "instance4", "instance5"); + preSetupForParallelInstancesStoppableTest(clusterName, instances); + + // Select instances with zone based + String content = String + .format("{\"%s\":\"%s\",\"%s\":[\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"]}", + InstanceAccessor.InstanceProperties.selection_base.name(), + InstanceAccessor.InstanceHealthSelectionBase.zone_based.name(), + InstanceAccessor.InstanceProperties.instances.name(), "instance0", "instance1", + "instance2", "instance3", "instance4", "instance5"); + Response response = + new JerseyUriRequestBuilder("clusters/{}/instances/stoppable").format(clusterName) + .post(this, Entity.entity(content, MediaType.APPLICATION_JSON_TYPE)); + String checkResult = response.readEntity(String.class); + Assert.assertEquals(checkResult, + "{\n \"instance_stoppable_parallel\" : [ \"instance0\", \"instance2\" ],\n" + + " \"instance_not_stoppable_with_reasons\" : {\n \"instance1\" : [ \"Helix:INSTANCE_NOT_STABLE\", \"Helix:INSTANCE_NOT_ENABLED\", \"Helix:EMPTY_RESOURCE_ASSIGNMENT\" ],\n" + + " \"instance3\" : [ \"Helix:HAS_DISABLED_PARTITION\" ],\n" + + " \"instance4\" : [ \"Helix:INSTANCE_NOT_STABLE\", \"Helix:INSTANCE_NOT_ALIVE\", \"Helix:EMPTY_RESOURCE_ASSIGNMENT\" ]\n }\n}\n"); + + // Disable one selected instance0, it should failed to check + String instance = "instance0"; + InstanceConfig instanceConfig = _configAccessor.getInstanceConfig(clusterName, instance); + instanceConfig.setInstanceEnabled(false); + instanceConfig.setInstanceEnabledForPartition("FakeResource", "FakePartition", false); + _configAccessor.setInstanceConfig(clusterName, instance, instanceConfig); + + Entity entity = + Entity.entity("", MediaType.APPLICATION_JSON_TYPE); + response = new JerseyUriRequestBuilder("clusters/{}/instances/{}/stoppable") + .format(clusterName, instance).post(this, entity); + checkResult = response.readEntity(String.class); + Assert.assertEquals(checkResult, + "{\"stoppable\":false,\"failedChecks\":[\"Helix:INSTANCE_NOT_STABLE\",\"Helix:HAS_DISABLED_PARTITION\",\"Helix:INSTANCE_NOT_ENABLED\"]}"); + + // Reenable instance0, it should passed the check + instanceConfig.setInstanceEnabled(true); + instanceConfig.setInstanceEnabledForPartition("FakeResource", "FakePartition", true); + _configAccessor.setInstanceConfig(clusterName, instance, instanceConfig); + HelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build(); + Assert.assertTrue(((BestPossibleExternalViewVerifier) verifier).verifyByPolling()); + + entity = + Entity.entity("", MediaType.APPLICATION_JSON_TYPE); + response = new JerseyUriRequestBuilder("clusters/{}/instances/{}/stoppable") + .format(clusterName, instance).post(this, entity); + checkResult = response.readEntity(String.class); + Assert.assertEquals(checkResult, + "{\"stoppable\":true,\"failedChecks\":[]}"); + } + + @Test(dependsOnMethods = "testEndToEndChecks") public void testIsInstanceStoppable() throws IOException { System.out.println("Start test :" + TestHelper.getTestMethodName()); Map<String, String> params = ImmutableMap.of("client", "espresso"); Entity entity = Entity.entity(OBJECT_MAPPER.writeValueAsString(params), MediaType.APPLICATION_JSON_TYPE); Response response = new JerseyUriRequestBuilder("clusters/{}/instances/{}/stoppable") - .format(CLUSTER_NAME, INSTANCE_NAME).post(this, entity); + .format("testEndToEndChecks", "instance1").post(this, entity); String stoppableCheckResult = response.readEntity(String.class); Assert.assertEquals(stoppableCheckResult, - "{\"stoppable\":false,\"failedChecks\":[\"Helix:EMPTY_RESOURCE_ASSIGNMENT\",\"Helix:INSTANCE_NOT_ALIVE\"]}"); + "{\"stoppable\":false,\"failedChecks\":[\"Helix:INSTANCE_NOT_STABLE\"," + + "\"Helix:INSTANCE_NOT_ENABLED\",\"Helix:EMPTY_RESOURCE_ASSIGNMENT\"]}"); } @Test (dependsOnMethods = "testIsInstanceStoppable") @@ -419,6 +481,7 @@ public class TestInstanceAccessor extends AbstractTestClass { .format(CLUSTER_NAME, instanceName).post(this, entity); } + @Test(dependsOnMethods = "checkUpdateFails") public void testCustomizedChecks() { // TODO: This is fake testing. Only validate it returns true value of this function. // For future, we need test: 1. mock the input of per participant API result to test validate logic @@ -430,4 +493,33 @@ public class TestInstanceAccessor extends AbstractTestClass { .checkCustomizedHealthStatusForInstance(_configAccessor, CLUSTER_NAME, instanceName, Collections.EMPTY_MAP, Collections.EMPTY_MAP, Collections.EMPTY_MAP)); } + + private void preSetupForParallelInstancesStoppableTest(String clusterName, List<String> instances) { + _gSetupTool.addCluster(clusterName, true); + ClusterConfig clusterConfig = _configAccessor.getClusterConfig(clusterName); + clusterConfig.setFaultZoneType("helixZoneId"); + clusterConfig.setPersistIntermediateAssignment(true); + _configAccessor.setClusterConfig(clusterName, clusterConfig); + // Create instance configs + List<InstanceConfig> instanceConfigs = new ArrayList<>(); + for (int i = 0; i < instances.size() - 1; i++) { + InstanceConfig instanceConfig = new InstanceConfig(instances.get(i)); + instanceConfig.setDomain("helixZoneId=zone1,host=instance" + i); + instanceConfigs.add(instanceConfig); + } + instanceConfigs.add(new InstanceConfig(instances.get(instances.size() - 1))); + instanceConfigs.get(instanceConfigs.size() - 1).setDomain("helixZoneId=zone2,host=instance5"); + + instanceConfigs.get(1).setInstanceEnabled(false); + instanceConfigs.get(3).setInstanceEnabledForPartition("FakeResource", "FakePartition", false); + + for (InstanceConfig instanceConfig : instanceConfigs) { + _gSetupTool.getClusterManagementTool().addInstance(clusterName, instanceConfig); + } + + // Start participant + startInstances(clusterName, new TreeSet<>(instances), 3); + createResources(clusterName, 1); + startController(clusterName); + } }
