This is an automated email from the ASF dual-hosted git repository. hulee pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/helix.git
commit 6e3f4734f0f43ad54f3ac7e89023a47c37d82e58 Author: Yi Wang <[email protected]> AuthorDate: Tue Mar 19 14:16:53 2019 -0700 Implementation of ClusterService's getClusterTopology method RB=1601257 G=helix-reviewers A=jxue Signed-off-by: Hunter Lee <[email protected]> --- .../java/org/apache/helix/model/ClusterConfig.java | 17 ++++- .../org/apache/helix/model/TestClusterConfig.java | 18 +++++ helix-rest/pom.xml | 5 ++ .../rest/server/json/cluster/ClusterTopology.java | 26 +++---- .../server/resources/helix/ClusterAccessor.java | 20 ++++- .../rest/server/service/ClusterServiceImpl.java | 83 ++++++++++++++++++++ .../helix/rest/server/AbstractTestClass.java | 4 + .../helix/rest/server/TestClusterAccessor.java | 21 ++++++ .../server/json/cluster/TestClusterTopology.java | 5 +- .../rest/server/service/TestClusterService.java | 88 ++++++++++++++++++++++ 10 files changed, 265 insertions(+), 22 deletions(-) diff --git a/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java b/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java index f78f586..ec9b1d3 100644 --- a/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java +++ b/helix-core/src/main/java/org/apache/helix/model/ClusterConfig.java @@ -19,13 +19,14 @@ package org.apache.helix.model; * under the License. */ -import com.google.common.collect.Maps; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; + import org.apache.helix.HelixException; import org.apache.helix.HelixProperty; import org.apache.helix.ZNRecord; @@ -34,10 +35,13 @@ import org.apache.helix.api.config.StateTransitionThrottleConfig; import org.apache.helix.api.config.StateTransitionTimeoutConfig; import org.apache.helix.api.config.ViewClusterSourceConfig; +import com.google.common.collect.Maps; + /** * Cluster configurations */ public class ClusterConfig extends HelixProperty { + private static final String TOPOLOGY_SPLITTER = "/"; /** * Configurable characteristics of a cluster. * NOTE: Do NOT use this field name directly, use its corresponding getter/setter in the @@ -334,6 +338,17 @@ public class ClusterConfig extends HelixProperty { } /** + * Get cluster topology by level. + * E.g, {zone, rack, host, instance} + * @return + */ + public String[] getTopologyLevel() { + String topology = getTopology(); + String[] parts = topology.split(TOPOLOGY_SPLITTER); + return Arrays.copyOfRange(parts, 1, parts.length); + } + + /** * Set cluster fault zone type, this should be set combined with {@link #setTopology(String)}. * @param faultZoneType */ diff --git a/helix-core/src/test/java/org/apache/helix/model/TestClusterConfig.java b/helix-core/src/test/java/org/apache/helix/model/TestClusterConfig.java new file mode 100644 index 0000000..0b4fb0a --- /dev/null +++ b/helix-core/src/test/java/org/apache/helix/model/TestClusterConfig.java @@ -0,0 +1,18 @@ +package org.apache.helix.model; + +import org.testng.Assert; +import org.testng.annotations.Test; + + +public class TestClusterConfig { + + @Test + public void testGetZoneId() { + ClusterConfig clusterConfig = new ClusterConfig("test"); + clusterConfig.setTopology("/zone/rack/host/instance"); + + String[] levels = clusterConfig.getTopologyLevel(); + + Assert.assertEquals(levels.length, 4); + } +} diff --git a/helix-rest/pom.xml b/helix-rest/pom.xml index 01eda64..390d578 100644 --- a/helix-rest/pom.xml +++ b/helix-rest/pom.xml @@ -147,6 +147,11 @@ under the License. <type>test-jar</type> <scope>test</scope> </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <resources> diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java index a04f65f..c581c5d 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java @@ -10,9 +10,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; * The Cluster Topology represents the hierarchy of the cluster: * Cluster * - Zone - * -- Rack + * -- Rack(Optional) * --- Instance - * ---- Partition * Each layer consists its id and metadata */ public class ClusterTopology { @@ -26,6 +25,14 @@ public class ClusterTopology { this.zones = zones; } + public String getClusterId() { + return clusterId; + } + + public List<Zone> getZones() { + return zones; + } + public static final class Zone { @JsonProperty("id") private final String id; @@ -53,24 +60,9 @@ public class ClusterTopology { public static final class Instance { @JsonProperty("id") private final String id; - @JsonProperty("partitions") - private List<String> partitions; public Instance(String id) { this.id = id; } - - public Instance(String id, List<String> partitions) { - this.id = id; - this.partitions = partitions; - } - - public List<String> getPartitions() { - return partitions; - } - - public void setPartitions(List<String> partitions) { - this.partitions = partitions; - } } } diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java index 4427d50..2266f97 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java @@ -19,12 +19,12 @@ package org.apache.helix.rest.server.resources.helix; * under the License. */ -import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; + import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; @@ -34,6 +34,7 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Response; + import org.apache.helix.AccessOption; import org.apache.helix.ConfigAccessor; import org.apache.helix.HelixAdmin; @@ -51,11 +52,17 @@ import org.apache.helix.model.MaintenanceSignal; import org.apache.helix.model.Message; import org.apache.helix.model.StateModelDefinition; import org.apache.helix.model.builder.HelixConfigScopeBuilder; +import org.apache.helix.rest.server.json.cluster.ClusterTopology; +import org.apache.helix.rest.server.service.ClusterService; +import org.apache.helix.rest.server.service.ClusterServiceImpl; import org.apache.helix.tools.ClusterSetup; import org.codehaus.jackson.type.TypeReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; + @Path("/clusters") public class ClusterAccessor extends AbstractHelixResource { private static Logger _logger = LoggerFactory.getLogger(ClusterAccessor.class.getName()); @@ -260,6 +267,17 @@ public class ClusterAccessor extends AbstractHelixResource { return JSONRepresentation(config.getRecord()); } + @GET + @Path("{clusterId}/topology") + public Response getClusterTopology(@PathParam("clusterId") String clusterId) throws IOException { + //TODO reduce the GC by dependency injection + ClusterService clusterService = new ClusterServiceImpl(getDataAccssor(clusterId), getConfigAccessor()); + ObjectMapper objectMapper = new ObjectMapper(); + ClusterTopology clusterTopology = clusterService.getClusterTopology(clusterId); + + return OK(objectMapper.writeValueAsString(clusterTopology)); + } + @POST @Path("{clusterId}/configs") public Response updateClusterConfig( diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/service/ClusterServiceImpl.java b/helix-rest/src/main/java/org/apache/helix/rest/server/service/ClusterServiceImpl.java new file mode 100644 index 0000000..5fb9c61 --- /dev/null +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/service/ClusterServiceImpl.java @@ -0,0 +1,83 @@ +package org.apache.helix.rest.server.service; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.helix.AccessOption; +import org.apache.helix.ConfigAccessor; +import org.apache.helix.HelixDataAccessor; +import org.apache.helix.PropertyKey; +import org.apache.helix.model.InstanceConfig; +import org.apache.helix.model.LiveInstance; +import org.apache.helix.rest.server.json.cluster.ClusterInfo; +import org.apache.helix.rest.server.json.cluster.ClusterTopology; + +public class ClusterServiceImpl implements ClusterService { + private final HelixDataAccessor _dataAccessor; + private final ConfigAccessor _configAccessor; + + public ClusterServiceImpl(HelixDataAccessor dataAccessor, ConfigAccessor configAccessor) { + _dataAccessor = dataAccessor; + _configAccessor = configAccessor; + } + + @Override + public ClusterTopology getClusterTopology(String cluster) { + String zoneField = _configAccessor.getClusterConfig(cluster).getFaultZoneType(); + PropertyKey.Builder keyBuilder = _dataAccessor.keyBuilder(); + List<InstanceConfig> instanceConfigs = + _dataAccessor.getChildValues(keyBuilder.instanceConfigs()); + Map<String, List<ClusterTopology.Instance>> instanceMapByZone = new HashMap<>(); + if (instanceConfigs != null && !instanceConfigs.isEmpty()) { + for (InstanceConfig instanceConfig : instanceConfigs) { + if (!instanceConfig.getDomainAsMap().containsKey(zoneField)) { + continue; + } + final String instanceName = instanceConfig.getInstanceName(); + final ClusterTopology.Instance instance = new ClusterTopology.Instance(instanceName); + final String zoneId = instanceConfig.getDomainAsMap().get(zoneField); + if (instanceMapByZone.containsKey(zoneId)) { + instanceMapByZone.get(zoneId).add(instance); + } else { + instanceMapByZone.put(zoneId, new ArrayList<ClusterTopology.Instance>() { + { + add(instance); + } + }); + } + } + } + List<ClusterTopology.Zone> zones = new ArrayList<>(); + for (String zoneId : instanceMapByZone.keySet()) { + ClusterTopology.Zone zone = new ClusterTopology.Zone(zoneId); + zone.setInstances(instanceMapByZone.get(zoneId)); + zones.add(zone); + } + + return new ClusterTopology(cluster, zones); + } + + @Override + public ClusterInfo getClusterInfo(String clusterId) { + ClusterInfo.Builder builder = new ClusterInfo.Builder(clusterId); + PropertyKey.Builder keyBuilder = _dataAccessor.keyBuilder(); + LiveInstance controller = + _dataAccessor.getProperty(_dataAccessor.keyBuilder().controllerLeader()); + if (controller != null) { + builder.controller(controller.getInstanceName()); + } else { + builder.controller("No Lead Controller"); + } + + return builder + .paused(_dataAccessor.getBaseDataAccessor().exists(keyBuilder.pause().getPath(), + AccessOption.PERSISTENT)) + .maintenance(_dataAccessor.getBaseDataAccessor().exists(keyBuilder.maintenance().getPath(), + AccessOption.PERSISTENT)) + .idealStates(_dataAccessor.getChildNames(keyBuilder.idealStates())) + .instances(_dataAccessor.getChildNames(keyBuilder.instances())) + .liveInstances(_dataAccessor.getChildNames(keyBuilder.liveInstances())).build(); + } +} diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java b/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java index 01dd7b8..9caed00 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java @@ -51,6 +51,7 @@ import org.apache.helix.manager.zk.ZNRecordSerializer; import org.apache.helix.manager.zk.ZkBaseDataAccessor; import org.apache.helix.manager.zk.client.DedicatedZkClientFactory; import org.apache.helix.manager.zk.client.HelixZkClient; +import org.apache.helix.model.ClusterConfig; import org.apache.helix.rest.common.ContextPropertyKeys; import org.apache.helix.rest.common.HelixRestNamespace; import org.apache.helix.rest.server.auditlog.AuditLog; @@ -264,6 +265,9 @@ public class AbstractTestClass extends JerseyTestNg.ContainerPerClassTest { for (String cluster : _clusters) { Set<String> instances = createInstances(cluster, 10); Set<String> liveInstances = startInstances(cluster, instances, 6); + ClusterConfig clusterConfig = new ClusterConfig(cluster); + clusterConfig.setFaultZoneType("helixZoneId"); + _configAccessor.setClusterConfig(cluster, clusterConfig); createResourceConfigs(cluster, 8); _workflowMap.put(cluster, createWorkflows(cluster, 3)); Set<String> resources = createResources(cluster, 8); diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java index 59da6e1..fc5d94a 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java @@ -38,12 +38,14 @@ import org.apache.helix.ZNRecord; import org.apache.helix.manager.zk.ZKHelixDataAccessor; import org.apache.helix.manager.zk.ZKUtil; import org.apache.helix.model.ClusterConfig; +import org.apache.helix.model.InstanceConfig; import org.apache.helix.model.MaintenanceSignal; import org.apache.helix.rest.common.HelixRestNamespace; import org.apache.helix.rest.server.auditlog.AuditLog; import org.apache.helix.rest.server.resources.AbstractResource; import org.apache.helix.rest.server.resources.AbstractResource.Command; import org.apache.helix.rest.server.resources.helix.ClusterAccessor; +import org.apache.helix.rest.server.util.JerseyUriRequestBuilder; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; @@ -86,6 +88,25 @@ public class TestClusterAccessor extends AbstractTestClass { } @Test(dependsOnMethods = "testGetClusters") + public void testGetClusterTopology() { + System.out.println("Start test :" + TestHelper.getTestMethodName()); + String cluster = "TestCluster_1"; + String instance = cluster + "localhost_12920"; + // set the fake zone id in instance configuration + HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(cluster, _baseAccessor); + InstanceConfig instanceConfig = + helixDataAccessor.getProperty(helixDataAccessor.keyBuilder().instanceConfig(instance)); + instanceConfig.setDomain("helixZoneId=123"); + helixDataAccessor.setProperty(helixDataAccessor.keyBuilder().instanceConfig(instance), + instanceConfig); + + String response = new JerseyUriRequestBuilder("clusters/{}/topology").format(cluster).get(this); + + Assert.assertEquals(response, + "{\"id\":\"TestCluster_1\",\"zones\":[{\"id\":\"123\",\"instances\":[{\"id\":\"TestCluster_1localhost_12920\"}]}]}"); + } + + @Test(dependsOnMethods = "testGetClusterTopology") public void testAddConfigFields() throws IOException { System.out.println("Start test :" + TestHelper.getTestMethodName()); String cluster = _clusters.iterator().next(); diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java b/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java index a2b90fe..eadaa58 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java @@ -13,9 +13,8 @@ public class TestClusterTopology { @Test public void whenSerializingClusterTopology() throws IOException { - List<String> partitions = ImmutableList.of("db0", "db1"); List<ClusterTopology.Instance> instances = - ImmutableList.of(new ClusterTopology.Instance("instance", partitions)); + ImmutableList.of(new ClusterTopology.Instance("instance")); List<ClusterTopology.Zone> zones = ImmutableList.of(new ClusterTopology.Zone("zone", instances)); @@ -24,6 +23,6 @@ public class TestClusterTopology { String result = mapper.writeValueAsString(clusterTopology); Assert.assertEquals(result, - "{\"id\":\"cluster0\",\"zones\":[{\"id\":\"zone\",\"instances\":[{\"id\":\"instance\",\"partitions\":[\"db0\",\"db1\"]}]}]}"); + "{\"id\":\"cluster0\",\"zones\":[{\"id\":\"zone\",\"instances\":[{\"id\":\"instance\"}]}]}"); } } diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/service/TestClusterService.java b/helix-rest/src/test/java/org/apache/helix/rest/server/service/TestClusterService.java new file mode 100644 index 0000000..a988326 --- /dev/null +++ b/helix-rest/src/test/java/org/apache/helix/rest/server/service/TestClusterService.java @@ -0,0 +1,88 @@ +package org.apache.helix.rest.server.service; + +import static org.mockito.Mockito.*; + +import java.util.List; + +import org.apache.helix.ConfigAccessor; +import org.apache.helix.HelixDataAccessor; +import org.apache.helix.HelixProperty; +import org.apache.helix.PropertyKey; +import org.apache.helix.model.ClusterConfig; +import org.apache.helix.model.InstanceConfig; +import org.apache.helix.rest.server.json.cluster.ClusterTopology; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableList; + + +public class TestClusterService { + private static final String TEST_CLUSTER = "Test_Cluster"; + + @Test + public void testGetClusterTopology_whenMultiZones() { + InstanceConfig instanceConfig1 = new InstanceConfig("instance0"); + instanceConfig1.setDomain("helixZoneId=zone0"); + InstanceConfig instanceConfig2 = new InstanceConfig("instance1"); + instanceConfig2.setDomain("helixZoneId=zone1"); + List<HelixProperty> instanceConfigs = (List) ImmutableList.of(instanceConfig1, instanceConfig2); + + Mock mock = new Mock(); + when(mock.dataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder(TEST_CLUSTER)); + when(mock.dataAccessor.getChildValues(any(PropertyKey.class))).thenReturn(instanceConfigs); + + ClusterTopology clusterTopology = mock.clusterService.getClusterTopology(TEST_CLUSTER); + + Assert.assertEquals(clusterTopology.getZones().size(), 2); + Assert.assertEquals(clusterTopology.getClusterId(), TEST_CLUSTER); + } + + @Test + public void testGetClusterTopology_whenZeroZones() { + InstanceConfig instanceConfig1 = new InstanceConfig("instance0"); + InstanceConfig instanceConfig2 = new InstanceConfig("instance1"); + List<HelixProperty> instanceConfigs = (List) ImmutableList.of(instanceConfig1, instanceConfig2); + + Mock mock = new Mock(); + when(mock.dataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder(TEST_CLUSTER)); + when(mock.dataAccessor.getChildValues(any(PropertyKey.class))).thenReturn(instanceConfigs); + + ClusterTopology clusterTopology = mock.clusterService.getClusterTopology(TEST_CLUSTER); + + Assert.assertEquals(clusterTopology.getZones().size(), 0); + Assert.assertEquals(clusterTopology.getClusterId(), TEST_CLUSTER); + } + + @Test + public void testGetClusterTopology_whenZoneHasMultiInstances() { + InstanceConfig instanceConfig1 = new InstanceConfig("instance0"); + instanceConfig1.setDomain("helixZoneId=zone0"); + InstanceConfig instanceConfig2 = new InstanceConfig("instance1"); + instanceConfig2.setDomain("helixZoneId=zone0"); + List<HelixProperty> instanceConfigs = (List) ImmutableList.of(instanceConfig1, instanceConfig2); + + Mock mock = new Mock(); + when(mock.dataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder(TEST_CLUSTER)); + when(mock.dataAccessor.getChildValues(any(PropertyKey.class))).thenReturn(instanceConfigs); + + ClusterTopology clusterTopology = mock.clusterService.getClusterTopology(TEST_CLUSTER); + + Assert.assertEquals(clusterTopology.getZones().size(), 1); + Assert.assertEquals(clusterTopology.getZones().get(0).getInstances().size(), 2); + Assert.assertEquals(clusterTopology.getClusterId(), TEST_CLUSTER); + } + + private final class Mock { + private HelixDataAccessor dataAccessor = mock(HelixDataAccessor.class); + private ConfigAccessor configAccessor = mock(ConfigAccessor.class); + private ClusterService clusterService; + + Mock() { + ClusterConfig mockConfig = new ClusterConfig(TEST_CLUSTER); + mockConfig.setFaultZoneType("helixZoneId"); + when(configAccessor.getClusterConfig(TEST_CLUSTER)).thenReturn(mockConfig); + clusterService = new ClusterServiceImpl(dataAccessor, configAccessor); + } + } +}
