This is an automated email from the ASF dual-hosted git repository. hulee pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/helix.git
commit aebb0cfed286819029875ad60e102851d8d53cf5 Author: Yi Wang <[email protected]> AuthorDate: Tue Mar 19 14:16:16 2019 -0700 Interface design for zone mapping information RB=1578905 BUG=helix-1646 G=helix-reviewers A=jxue Signed-off-by: Hunter Lee <[email protected]> --- .../controller/rebalancer/topology/Topology.java | 2 +- .../org/apache/helix/model/InstanceConfig.java | 19 ++++- .../TestConstraintRebalanceStrategy.java | 2 +- .../rebalancer/CrushRebalancers/TestNodeSwap.java | 3 +- .../org/apache/helix/model/TestInstanceConfig.java | 22 ++++++ helix-rest/pom.xml | 10 +++ .../rest/server/json/cluster/ClusterInfo.java | 80 ++++++++++++++++++++++ .../rest/server/json/cluster/ClusterTopology.java | 76 ++++++++++++++++++++ .../helix/rest/server/service/ClusterService.java | 25 +++++++ .../rest/server/json/cluster/TestClusterInfo.java | 27 ++++++++ .../server/json/cluster/TestClusterTopology.java | 29 ++++++++ 11 files changed, 290 insertions(+), 5 deletions(-) diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/topology/Topology.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/topology/Topology.java index f5b6141..505052e 100644 --- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/topology/Topology.java +++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/topology/Topology.java @@ -270,7 +270,7 @@ public class Topology { for (String ins : _allInstances) { InstanceConfig insConfig = _instanceConfigMap.get(ins); - String domain = insConfig.getDomain(); + String domain = insConfig.getDomainAsString(); if (domain == null) { if (insConfig.getInstanceEnabled() && (_clusterConfig.getDisabledInstances() == null || !_clusterConfig.getDisabledInstances().containsKey(ins))) { diff --git a/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java b/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java index 3cc3c58..74ba9d7 100644 --- a/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java +++ b/helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java @@ -35,6 +35,8 @@ import org.apache.helix.util.HelixUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Splitter; + /** * Instance configurations */ @@ -55,6 +57,7 @@ public class InstanceConfig extends HelixProperty { DELAY_REBALANCE_ENABLED, MAX_CONCURRENT_TASK } + public static final int WEIGHT_NOT_SET = -1; public static final int MAX_CONCURRENT_TASK_NOT_SET = -1; @@ -126,11 +129,25 @@ public class InstanceConfig extends HelixProperty { * Domain represents a hierarchy identifier for an instance. * @return */ - public String getDomain() { + public String getDomainAsString() { return _record.getSimpleField(InstanceConfigProperty.DOMAIN.name()); } /** + * Parse the key value pairs of domain and return a map structure + * @return + */ + public Map<String, String> getDomainAsMap() { + String domain = getDomainAsString(); + if (domain == null || domain.isEmpty()) { + return Collections.emptyMap(); + } + + return Splitter.on(',').trimResults() + .withKeyValueSeparator(Splitter.on('=').limit(2).trimResults()).split(domain); + } + + /** * Domain represents a hierarchy identifier for an instance. * Example: "cluster=myCluster,zone=myZone1,rack=myRack,host=hostname,instance=instance001". * @return diff --git a/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestConstraintRebalanceStrategy.java b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestConstraintRebalanceStrategy.java index cb4bb75..a9e53f8 100644 --- a/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestConstraintRebalanceStrategy.java +++ b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestConstraintRebalanceStrategy.java @@ -447,7 +447,7 @@ public class TestConstraintRebalanceStrategy { domainPartitionMap.clear(); for (String partition : partitionMap.keySet()) { for (String instance : partitionMap.get(partition).keySet()) { - String domain = instanceConfigs.get(instance).getDomain().split(",")[0].split("=")[1]; + String domain = instanceConfigs.get(instance).getDomainAsString().split(",")[0].split("=")[1]; if (domainPartitionMap.containsKey(domain)) { Assert.assertFalse(domainPartitionMap.get(domain).contains(partition)); } else { diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestNodeSwap.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestNodeSwap.java index 3d20f0a..61e4d55 100644 --- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestNodeSwap.java +++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestNodeSwap.java @@ -28,7 +28,6 @@ import java.util.Map; import java.util.Set; import org.apache.helix.ConfigAccessor; -import org.apache.helix.TestHelper; import org.apache.helix.common.ZkTestBase; import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy; import org.apache.helix.controller.rebalancer.strategy.CrushRebalanceStrategy; @@ -180,7 +179,7 @@ public class TestNodeSwap extends ZkTestBase { _gSetupTool.addInstanceToCluster(CLUSTER_NAME, newParticipantName); InstanceConfig newConfig = configAccessor.getInstanceConfig(CLUSTER_NAME, newParticipantName); - newConfig.setDomain(instanceConfig.getDomain()); + newConfig.setDomain(instanceConfig.getDomainAsString()); _gSetupTool.getClusterManagementTool() .setInstanceConfig(CLUSTER_NAME, newParticipantName, newConfig); diff --git a/helix-core/src/test/java/org/apache/helix/model/TestInstanceConfig.java b/helix-core/src/test/java/org/apache/helix/model/TestInstanceConfig.java index 69a3d9f..38b1c92 100644 --- a/helix-core/src/test/java/org/apache/helix/model/TestInstanceConfig.java +++ b/helix-core/src/test/java/org/apache/helix/model/TestInstanceConfig.java @@ -19,9 +19,13 @@ package org.apache.helix.model; * under the License. */ +import java.util.Map; + +import org.apache.helix.ZNRecord; import org.testng.Assert; import org.testng.annotations.Test; + /** * Created with IntelliJ IDEA. * User: zzhang @@ -36,4 +40,22 @@ public class TestInstanceConfig { Assert.assertTrue(config.isValid(), "HELIX-65: should not check host/port existence for instance-config"); } + + @Test + public void testGetParsedDomain() { + InstanceConfig instanceConfig = new InstanceConfig(new ZNRecord("id")); + instanceConfig.setDomain("cluster=myCluster,zone=myZone1,rack=myRack,host=hostname,instance=instance001"); + + Map<String, String> parsedDomain = instanceConfig.getDomainAsMap(); + Assert.assertEquals(parsedDomain.size(), 5); + Assert.assertEquals(parsedDomain.get("zone"), "myZone1"); + } + + @Test + public void testGetParsedDomain_emptyDomain() { + InstanceConfig instanceConfig = new InstanceConfig(new ZNRecord("id")); + + Map<String, String> parsedDomain = instanceConfig.getDomainAsMap(); + Assert.assertTrue(parsedDomain.isEmpty()); + } } diff --git a/helix-rest/pom.xml b/helix-rest/pom.xml index c3e9403..01eda64 100644 --- a/helix-rest/pom.xml +++ b/helix-rest/pom.xml @@ -122,6 +122,16 @@ under the License. <version>1.8.5</version> </dependency> <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + <version>2.9.5</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>2.9.5</version> + </dependency> + <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> <version>1.2</version> diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterInfo.java b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterInfo.java new file mode 100644 index 0000000..ac1831f --- /dev/null +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterInfo.java @@ -0,0 +1,80 @@ +package org.apache.helix.rest.server.json.cluster; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + + +public class ClusterInfo { + @JsonProperty("id") + private final String id; + @JsonProperty("controller") + private final String controller; + @JsonProperty("paused") + private final boolean paused; + @JsonProperty("maintenance") + private final boolean maintenance; + @JsonProperty("resources") + private final List<String> idealStates; + @JsonProperty("instances") + private final List<String> instances; + @JsonProperty("liveInstances") + private final List<String> liveInstances; + + private ClusterInfo(Builder builder) { + id = builder.id; + controller = builder.controller; + paused = builder.paused; + maintenance = builder.maintenance; + idealStates = builder.idealStates; + instances = builder.instances; + liveInstances = builder.liveInstances; + } + + public static final class Builder { + private String id; + private String controller; + private boolean paused; + private boolean maintenance; + private List<String> idealStates; + private List<String> instances; + private List<String> liveInstances; + + public Builder(String id) { + this.id = id; + } + + public Builder controller(String controller) { + this.controller = controller; + return this; + } + + public Builder paused(boolean paused) { + this.paused = paused; + return this; + } + + public Builder maintenance(boolean maintenance) { + this.maintenance = maintenance; + return this; + } + + public Builder idealStates(List<String> idealStates) { + this.idealStates = idealStates; + return this; + } + + public Builder instances(List<String> instances) { + this.instances = instances; + return this; + } + + public Builder liveInstances(List<String> liveInstances) { + this.liveInstances = liveInstances; + return this; + } + + public ClusterInfo build() { + return new ClusterInfo(this); + } + } +} diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java new file mode 100644 index 0000000..a04f65f --- /dev/null +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/json/cluster/ClusterTopology.java @@ -0,0 +1,76 @@ +package org.apache.helix.rest.server.json.cluster; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + + +/** + * POJO class that can be easily convert to JSON object + * The Cluster Topology represents the hierarchy of the cluster: + * Cluster + * - Zone + * -- Rack + * --- Instance + * ---- Partition + * Each layer consists its id and metadata + */ +public class ClusterTopology { + @JsonProperty("id") + private final String clusterId; + @JsonProperty("zones") + private List<Zone> zones; + + public ClusterTopology(String clusterId, List<Zone> zones) { + this.clusterId = clusterId; + this.zones = zones; + } + + public static final class Zone { + @JsonProperty("id") + private final String id; + @JsonProperty("instances") + private List<Instance> instances; + + public Zone(String id) { + this.id = id; + } + + public Zone(String id, List<Instance> instances) { + this.id = id; + this.instances = instances; + } + + public List<Instance> getInstances() { + return instances; + } + + public void setInstances(List<Instance> instances) { + this.instances = instances; + } + } + + public static final class Instance { + @JsonProperty("id") + private final String id; + @JsonProperty("partitions") + private List<String> partitions; + + public Instance(String id) { + this.id = id; + } + + public Instance(String id, List<String> partitions) { + this.id = id; + this.partitions = partitions; + } + + public List<String> getPartitions() { + return partitions; + } + + public void setPartitions(List<String> partitions) { + this.partitions = partitions; + } + } +} diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/service/ClusterService.java b/helix-rest/src/main/java/org/apache/helix/rest/server/service/ClusterService.java new file mode 100644 index 0000000..7234099 --- /dev/null +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/service/ClusterService.java @@ -0,0 +1,25 @@ +package org.apache.helix.rest.server.service; + +import org.apache.helix.rest.server.json.cluster.ClusterInfo; +import org.apache.helix.rest.server.json.cluster.ClusterTopology; + + +/** + * A rest wrapper service that provides information about cluster + * TODO add more business logic and simplify the workload on ClusterAccessor + */ +public interface ClusterService { + /** + * Get cluster topology + * @param cluster + * @return + */ + ClusterTopology getClusterTopology(String cluster); + + /** + * Get cluster basic information + * @param clusterId + * @return + */ + ClusterInfo getClusterInfo(String clusterId); +} diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterInfo.java b/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterInfo.java new file mode 100644 index 0000000..1b89ec8 --- /dev/null +++ b/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterInfo.java @@ -0,0 +1,27 @@ +package org.apache.helix.rest.server.json.cluster; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; + +public class TestClusterInfo { + @Test + public void whenSerializingClusterInfo() throws JsonProcessingException { + ClusterInfo clusterInfo = new ClusterInfo.Builder("cluster0") + .controller("controller") + .idealStates(ImmutableList.of("idealState0")) + .instances(ImmutableList.of("instance0")) + .maintenance(true) + .paused(true) + .liveInstances(ImmutableList.of("instance0")) + .build(); + ObjectMapper mapper = new ObjectMapper(); + String result = mapper.writeValueAsString(clusterInfo); + + Assert.assertEquals(result, + "{\"id\":\"cluster0\",\"controller\":\"controller\",\"paused\":true,\"maintenance\":true,\"resources\":[\"idealState0\"],\"instances\":[\"instance0\"],\"liveInstances\":[\"instance0\"]}"); + } +} diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java b/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java new file mode 100644 index 0000000..a2b90fe --- /dev/null +++ b/helix-rest/src/test/java/org/apache/helix/rest/server/json/cluster/TestClusterTopology.java @@ -0,0 +1,29 @@ +package org.apache.helix.rest.server.json.cluster; + +import java.io.IOException; +import java.util.List; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; + +public class TestClusterTopology { + + @Test + public void whenSerializingClusterTopology() throws IOException { + List<String> partitions = ImmutableList.of("db0", "db1"); + List<ClusterTopology.Instance> instances = + ImmutableList.of(new ClusterTopology.Instance("instance", partitions)); + + List<ClusterTopology.Zone> zones = ImmutableList.of(new ClusterTopology.Zone("zone", instances)); + + ClusterTopology clusterTopology = new ClusterTopology("cluster0", zones); + ObjectMapper mapper = new ObjectMapper(); + String result = mapper.writeValueAsString(clusterTopology); + + Assert.assertEquals(result, + "{\"id\":\"cluster0\",\"zones\":[{\"id\":\"zone\",\"instances\":[{\"id\":\"instance\",\"partitions\":[\"db0\",\"db1\"]}]}]}"); + } +}
