HBASE-15631 Backport Regionserver Groups (HBASE-6721) to branch-1 (Francis Liu and Andrew Purtell)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c3200076 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c3200076 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c3200076 Branch: refs/heads/branch-1.4 Commit: c320007638fe57f858aeac974cdd6bbd6b9dd5eb Parents: 737b5a5 Author: Andrew Purtell <apurt...@apache.org> Authored: Mon Oct 23 14:15:06 2017 -0700 Committer: Andrew Purtell <apurt...@apache.org> Committed: Mon Oct 23 17:10:23 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/ServerName.java | 25 +- .../org/apache/hadoop/hbase/net/Address.java | 89 + hbase-it/pom.xml | 41 + .../hbase/rsgroup/IntegrationTestRSGroup.java | 99 + hbase-protocol/pom.xml | 2 + .../hbase/protobuf/generated/ClientProtos.java | 16 +- .../protobuf/generated/RSGroupAdminProtos.java | 13571 +++++++++++++++++ .../hbase/protobuf/generated/RSGroupProtos.java | 1332 ++ hbase-protocol/src/main/protobuf/RSGroup.proto | 35 + .../src/main/protobuf/RSGroupAdmin.proto | 149 + hbase-rsgroup/pom.xml | 278 + .../hadoop/hbase/rsgroup/RSGroupAdmin.java | 92 + .../hbase/rsgroup/RSGroupAdminClient.java | 212 + .../hbase/rsgroup/RSGroupAdminEndpoint.java | 1049 ++ .../hbase/rsgroup/RSGroupAdminServer.java | 516 + .../hbase/rsgroup/RSGroupBasedLoadBalancer.java | 431 + .../hadoop/hbase/rsgroup/RSGroupInfo.java | 190 + .../hbase/rsgroup/RSGroupInfoManager.java | 116 + .../hbase/rsgroup/RSGroupInfoManagerImpl.java | 795 + .../hbase/rsgroup/RSGroupProtobufUtil.java | 61 + .../hadoop/hbase/rsgroup/RSGroupSerDe.java | 88 + .../hbase/rsgroup/RSGroupableBalancer.java | 32 + .../balancer/TestRSGroupBasedLoadBalancer.java | 573 + .../hadoop/hbase/rsgroup/TestRSGroups.java | 300 + .../hadoop/hbase/rsgroup/TestRSGroupsBase.java | 815 + .../hbase/rsgroup/TestRSGroupsOfflineMode.java | 187 + .../rsgroup/VerifyingRSGroupAdminClient.java | 155 + .../hbase/tmpl/master/MasterStatusTmpl.jamon | 2 + .../apache/hadoop/hbase/LocalHBaseCluster.java | 3 + .../BaseMasterAndRegionObserver.java | 62 + .../hbase/coprocessor/BaseMasterObserver.java | 63 + .../hbase/coprocessor/MasterObserver.java | 113 + .../hadoop/hbase/master/AssignmentManager.java | 16 +- .../org/apache/hadoop/hbase/master/HMaster.java | 5 + .../hadoop/hbase/master/LoadBalancer.java | 3 + .../hbase/master/MasterCoprocessorHost.java | 160 + .../hadoop/hbase/master/MasterServices.java | 5 + .../hbase/security/access/AccessController.java | 37 + .../hbase/coprocessor/TestMasterObserver.java | 61 + .../hbase/master/MockNoopMasterServices.java | 5 + .../master/TestAssignmentManagerOnCluster.java | 127 +- .../hadoop/hbase/master/TestCatalogJanitor.java | 3 + .../hbase/master/TestMasterStatusServlet.java | 12 +- .../normalizer/TestSimpleRegionNormalizer.java | 2 +- .../security/access/TestAccessController.java | 75 + hbase-shell/pom.xml | 35 + hbase-shell/src/main/ruby/hbase.rb | 1 + hbase-shell/src/main/ruby/hbase/hbase.rb | 4 + .../src/main/ruby/hbase/rsgroup_admin.rb | 164 + hbase-shell/src/main/ruby/shell.rb | 22 + hbase-shell/src/main/ruby/shell/commands.rb | 4 + .../src/main/ruby/shell/commands/add_rsgroup.rb | 39 + .../main/ruby/shell/commands/balance_rsgroup.rb | 37 + .../src/main/ruby/shell/commands/get_rsgroup.rb | 43 + .../ruby/shell/commands/get_server_rsgroup.rb | 39 + .../ruby/shell/commands/get_table_rsgroup.rb | 40 + .../main/ruby/shell/commands/list_rsgroups.rb | 49 + .../ruby/shell/commands/move_servers_rsgroup.rb | 37 + .../commands/move_servers_tables_rsgroup.rb | 37 + .../ruby/shell/commands/move_tables_rsgroup.rb | 37 + .../main/ruby/shell/commands/remove_rsgroup.rb | 37 + .../apache/hadoop/hbase/client/TestShell.java | 2 +- .../hbase/client/rsgroup/TestShellRSGroups.java | 111 + .../src/test/ruby/shell/rsgroup_shell_test.rb | 96 + hbase-shell/src/test/ruby/test_helper.rb | 4 + pom.xml | 23 + 66 files changed, 22843 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/c3200076/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java index c90e7e1..339f588 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java @@ -24,6 +24,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.util.Addressing; @@ -106,7 +107,7 @@ public class ServerName implements Comparable<ServerName>, Serializable { this.hostnameOnly = hostname; this.port = port; this.startcode = startcode; - this.servername = getServerName(this.hostnameOnly, port, startcode); + this.servername = getServerName(hostname, port, startcode); } /** @@ -402,4 +403,26 @@ public class ServerName implements Comparable<ServerName>, Serializable { int port = Addressing.parsePort(str); return valueOf(hostname, port, -1L); } + + /** + * @return an Address constructed from the hostname and port carried by this ServerName + */ + public Address getAddress() { + return Address.fromParts(getHostname(), getPort()); + } + + /** + * @param left + * @param right + * @return True if <code>other</code> has same hostname and port. + */ + public static boolean isSameAddress(final ServerName left, + final ServerName right) { + // TODO: Make this left.getAddress().equals(right.getAddress()) + if (left == null) return false; + if (right == null) return false; + return left.getHostname().compareToIgnoreCase(right.getHostname()) == 0 && + left.getPort() == right.getPort(); + } + } http://git-wip-us.apache.org/repos/asf/hbase/blob/c3200076/hbase-common/src/main/java/org/apache/hadoop/hbase/net/Address.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/net/Address.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/net/Address.java new file mode 100644 index 0000000..15b4960 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/net/Address.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.net; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +import com.google.common.net.HostAndPort; + +/** + * An immutable type to hold a hostname and port combo, like an Endpoint + * or java.net.InetSocketAddress (but without danger of our calling + * resolve -- we do NOT want a resolve happening every time we want + * to hold a hostname and port combo). This class is also <<Comparable>>. + * <p>In implementation this class is a facade over Guava's {@link HostAndPort}. + * We cannot have Guava classes in our API hence this Type. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public class Address implements Comparable<Address> { + private HostAndPort hostAndPort; + + private Address(HostAndPort hostAndPort) { + this.hostAndPort = hostAndPort; + } + + public static Address fromParts(String hostname, int port) { + return new Address(HostAndPort.fromParts(hostname, port)); + } + + public static Address fromString(String hostnameAndPort) { + return new Address(HostAndPort.fromString(hostnameAndPort)); + } + + public String getHostname() { + return this.hostAndPort.getHostText(); + } + + public int getPort() { + return this.hostAndPort.getPort(); + } + + @Override + public String toString() { + return this.hostAndPort.toString(); + } + + @Override + // Don't use HostAndPort equals... It is wonky including + // ipv6 brackets + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other instanceof Address) { + Address that = (Address)other; + return this.getHostname().equals(that.getHostname()) && + this.getPort() == that.getPort(); + } + return false; + } + + @Override + public int hashCode() { + return this.getHostname().hashCode() ^ getPort(); + } + + @Override + public int compareTo(Address that) { + int compare = this.getHostname().compareTo(that.getHostname()); + if (compare != 0) return compare; + return this.getPort() - that.getPort(); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/c3200076/hbase-it/pom.xml ---------------------------------------------------------------------- diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index 4c87bf1..46f251c 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -265,6 +265,47 @@ </dependencies> <profiles> + <profile> + <id>rsgroup</id> + <activation> + <property> + <name>!skip-rsgroup</name> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-rsgroup</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-rsgroup</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>add-test-source</id> + <goals> + <goal>add-test-source</goal> + </goals> + <configuration> + <sources> + <source>src/test/rsgroup</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> <!-- Skip the tests in this module --> <profile> <id>skipIntegrationTests</id> http://git-wip-us.apache.org/repos/asf/hbase/blob/c3200076/hbase-it/src/test/rsgroup/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/rsgroup/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java b/hbase-it/src/test/rsgroup/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java new file mode 100644 index 0000000..e5bb995 --- /dev/null +++ b/hbase-it/src/test/rsgroup/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java @@ -0,0 +1,99 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.IntegrationTestingUtility; +import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.testclassification.IntegrationTests; +import org.junit.After; +import org.junit.Before; +import org.junit.experimental.categories.Category; + +/** + * Runs all of the units tests defined in TestGroupBase + * as an integration test. + * Requires TestRSGroupBase.NUM_SLAVE_BASE servers to run. + */ +@Category(IntegrationTests.class) +public class IntegrationTestRSGroup extends TestRSGroupsBase { + //Integration specific + private final static Log LOG = LogFactory.getLog(IntegrationTestRSGroup.class); + private static boolean initialized = false; + + @Before + public void beforeMethod() throws Exception { + if(!initialized) { + LOG.info("Setting up IntegrationTestGroup"); + LOG.info("Initializing cluster with " + NUM_SLAVES_BASE + " servers"); + TEST_UTIL = new IntegrationTestingUtility(); + ((IntegrationTestingUtility)TEST_UTIL).initializeCluster(NUM_SLAVES_BASE); + //set shared configs + admin = TEST_UTIL.getHBaseAdmin(); + cluster = TEST_UTIL.getHBaseClusterInterface(); + rsGroupAdmin = new VerifyingRSGroupAdminClient(new RSGroupAdminClient(TEST_UTIL.getConnection()), + TEST_UTIL.getConfiguration()); + LOG.info("Done initializing cluster"); + initialized = true; + //cluster may not be clean + //cleanup when initializing + afterMethod(); + } + } + + @After + public void afterMethod() throws Exception { + LOG.info("Cleaning up previous test run"); + //cleanup previous artifacts + deleteTableIfNecessary(); + deleteNamespaceIfNecessary(); + deleteGroups(); + admin.setBalancerRunning(true, true); + + LOG.info("Restoring the cluster"); + ((IntegrationTestingUtility)TEST_UTIL).restoreCluster(); + LOG.info("Done restoring the cluster"); + + TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { + @Override + public boolean evaluate() throws Exception { + LOG.info("Waiting for cleanup to finish "+ rsGroupAdmin.listRSGroups()); + //Might be greater since moving servers back to default + //is after starting a server + return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size() + >= NUM_SLAVES_BASE; + } + }); + + TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { + @Override + public boolean evaluate() throws Exception { + LOG.info("Waiting for regionservers to be registered "+ rsGroupAdmin.listRSGroups()); + //Might be greater since moving servers back to default + //is after starting a server + return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size() + == getNumServers(); + } + }); + + LOG.info("Done cleaning up previous test run"); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/c3200076/hbase-protocol/pom.xml ---------------------------------------------------------------------- diff --git a/hbase-protocol/pom.xml b/hbase-protocol/pom.xml index 496b3cf..a389d78 100644 --- a/hbase-protocol/pom.xml +++ b/hbase-protocol/pom.xml @@ -195,6 +195,8 @@ <include>RegionNormalizer.proto</include> <include>RegionServerStatus.proto</include> <include>RowProcessor.proto</include> + <include>RSGroup.proto</include> + <include>RSGroupAdmin.proto</include> <include>SecureBulkLoad.proto</include> <include>Snapshot.proto</include> <include>Table.proto</include> http://git-wip-us.apache.org/repos/asf/hbase/blob/c3200076/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java ---------------------------------------------------------------------- diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java index 06ebc65..35fddc2 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java @@ -1986,7 +1986,7 @@ public final class ClientProtos { * <code>optional bool load_column_families_on_demand = 14;</code> * * <pre> - * DO NOT add defaults to load_column_families_on_demand. + * DO NOT add defaults to load_column_families_on_demand. * </pre> */ boolean hasLoadColumnFamiliesOnDemand(); @@ -1994,7 +1994,7 @@ public final class ClientProtos { * <code>optional bool load_column_families_on_demand = 14;</code> * * <pre> - * DO NOT add defaults to load_column_families_on_demand. + * DO NOT add defaults to load_column_families_on_demand. * </pre> */ boolean getLoadColumnFamiliesOnDemand(); @@ -2515,7 +2515,7 @@ public final class ClientProtos { * <code>optional bool load_column_families_on_demand = 14;</code> * * <pre> - * DO NOT add defaults to load_column_families_on_demand. + * DO NOT add defaults to load_column_families_on_demand. * </pre> */ public boolean hasLoadColumnFamiliesOnDemand() { @@ -2525,7 +2525,7 @@ public final class ClientProtos { * <code>optional bool load_column_families_on_demand = 14;</code> * * <pre> - * DO NOT add defaults to load_column_families_on_demand. + * DO NOT add defaults to load_column_families_on_demand. * </pre> */ public boolean getLoadColumnFamiliesOnDemand() { @@ -4577,7 +4577,7 @@ public final class ClientProtos { * <code>optional bool load_column_families_on_demand = 14;</code> * * <pre> - * DO NOT add defaults to load_column_families_on_demand. + * DO NOT add defaults to load_column_families_on_demand. * </pre> */ public boolean hasLoadColumnFamiliesOnDemand() { @@ -4587,7 +4587,7 @@ public final class ClientProtos { * <code>optional bool load_column_families_on_demand = 14;</code> * * <pre> - * DO NOT add defaults to load_column_families_on_demand. + * DO NOT add defaults to load_column_families_on_demand. * </pre> */ public boolean getLoadColumnFamiliesOnDemand() { @@ -4597,7 +4597,7 @@ public final class ClientProtos { * <code>optional bool load_column_families_on_demand = 14;</code> * * <pre> - * DO NOT add defaults to load_column_families_on_demand. + * DO NOT add defaults to load_column_families_on_demand. * </pre> */ public Builder setLoadColumnFamiliesOnDemand(boolean value) { @@ -4610,7 +4610,7 @@ public final class ClientProtos { * <code>optional bool load_column_families_on_demand = 14;</code> * * <pre> - * DO NOT add defaults to load_column_families_on_demand. + * DO NOT add defaults to load_column_families_on_demand. * </pre> */ public Builder clearLoadColumnFamiliesOnDemand() {
