HBASE-11059 ZK-less region assignment
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/58549428 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/58549428 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/58549428 Branch: refs/heads/master Commit: 58549428a66550aed83bfb6f1da405decc1b0f61 Parents: 047463c Author: Jimmy Xiang <[email protected]> Authored: Wed Apr 23 16:56:36 2014 -0700 Committer: Jimmy Xiang <[email protected]> Committed: Sat Jun 14 08:39:29 2014 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/HRegionInfo.java | 1 + .../org/apache/hadoop/hbase/RegionLoad.java | 4 +- .../apache/hadoop/hbase/catalog/MetaReader.java | 10 +- .../apache/hadoop/hbase/master/RegionState.java | 37 +- .../hbase/zookeeper/ZooKeeperWatcher.java | 4 +- .../org/apache/hadoop/hbase/HConstants.java | 10 + .../generated/RegionServerStatusProtos.java | 2959 +++++++++++++++++- .../src/main/protobuf/RegionServerStatus.proto | 49 + .../hadoop/hbase/master/AssignCallable.java | 7 +- .../hadoop/hbase/master/AssignmentManager.java | 782 ++++- .../hadoop/hbase/master/MasterRpcServices.java | 35 +- .../hadoop/hbase/master/RegionStateStore.java | 226 ++ .../hadoop/hbase/master/RegionStates.java | 295 +- .../hadoop/hbase/master/ServerManager.java | 12 +- .../hadoop/hbase/master/UnAssignCallable.java | 47 + .../master/handler/ServerShutdownHandler.java | 27 +- .../hbase/regionserver/HRegionServer.java | 94 +- .../regionserver/RegionMergeTransaction.java | 111 +- .../regionserver/RegionServerServices.java | 11 + .../hbase/regionserver/SplitTransaction.java | 53 +- .../handler/CloseRegionHandler.java | 15 +- .../regionserver/handler/OpenRegionHandler.java | 46 +- .../apache/hadoop/hbase/util/ConfigUtil.java | 33 + .../hadoop/hbase/MockRegionServerServices.java | 15 +- .../apache/hadoop/hbase/TestDrainingServer.java | 1 + .../client/TestScannersFromClientSide.java | 5 +- .../hadoop/hbase/master/MockRegionServer.java | 14 +- .../hbase/master/TestAssignmentManager.java | 4 +- .../master/TestAssignmentManagerOnCluster.java | 42 +- .../hadoop/hbase/master/TestMasterFailover.java | 104 + .../hbase/master/TestOpenedRegionHandler.java | 2 + .../hadoop/hbase/master/TestRestartCluster.java | 1 + .../master/TestZKBasedOpenCloseRegion.java | 1 + .../hbase/master/TestZKLessAMOnCluster.java | 42 + .../TestEndToEndSplitTransaction.java | 8 +- .../TestRegionMergeTransactionOnCluster.java | 20 +- .../regionserver/TestRegionServerNoMaster.java | 1 + .../TestSplitTransactionOnCluster.java | 134 +- .../regionserver/TestZKLessMergeOnCluster.java | 45 + .../regionserver/TestZKLessSplitOnCluster.java | 45 + .../handler/TestCloseRegionHandler.java | 1 + .../handler/TestOpenRegionHandler.java | 1 + .../apache/hadoop/hbase/util/TestHBaseFsck.java | 6 + .../zookeeper/TestRecoverableZooKeeper.java | 2 +- 44 files changed, 4907 insertions(+), 455 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index 78e175d..cc87c49 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -1048,6 +1048,7 @@ public class HRegionInfo implements Comparable<HRegionInfo> { * @param r Result to pull from * @return A ServerName instance or null if necessary fields not found or empty. */ + // TODO: ServerName should be a better place for this method? public static ServerName getServerName(final Result r) { Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); if (cell == null || cell.getValueLength() == 0) return null; http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java index 2fa45e4..28e6d6b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java @@ -203,6 +203,8 @@ public class RegionLoad { } sb = Strings.appendKeyValue(sb, "compactionProgressPct", compactionProgressPct); + sb = Strings.appendKeyValue(sb, "completeSequenceId", + this.getCompleteSequenceId()); return sb.toString(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java index 4f15390..ec0f4a0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java @@ -160,6 +160,7 @@ public class MetaReader { * @return An {@link HTable} for <code>tableName</code> * @throws IOException */ + @SuppressWarnings("deprecation") private static HTable getHTable(final CatalogTracker catalogTracker, final TableName tableName) throws IOException { @@ -482,8 +483,11 @@ public class MetaReader { @Override void add(Result r) { if (r == null || r.isEmpty()) return; + if (HRegionInfo.getHRegionInfo(r) == null) return; ServerName sn = HRegionInfo.getServerName(r); - if (sn != null && sn.equals(serverName)) this.results.add(r); + if (sn != null && sn.equals(serverName)) { + this.results.add(r); + } } }; fullScan(catalogTracker, v); @@ -491,8 +495,8 @@ public class MetaReader { if (results != null && !results.isEmpty()) { // Convert results to Map keyed by HRI for (Result r: results) { - Pair<HRegionInfo, ServerName> p = HRegionInfo.getHRegionInfoAndServerName(r); - if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r); + HRegionInfo hri = HRegionInfo.getHRegionInfo(r); + if (hri != null) hris.put(hri, r); } } return hris; http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java index 1170387..3289ac1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java @@ -17,9 +17,6 @@ */ package org.apache.hadoop.hbase.master; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; import java.util.Date; import java.util.concurrent.atomic.AtomicLong; @@ -35,7 +32,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos; * So it is almost immutable. */ @InterfaceAudience.Private -public class RegionState implements org.apache.hadoop.io.Writable { +public class RegionState { @InterfaceAudience.Public @InterfaceStability.Evolving @@ -77,6 +74,11 @@ public class RegionState implements org.apache.hadoop.io.Writable { } public RegionState(HRegionInfo region, + State state, ServerName serverName) { + this(region, state, System.currentTimeMillis(), serverName); + } + + public RegionState(HRegionInfo region, State state, long stamp, ServerName serverName) { this.hri = region; this.state = state; @@ -383,25 +385,26 @@ public class RegionState implements org.apache.hadoop.io.Writable { } /** - * @deprecated Writables are going away + * Check if two states are the same, except timestamp */ - @Deprecated @Override - public void readFields(DataInput in) throws IOException { - hri = new HRegionInfo(); - hri.readFields(in); - state = State.valueOf(in.readUTF()); - stamp.set(in.readLong()); + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) { + return false; + } + RegionState tmp = (RegionState)obj; + return tmp.hri.equals(hri) && tmp.state == state + && ((serverName != null && serverName.equals(tmp.serverName)) + || (tmp.serverName == null && serverName == null)); } /** - * @deprecated Writables are going away + * Don't count timestamp in hash code calculation */ - @Deprecated @Override - public void write(DataOutput out) throws IOException { - hri.write(out); - out.writeUTF(state.name()); - out.writeLong(stamp.get()); + public int hashCode() { + return (serverName != null ? serverName.hashCode() * 11 : 0) + + hri.hashCode() + 5 * state.ordinal(); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java index 52981e4..6b9c8ba 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java @@ -171,7 +171,9 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable { try { // Create all the necessary "directories" of znodes ZKUtil.createWithParents(this, baseZNode); - ZKUtil.createAndFailSilent(this, assignmentZNode); + if (conf.getBoolean("hbase.assignment.usezk", true)) { + ZKUtil.createAndFailSilent(this, assignmentZNode); + } ZKUtil.createAndFailSilent(this, rsZNode); ZKUtil.createAndFailSilent(this, drainingZNode); ZKUtil.createAndFailSilent(this, tableZNode); http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 3bf1249..b27679c 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -405,6 +405,16 @@ public final class HConstants { /** The open seqnum column qualifier */ public static final byte [] SEQNUM_QUALIFIER = Bytes.toBytes("seqnumDuringOpen"); + /** The state column qualifier */ + public static final byte [] STATE_QUALIFIER = Bytes.toBytes("state"); + + /** + * The serverName column qualifier. Its the server where the region is + * transitioning on, while column server is the server where the region is + * opened on. They are the same when the region is in state OPEN. + */ + public static final byte [] SERVERNAME_QUALIFIER = Bytes.toBytes("sn"); + /** The lower-half split region column qualifier */ public static final byte [] SPLITA_QUALIFIER = Bytes.toBytes("splitA");
