Repository: hbase Updated Branches: refs/heads/master 17ac2fe9c -> 8ab7b20f4
HBASE-20116 Optimize the region last pushed sequence id layout on zk Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8ab7b20f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8ab7b20f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8ab7b20f Branch: refs/heads/master Commit: 8ab7b20f48951d77945181024f5e15842bc253c4 Parents: 17ac2fe Author: huzheng <open...@gmail.com> Authored: Tue Mar 20 10:13:15 2018 +0800 Committer: huzheng <open...@gmail.com> Committed: Wed Mar 21 14:24:55 2018 +0800 ---------------------------------------------------------------------- .../replication/ZKReplicationQueueStorage.java | 22 ++++++++++++++------ .../replication/TestReplicationStateBasic.java | 3 ++- .../TestZKReplicationQueueStorage.java | 10 +++++++++ 3 files changed, 28 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/8ab7b20f/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java ---------------------------------------------------------------------- diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java index 5a2b508..499ee0a 100644 --- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java +++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java @@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CollectionUtils; @@ -139,19 +140,28 @@ class ZKReplicationQueueStorage extends ZKReplicationStorageBase * So the final znode path will be format like this: * * <pre> - * /hbase/replication/regions/254/dd04e76a6966d4ffa908ed0586764767-100 + * /hbase/replication/regions/e1/ff/dd04e76a6966d4ffa908ed0586764767-100 * </pre> * - * The 254 indicate the hash of encoded region name, the 100 indicate the peer id. + * The e1 indicate the first level hash of encoded region name, and the ff indicate the second + * level hash of encoded region name, the 100 indicate the peer id. <br> + * Note that here we use two-level hash because if only one-level hash (such as mod 65535), it + * will still lead to too many children under the /hbase/replication/regions znode. * @param encodedRegionName the encoded region name. * @param peerId peer id for replication. * @return ZNode path to persist the max sequence id that we've pushed for the given region and * peer. */ - private String getSerialReplicationRegionPeerNode(String encodedRegionName, String peerId) { - int hash = encodedRegionName.hashCode() & 0x0000FFFF; - String hashPath = ZNodePaths.joinZNode(regionsZNode, String.valueOf(hash)); - return ZNodePaths.joinZNode(hashPath, String.format("%s-%s", encodedRegionName, peerId)); + @VisibleForTesting + public String getSerialReplicationRegionPeerNode(String encodedRegionName, String peerId) { + if (encodedRegionName == null || encodedRegionName.length() != RegionInfo.MD5_HEX_LENGTH) { + throw new IllegalArgumentException( + "Invalid encoded region name: " + encodedRegionName + ", length should be 32."); + } + return new StringBuilder(regionsZNode).append(ZNodePaths.ZNODE_PATH_SEPARATOR) + .append(encodedRegionName.substring(0, 2)).append(ZNodePaths.ZNODE_PATH_SEPARATOR) + .append(encodedRegionName.substring(2, 4)).append(ZNodePaths.ZNODE_PATH_SEPARATOR) + .append(encodedRegionName).append("-").append(peerId).toString(); } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/8ab7b20f/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java ---------------------------------------------------------------------- diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java index 21b09aa..3ed4121 100644 --- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java +++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java @@ -285,7 +285,8 @@ public abstract class TestReplicationStateBasic { ServerName serverName1 = ServerName.valueOf("127.0.0.1", 8000, 10000); assertTrue(rqs.getAllQueues(serverName1).isEmpty()); String queue1 = "1"; - String region0 = "region0", region1 = "region1"; + String region0 = "6b2c8f8555335cc9af74455b94516cbe", + region1 = "6ecd2e9e010499f8ddef97ee8f70834f"; for (int i = 0; i < 10; i++) { rqs.addWAL(serverName1, queue1, getFileName("file1", i)); } http://git-wip-us.apache.org/repos/asf/hbase/blob/8ab7b20f/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java ---------------------------------------------------------------------- diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java index c813870..28cdff1 100644 --- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java +++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.zookeeper.KeeperException; import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -251,4 +252,13 @@ public class TestZKReplicationQueueStorage { assertEquals(1, allHFileRefs.size()); assertThat(allHFileRefs, hasItems("test")); } + + @Test + public void testRegionsZNodeLayout() throws Exception { + String peerId = "1"; + String encodedRegionName = "31d9792f4435b99d9fb1016f6fbc8dc7"; + String expectedPath = "/hbase/replication/regions/31/d9/" + encodedRegionName + "-" + peerId; + String path = STORAGE.getSerialReplicationRegionPeerNode(encodedRegionName, peerId); + Assert.assertEquals(expectedPath, path); + } }
