HBASE-16499 slow replication for small HBase clusters Signed-off-by: Ashish Singhi <ashishsin...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5937202f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5937202f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5937202f Branch: refs/heads/HBASE-19064 Commit: 5937202fd5d6c5fba74bae21846f62da4ee35583 Parents: b1b0db3 Author: Ashish Singhi <ashishsin...@apache.org> Authored: Wed Apr 4 09:54:41 2018 +0530 Committer: Ashish Singhi <ashishsin...@apache.org> Committed: Wed Apr 4 09:54:41 2018 +0530 ---------------------------------------------------------------------- .../regionserver/ReplicationSinkManager.java | 2 +- .../TestReplicationSinkManager.java | 36 +++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/5937202f/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java index af6888c..3cd7884 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java @@ -58,7 +58,7 @@ public class ReplicationSinkManager { * Default ratio of the total number of peer cluster region servers to consider * replicating to. */ - static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.1f; + static final float DEFAULT_REPLICATION_SOURCE_RATIO = 0.5f; private final Connection conn; http://git-wip-us.apache.org/repos/asf/hbase/blob/5937202f/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java index 3be3bfb..39dabb4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSinkManager.java @@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint; -import org.apache.hadoop.hbase.replication.ReplicationPeers; import org.apache.hadoop.hbase.replication.regionserver.ReplicationSinkManager.SinkPeer; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -49,13 +48,11 @@ public class TestReplicationSinkManager { private static final String PEER_CLUSTER_ID = "PEER_CLUSTER_ID"; - private ReplicationPeers replicationPeers; private HBaseReplicationEndpoint replicationEndpoint; private ReplicationSinkManager sinkManager; @Before public void setUp() { - replicationPeers = mock(ReplicationPeers.class); replicationEndpoint = mock(HBaseReplicationEndpoint.class); sinkManager = new ReplicationSinkManager(mock(ClusterConnection.class), PEER_CLUSTER_ID, replicationEndpoint, new Configuration()); @@ -64,7 +61,8 @@ public class TestReplicationSinkManager { @Test public void testChooseSinks() { List<ServerName> serverNames = Lists.newArrayList(); - for (int i = 0; i < 20; i++) { + int totalServers = 20; + for (int i = 0; i < totalServers; i++) { serverNames.add(mock(ServerName.class)); } @@ -73,7 +71,8 @@ public class TestReplicationSinkManager { sinkManager.chooseSinks(); - assertEquals(2, sinkManager.getNumSinks()); + int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO); + assertEquals(expected, sinkManager.getNumSinks()); } @@ -117,7 +116,8 @@ public class TestReplicationSinkManager { @Test public void testReportBadSink_PastThreshold() { List<ServerName> serverNames = Lists.newArrayList(); - for (int i = 0; i < 30; i++) { + int totalServers = 30; + for (int i = 0; i < totalServers; i++) { serverNames.add(mock(ServerName.class)); } when(replicationEndpoint.getRegionServers()) @@ -126,7 +126,8 @@ public class TestReplicationSinkManager { sinkManager.chooseSinks(); // Sanity check - assertEquals(3, sinkManager.getNumSinks()); + int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO); + assertEquals(expected, sinkManager.getNumSinks()); ServerName serverName = sinkManager.getSinksForTesting().get(0); @@ -139,7 +140,7 @@ public class TestReplicationSinkManager { // Reporting a bad sink more than the threshold count should remove it // from the list of potential sinks - assertEquals(2, sinkManager.getNumSinks()); + assertEquals(expected - 1, sinkManager.getNumSinks()); // // now try a sink that has some successes @@ -154,23 +155,24 @@ public class TestReplicationSinkManager { sinkManager.reportBadSink(sinkPeer); // did not remove the sink, since we had one successful try - assertEquals(2, sinkManager.getNumSinks()); + assertEquals(expected - 1, sinkManager.getNumSinks()); for (int i = 0; i <= ReplicationSinkManager.DEFAULT_BAD_SINK_THRESHOLD-2; i++) { sinkManager.reportBadSink(sinkPeer); } // still not remove, since the success reset the counter - assertEquals(2, sinkManager.getNumSinks()); + assertEquals(expected - 1, sinkManager.getNumSinks()); sinkManager.reportBadSink(sinkPeer); // but we exhausted the tries - assertEquals(1, sinkManager.getNumSinks()); + assertEquals(expected - 2, sinkManager.getNumSinks()); } @Test public void testReportBadSink_DownToZeroSinks() { List<ServerName> serverNames = Lists.newArrayList(); - for (int i = 0; i < 20; i++) { + int totalServers = 4; + for (int i = 0; i < totalServers; i++) { serverNames.add(mock(ServerName.class)); } when(replicationEndpoint.getRegionServers()) @@ -179,9 +181,9 @@ public class TestReplicationSinkManager { sinkManager.chooseSinks(); // Sanity check - List<ServerName> sinkList = sinkManager.getSinksForTesting(); - assertEquals(2, sinkList.size()); + int expected = (int) (totalServers * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO); + assertEquals(expected, sinkList.size()); ServerName serverNameA = sinkList.get(0); ServerName serverNameB = sinkList.get(1); @@ -195,8 +197,10 @@ public class TestReplicationSinkManager { } // We've gone down to 0 good sinks, so the replication sinks - // should have been refreshed now - assertEquals(2, sinkManager.getNumSinks()); + // should have been refreshed now, so out of 4 servers, 2 are not considered as they are + // reported as bad. + expected = (int) ((totalServers - 2) * ReplicationSinkManager.DEFAULT_REPLICATION_SOURCE_RATIO); + assertEquals(expected, sinkManager.getNumSinks()); } }
