HBASE-16794 TestDispatchMergingRegionsProcedure#testMergeRegionsConcurrently is flaky
Signed-off-by: Matteo Bertozzi <matteo.berto...@cloudera.com> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8a8c6088 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8a8c6088 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8a8c6088 Branch: refs/heads/hbase-12439 Commit: 8a8c60889cf67b581d7adb4245e0bcc02cdfdc93 Parents: e06c367 Author: ChiaPing Tsai <chia7...@gmail.com> Authored: Sun Oct 9 16:52:54 2016 -0700 Committer: Matteo Bertozzi <matteo.berto...@cloudera.com> Committed: Sun Oct 9 16:53:29 2016 -0700 ---------------------------------------------------------------------- .../hbase/regionserver/CompactSplitThread.java | 5 +++ .../TestDispatchMergingRegionsProcedure.java | 45 +++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/8a8c6088/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java index c1f82b9..a454f0e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java @@ -724,6 +724,11 @@ public class CompactSplitThread implements CompactionRequestor, PropagatingConfi } @VisibleForTesting + public long getCompletedMergeTaskCount() { + return mergePool.getCompletedTaskCount(); + } + + @VisibleForTesting /** * Shutdown the long compaction thread pool. * Should only be used in unit test to prevent long compaction thread pool from stealing job http://git-wip-us.apache.org/repos/asf/hbase/blob/8a8c6088/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java index 601f22f..a7dd4a8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hbase.master.procedure; +import java.io.IOException; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -36,6 +38,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.D import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -125,20 +128,21 @@ public class TestDispatchMergingRegionsProcedure { regionsToMerge[0] = tableRegions.get(0); regionsToMerge[1] = tableRegions.get(1); + final int initCompletedTaskCount = countOfCompletedMergeTaskCount(); long procId = procExec.submitProcedure(new DispatchMergingRegionsProcedure( procExec.getEnvironment(), tableName, regionsToMerge, true)); ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); - assertRegionCount(tableName, 2); + assertRegionCount(tableName, 2, 1, initCompletedTaskCount); } /** * This tests two concurrent region merges */ - @Test(timeout=90000) + @Test(timeout=60000) public void testMergeRegionsConcurrently() throws Exception { - final TableName tableName = TableName.valueOf("testMergeTwoRegions"); + final TableName tableName = TableName.valueOf("testMergeRegionsConcurrently"); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); List<HRegionInfo> tableRegions = createTable(tableName, 4); @@ -150,6 +154,7 @@ public class TestDispatchMergingRegionsProcedure { regionsToMerge2[0] = tableRegions.get(2); regionsToMerge2[1] = tableRegions.get(3); + final int initCompletedTaskCount = countOfCompletedMergeTaskCount(); long procId1 = procExec.submitProcedure(new DispatchMergingRegionsProcedure( procExec.getEnvironment(), tableName, regionsToMerge1, true)); long procId2 = procExec.submitProcedure(new DispatchMergingRegionsProcedure( @@ -158,8 +163,28 @@ public class TestDispatchMergingRegionsProcedure { ProcedureTestingUtility.waitProcedure(procExec, procId2); ProcedureTestingUtility.assertProcNotFailed(procExec, procId1); ProcedureTestingUtility.assertProcNotFailed(procExec, procId2); + assertRegionCount(tableName, 2, 2, initCompletedTaskCount); + } + + private void waitForCompletedMergeTask(int expectedTaskCount, int initCompletedTaskCount) + throws IOException, InterruptedException { + while (true) { + long currentCompletedTaskCount = countOfCompletedMergeTaskCount() - initCompletedTaskCount; + if (currentCompletedTaskCount == expectedTaskCount) { + return; + } + LOG.info("There are " + (expectedTaskCount - currentCompletedTaskCount) + + " merge requests are not completed, wait 100 ms"); + TimeUnit.MILLISECONDS.sleep(100); + } + } - assertRegionCount(tableName, 2); + private static int countOfCompletedMergeTaskCount() { + int completedTaskCount = 0; + for (RegionServerThread server : UTIL.getMiniHBaseCluster().getRegionServerThreads()) { + completedTaskCount += server.getRegionServer().getCompactSplitThread().getCompletedMergeTaskCount(); + } + return completedTaskCount; } @Test(timeout=60000) @@ -173,6 +198,7 @@ public class TestDispatchMergingRegionsProcedure { regionsToMerge[0] = tableRegions.get(0); regionsToMerge[1] = tableRegions.get(1); + final int initCompletedTaskCount = countOfCompletedMergeTaskCount(); long procId1 = procExec.submitProcedure(new DispatchMergingRegionsProcedure( procExec.getEnvironment(), tableName, regionsToMerge, true), nonceGroup, nonce); long procId2 = procExec.submitProcedure(new DispatchMergingRegionsProcedure( @@ -185,7 +211,7 @@ public class TestDispatchMergingRegionsProcedure { ProcedureTestingUtility.waitProcedure(procExec, procId2); ProcedureTestingUtility.assertProcNotFailed(procExec, procId2); - assertRegionCount(tableName, 2); + assertRegionCount(tableName, 2, 1, initCompletedTaskCount); } @Test(timeout=60000) @@ -202,6 +228,7 @@ public class TestDispatchMergingRegionsProcedure { regionsToMerge[0] = tableRegions.get(0); regionsToMerge[1] = tableRegions.get(1); + final int initCompletedTaskCount = countOfCompletedMergeTaskCount(); long procId = procExec.submitProcedure( new DispatchMergingRegionsProcedure( procExec.getEnvironment(), tableName, regionsToMerge, true)); @@ -211,7 +238,7 @@ public class TestDispatchMergingRegionsProcedure { MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); - assertRegionCount(tableName, 2); + assertRegionCount(tableName, 2, 1, initCompletedTaskCount); } @Test(timeout = 60000) @@ -256,6 +283,12 @@ public class TestDispatchMergingRegionsProcedure { return tableRegions; } + public List<HRegionInfo> assertRegionCount(final TableName tableName, final int nregions, + int expectedTaskCount, int initCompletedTaskCount) throws Exception { + waitForCompletedMergeTask(expectedTaskCount, initCompletedTaskCount); + return assertRegionCount(tableName, nregions); + } + private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() { return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); }