Author: larsh
Date: Thu Apr 4 23:30:12 2013
New Revision: 1464787
URL: http://svn.apache.org/r1464787
Log:
HBASE-8169 TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS may fail
due to regions randomly assigned to a RS (Jeffrey Zhong)
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1464787&r1=1464786&r2=1464787&view=diff
==============================================================================
---
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
(original)
+++
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
Thu Apr 4 23:30:12 2013
@@ -2236,5 +2236,24 @@ public class HBaseTestingUtility {
HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), htd);
return region;
}
+
+ /**
+ * Create region split keys between startkey and endKey
+ *
+ * @param startKey
+ * @param endKey
+ * @param numRegions the number of regions to be created. it has to be
greater than 3.
+ * @return
+ */
+ public byte[][] getRegionSplitStartKeys(byte[] startKey, byte[] endKey, int
numRegions){
+ assertTrue(numRegions>3);
+ byte [][] tmpSplitKeys = Bytes.split(startKey, endKey, numRegions - 3);
+ byte [][] result = new byte[tmpSplitKeys.length+1][];
+ for (int i=0;i<tmpSplitKeys.length;i++) {
+ result[i+1] = tmpSplitKeys[i];
+ }
+ result[0] = HConstants.EMPTY_BYTE_ARRAY;
+ return result;
+ }
}
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java?rev=1464787&r1=1464786&r2=1464787&view=diff
==============================================================================
---
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
(original)
+++
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
Thu Apr 4 23:30:12 2013
@@ -26,6 +26,8 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
@@ -47,6 +49,7 @@ import org.apache.hadoop.hbase.LargeTest
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.executor.EventHandler.EventType;
import org.apache.hadoop.hbase.executor.RegionTransitionData;
import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
@@ -617,8 +620,8 @@ public class TestMasterFailover {
// Create config to use for this cluster
Configuration conf = HBaseConfiguration.create();
// Need to drop the timeout much lower
- conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
- conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 4000);
+ conf.setInt("hbase.master.assignment.timeoutmonitor.period", 4000);
+ conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 8000);
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
@@ -658,14 +661,10 @@ public class TestMasterFailover {
// disable load balancing on this master
master.balanceSwitch(false);
- // create two tables in META, each with 10 regions
+ // create two tables in META, each with 30 regions
byte [] FAMILY = Bytes.toBytes("family");
- byte [][] SPLIT_KEYS = new byte [][] {
- new byte[0], Bytes.toBytes("aaa"), Bytes.toBytes("bbb"),
- Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"),
- Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"),
- Bytes.toBytes("iii"), Bytes.toBytes("jjj")
- };
+ byte[][] SPLIT_KEYS =
+ TEST_UTIL.getRegionSplitStartKeys(Bytes.toBytes("aaa"),
Bytes.toBytes("zzz"), 30);
byte [] enabledTable = Bytes.toBytes("enabledTable");
HTableDescriptor htdEnabled = new HTableDescriptor(enabledTable);
@@ -710,11 +709,11 @@ public class TestMasterFailover {
// we'll need some regions to already be assigned out properly on live RS
List<HRegionInfo> enabledAndAssignedRegions = new ArrayList<HRegionInfo>();
- enabledAndAssignedRegions.add(enabledRegions.remove(0));
- enabledAndAssignedRegions.add(enabledRegions.remove(0));
+ enabledAndAssignedRegions.addAll(enabledRegions.subList(0, 6));
+ enabledRegions.removeAll(enabledAndAssignedRegions);
List<HRegionInfo> disabledAndAssignedRegions = new
ArrayList<HRegionInfo>();
- disabledAndAssignedRegions.add(disabledRegions.remove(0));
- disabledAndAssignedRegions.add(disabledRegions.remove(0));
+ disabledAndAssignedRegions.addAll(disabledRegions.subList(0, 6));
+ disabledRegions.removeAll(disabledAndAssignedRegions);
// now actually assign them
for (HRegionInfo hri : enabledAndAssignedRegions) {
@@ -728,15 +727,20 @@ public class TestMasterFailover {
master.assignRegion(hri);
}
+ log("Waiting for assignment to finish");
+ ZKAssign.blockUntilNoRIT(zkw);
+ master.assignmentManager.waitUntilNoRegionsInTransition(60000);
+ log("Assignment completed");
+
assertTrue(" Table must be enabled.", master.getAssignmentManager()
.getZKTable().isEnabledTable("enabledTable"));
// we also need regions assigned out on the dead server
List<HRegionInfo> enabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
- enabledAndOnDeadRegions.add(enabledRegions.remove(0));
- enabledAndOnDeadRegions.add(enabledRegions.remove(0));
+ enabledAndOnDeadRegions.addAll(enabledRegions.subList(0, 6));
+ enabledRegions.removeAll(enabledAndOnDeadRegions);
List<HRegionInfo> disabledAndOnDeadRegions = new ArrayList<HRegionInfo>();
- disabledAndOnDeadRegions.add(disabledRegions.remove(0));
- disabledAndOnDeadRegions.add(disabledRegions.remove(0));
+ disabledAndOnDeadRegions.addAll(disabledRegions.subList(0, 6));
+ disabledRegions.removeAll(disabledAndOnDeadRegions);
// set region plan to server to be killed and trigger assign
for (HRegionInfo hri : enabledAndOnDeadRegions) {
@@ -753,8 +757,25 @@ public class TestMasterFailover {
// wait for no more RIT
log("Waiting for assignment to finish");
ZKAssign.blockUntilNoRIT(zkw);
+ master.assignmentManager.waitUntilNoRegionsInTransition(60000);
log("Assignment completed");
+ // Due to master.assignRegion(hri) could fail to assign a region to a
specified RS
+ // therefore, we need make sure that regions are in the expected RS
+ verifyRegionLocation(hrs, enabledAndAssignedRegions);
+ verifyRegionLocation(hrs, disabledAndAssignedRegions);
+ verifyRegionLocation(hrsDead, enabledAndOnDeadRegions);
+ verifyRegionLocation(hrsDead, disabledAndOnDeadRegions);
+
+ assertTrue(" Didn't get enough regions of enabledTalbe on live rs.",
+ enabledAndAssignedRegions.size() >= 2);
+ assertTrue(" Didn't get enough regions of disalbedTable on live rs.",
+ disabledAndAssignedRegions.size() >= 2);
+ assertTrue(" Didn't get enough regions of enabledTalbe on dead rs.",
+ enabledAndOnDeadRegions.size() >= 2);
+ assertTrue(" Didn't get enough regions of disalbedTable on dead rs.",
+ disabledAndOnDeadRegions.size() >= 2);
+
// Stop the master
log("Aborting master");
cluster.abortMaster(0);
@@ -1009,6 +1030,21 @@ public class TestMasterFailover {
TEST_UTIL.shutdownMiniCluster();
}
+ /**
+ * Verify regions are on the expected region server
+ */
+ private void verifyRegionLocation(HRegionServer hrs, List<HRegionInfo>
regions)
+ throws IOException {
+ List<HRegionInfo> tmpOnlineRegions = hrs.getOnlineRegions();
+ Iterator<HRegionInfo> itr = regions.iterator();
+ while (itr.hasNext()) {
+ HRegionInfo tmp = itr.next();
+ if (!tmpOnlineRegions.contains(tmp)) {
+ itr.remove();
+ }
+ }
+ }
+
HRegion createRegion(final HRegionInfo hri, final Path rootdir, final
Configuration c,
final HTableDescriptor htd)
throws IOException {
