Author: tedyu
Date: Tue Aug 16 01:16:43 2011
New Revision: 1158080
URL: http://svn.apache.org/viewvc?rev=1158080&view=rev
Log:
HBASE-2399 Forced splits only act on the first family in a table (Ming Ma)
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
Modified: hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1158080&r1=1158079&r2=1158080&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Tue Aug 16 01:16:43 2011
@@ -199,6 +199,7 @@ Release 0.91.0 - Unreleased
set in config file (Ming Ma)
HBASE-4186 No region is added to regionsInTransitionInRS
HBASE-4194 RegionSplitter: Split on under-loaded region servers first
+ HBASE-2399 Forced splits only act on the first family in a table (Ming Ma)
IMPROVEMENTS
HBASE-3290 Max Compaction Size (Nicolas Spiegelberg via Stack)
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1158080&r1=1158079&r2=1158080&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Tue Aug 16 01:16:43 2011
@@ -3875,18 +3875,27 @@ public class HRegion implements HeapSize
// nothing
}
+ /**
+ * Return the splitpoint. null indicates the region isn't splittable
+ * If the splitpoint isn't explicitly specified, it will go over the stores
+ * to find the best splitpoint. Currently the criteria of best splitpoint
+ * is based on the size of the store.
+ */
public byte[] checkSplit() {
if (this.splitPoint != null) {
return this.splitPoint;
}
- byte[] splitPoint = null;
+ byte[] splitPointFromLargestStore = null;
+ long largestStoreSize = 0;
for (Store s : stores.values()) {
- splitPoint = s.checkSplit();
- if (splitPoint != null) {
- return splitPoint;
+ byte[] splitPoint = s.checkSplit();
+ long storeSize = s.getSize();
+ if (splitPoint != null && largestStoreSize < storeSize) {
+ splitPointFromLargestStore = splitPoint;
+ largestStoreSize = storeSize;
}
}
- return null;
+ return splitPointFromLargestStore;
}
/**
Modified:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1158080&r1=1158079&r2=1158080&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
(original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
Tue Aug 16 01:16:43 2011
@@ -708,6 +708,31 @@ public class HBaseTestingUtility {
* @throws IOException
*/
public HTable createTable(byte[] tableName, byte[][] families,
+ int numVersions, int blockSize) throws IOException {
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ for (byte[] family : families) {
+ HColumnDescriptor hcd = new HColumnDescriptor(family, numVersions,
+ HColumnDescriptor.DEFAULT_COMPRESSION,
+ HColumnDescriptor.DEFAULT_IN_MEMORY,
+ HColumnDescriptor.DEFAULT_BLOCKCACHE,
+ blockSize, HColumnDescriptor.DEFAULT_TTL,
+ HColumnDescriptor.DEFAULT_BLOOMFILTER,
+ HColumnDescriptor.DEFAULT_REPLICATION_SCOPE);
+ desc.addFamily(hcd);
+ }
+ getHBaseAdmin().createTable(desc);
+ return new HTable(new Configuration(getConfiguration()), tableName);
+ }
+
+ /**
+ * Create a table.
+ * @param tableName
+ * @param families
+ * @param numVersions
+ * @return An HTable instance for the created table.
+ * @throws IOException
+ */
+ public HTable createTable(byte[] tableName, byte[][] families,
int[] numVersions)
throws IOException {
HTableDescriptor desc = new HTableDescriptor(tableName);
Modified:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java?rev=1158080&r1=1158079&r2=1158080&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
(original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java Tue
Aug 16 01:16:43 2011
@@ -564,29 +564,71 @@ public class TestAdmin {
*/
@Test
public void testForceSplit() throws Exception {
- splitTest(null);
- splitTest(Bytes.toBytes("pwn"));
- }
+ byte[][] familyNames = new byte[][] { Bytes.toBytes("cf") };
+ int[] rowCounts = new int[] { 6000 };
+ int numVersions = HColumnDescriptor.DEFAULT_VERSIONS;
+ int blockSize = 256;
+ splitTest(null, familyNames, rowCounts, numVersions, blockSize);
+
+ byte[] splitKey = Bytes.toBytes(3500);
+ splitTest(splitKey, familyNames, rowCounts, numVersions, blockSize);
+ }
+
+ /**
+ * Multi-family scenario. Tests forcing split from client and
+ * having scanners successfully ride over split.
+ * @throws Exception
+ * @throws IOException
+ */
+ @Test
+ public void testForceSplitMultiFamily() throws Exception {
+ int numVersions = HColumnDescriptor.DEFAULT_VERSIONS;
+
+ // use small HFile block size so that we can have lots of blocks in HFile
+ // Otherwise, if there is only one block,
+ // HFileBlockIndex.midKey()'s value == startKey
+ int blockSize = 256;
+ byte[][] familyNames = new byte[][] { Bytes.toBytes("cf1"),
+ Bytes.toBytes("cf2") };
+
+ // one of the column families isn't splittable
+ int[] rowCounts = new int[] { 6000, 1 };
+ splitTest(null, familyNames, rowCounts, numVersions, blockSize);
+
+ rowCounts = new int[] { 1, 6000 };
+ splitTest(null, familyNames, rowCounts, numVersions, blockSize);
+
+ // one column family has much smaller data than the other
+ // the split key should be based on the largest column family
+ rowCounts = new int[] { 6000, 300 };
+ splitTest(null, familyNames, rowCounts, numVersions, blockSize);
- void splitTest(byte[] splitPoint) throws Exception {
- byte [] familyName = HConstants.CATALOG_FAMILY;
+ rowCounts = new int[] { 300, 6000 };
+ splitTest(null, familyNames, rowCounts, numVersions, blockSize);
+
+ }
+
+ void splitTest(byte[] splitPoint, byte[][] familyNames, int[] rowCounts,
+ int numVersions, int blockSize) throws Exception {
byte [] tableName = Bytes.toBytes("testForceSplit");
assertFalse(admin.tableExists(tableName));
- final HTable table = TEST_UTIL.createTable(tableName, familyName);
+ final HTable table = TEST_UTIL.createTable(tableName, familyNames,
+ numVersions, blockSize);
try {
- byte[] k = new byte[3];
int rowCount = 0;
- for (byte b1 = 'a'; b1 < 'z'; b1++) {
- for (byte b2 = 'a'; b2 < 'z'; b2++) {
- for (byte b3 = 'a'; b3 < 'z'; b3++) {
- k[0] = b1;
- k[1] = b2;
- k[2] = b3;
- Put put = new Put(k);
- put.add(familyName, new byte[0], k);
- table.put(put);
- rowCount++;
- }
+
+ // insert rows into column families. The number of rows that have values
+ // in a specific column family is decided by rowCounts[familyIndex]
+ for (int index = 0; index < familyNames.length; index++) {
+ for (int i = 0; i < rowCounts[index]; i++) {
+ byte[] k = Bytes.toBytes(i);
+ Put put = new Put(k);
+ put.add(familyNames[index], new byte[0], k);
+ table.put(put);
+ }
+
+ if ( rowCount < rowCounts[index] ) {
+ rowCount = rowCounts[index];
}
}
@@ -651,21 +693,32 @@ public class TestAdmin {
scanner.close();
assertEquals(rowCount, rows);
+ Map<HRegionInfo, HServerAddress> regions = null;
+ try {
+ regions = table.getRegionsInfo();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ assertEquals(2, regions.size());
+ HRegionInfo[] r = regions.keySet().toArray(new HRegionInfo[0]);
if (splitPoint != null) {
// make sure the split point matches our explicit configuration
- Map<HRegionInfo, HServerAddress> regions = null;
- try {
- regions = table.getRegionsInfo();
- } catch (IOException e) {
- e.printStackTrace();
- }
- assertEquals(2, regions.size());
- HRegionInfo[] r = regions.keySet().toArray(new HRegionInfo[0]);
assertEquals(Bytes.toString(splitPoint),
Bytes.toString(r[0].getEndKey()));
assertEquals(Bytes.toString(splitPoint),
Bytes.toString(r[1].getStartKey()));
LOG.debug("Properly split on " + Bytes.toString(splitPoint));
+ } else {
+ if (familyNames.length > 1) {
+ int splitKey = Bytes.toInt(r[0].getEndKey());
+ // check if splitKey is based on the largest column family
+ // in terms of it store size
+ int deltaForLargestFamily = Math.abs(rowCount/2 - splitKey);
+ for (int index = 0; index < familyNames.length; index++) {
+ int delta = Math.abs(rowCounts[index]/2 - splitKey);
+ assertTrue(delta >= deltaForLargestFamily);
+ }
+ }
}
} finally {
TEST_UTIL.deleteTable(tableName);
