Author: tedyu
Date: Thu Sep 22 04:31:02 2011
New Revision: 1173948
URL: http://svn.apache.org/viewvc?rev=1173948&view=rev
Log:
HBASE-4449 LoadIncrementalHFiles can't handle CFs with blooms (David Revell)
Modified:
hbase/branches/0.90/CHANGES.txt
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
Modified: hbase/branches/0.90/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1173948&r1=1173947&r2=1173948&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Thu Sep 22 04:31:02 2011
@@ -57,6 +57,7 @@ Release 0.90.5 - Unreleased
HBASE-3421 Very wide rows -- 30M plus -- cause us OOME (Nate Putnam)
HBASE-4453 TestReplication failing up on builds.a.o because already running
zk with new format root servername
+ HBASE-4449 LoadIncrementalHFiles can't handle CFs with blooms (David
Revell)
IMPROVEMENT
HBASE-4205 Enhance HTable javadoc (Eric Charles)
Modified:
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java?rev=1173948&r1=1173947&r2=1173948&view=diff
==============================================================================
---
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
(original)
+++
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
Thu Sep 22 04:31:02 2011
@@ -271,13 +271,17 @@ public class LoadIncrementalHFiles exten
halfReader = new HalfStoreFileReader(fs, inFile, null, reference);
Map<byte[], byte[]> fileInfo = halfReader.loadFileInfo();
+ // We overestimate the number of entries in each split as the number
+ // of entries in the parent file.
+ int maxBloomEntries = halfReader.getFilterEntries();
+
int blocksize = familyDescriptor.getBlocksize();
Algorithm compression = familyDescriptor.getCompression();
BloomType bloomFilterType = familyDescriptor.getBloomFilterType();
halfWriter = new StoreFile.Writer(
fs, outFile, blocksize, compression, conf, KeyValue.COMPARATOR,
- bloomFilterType, 0);
+ bloomFilterType, maxBloomEntries);
HFileScanner scanner = halfReader.getScanner(false, false);
scanner.seekTo();
do {
Modified:
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java?rev=1173948&r1=1173947&r2=1173948&view=diff
==============================================================================
---
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
(original)
+++
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
Thu Sep 22 04:31:02 2011
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.client.HT
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Test;
@@ -68,7 +69,7 @@ public class TestLoadIncrementalHFiles {
*/
@Test
public void testSimpleLoad() throws Exception {
- runTest("testSimpleLoad",
+ runTest("testSimpleLoad", BloomType.NONE,
new byte[][][] {
new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("cccc") },
new byte[][]{ Bytes.toBytes("ddd"), Bytes.toBytes("ooo") },
@@ -81,15 +82,39 @@ public class TestLoadIncrementalHFiles {
*/
@Test
public void testRegionCrossingLoad() throws Exception {
- runTest("testRegionCrossingLoad",
+ runTest("testRegionCrossingLoad", BloomType.NONE,
new byte[][][] {
new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
});
}
- private void runTest(String testName, byte[][][] hfileRanges)
- throws Exception {
+ /**
+ * Test loading into a column family that has a ROW bloom filter.
+ */
+ @Test
+ public void testRegionCrossingRowBloom() throws Exception {
+ runTest("testRegionCrossingLoadRowBloom", BloomType.ROW,
+ new byte[][][] {
+ new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
+ new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
+ });
+ }
+
+ /**
+ * Test loading into a column family that has a ROWCOL bloom filter.
+ */
+ @Test
+ public void testRegionCrossingRowColBloom() throws Exception {
+ runTest("testRegionCrossingLoadRowColBloom", BloomType.ROWCOL,
+ new byte[][][] {
+ new byte[][]{ Bytes.toBytes("aaaa"), Bytes.toBytes("eee") },
+ new byte[][]{ Bytes.toBytes("fff"), Bytes.toBytes("zzz") },
+ });
+ }
+
+ private void runTest(String testName, BloomType bloomType,
+ byte[][][] hfileRanges) throws Exception {
Path dir = HBaseTestingUtility.getTestDir(testName);
FileSystem fs = util.getTestFileSystem();
dir = dir.makeQualified(fs);
@@ -109,7 +134,9 @@ public class TestLoadIncrementalHFiles {
try {
HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
HTableDescriptor htd = new HTableDescriptor(TABLE);
- htd.addFamily(new HColumnDescriptor(FAMILY));
+ HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
+ familyDesc.setBloomFilterType(bloomType);
+ htd.addFamily(familyDesc);
admin.createTable(htd, SPLIT_KEYS);
HTable table = new HTable(util.getConfiguration(), TABLE);
