svn commit: r1173338 - in /hbase/branches/0.90: CHANGES.txt src/main/java/org/apache/hadoop/hbase/regionserver/Store.java

[tedyu]

Author: tedyu
Date: Tue Sep 20 20:27:33 2011
New Revision: 1173338

URL: http://svn.apache.org/viewvc?rev=1173338&view=rev
Log:
HBASE-3421  Very wide rows -- 30M plus -- cause us OOME (Nate Putnam)

Modified:
    hbase/branches/0.90/CHANGES.txt
    
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java

Modified: hbase/branches/0.90/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1173338&r1=1173337&r2=1173338&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Tue Sep 20 20:27:33 2011
@@ -54,6 +54,7 @@ Release 0.90.5 - Unreleased
    HBASE-4400  .META. getting stuck if RS hosting it is dead and znode state 
is in
                RS_ZK_REGION_OPENED (ramkrishna.s.vasudevan)
    HBASE-4445  Not passing --config when checking if distributed mode or not
+   HBASE-3421  Very wide rows -- 30M plus -- cause us OOME (Nate Putnam)
 
   IMPROVEMENT
    HBASE-4205  Enhance HTable javadoc (Eric Charles)

Modified: 
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: 
http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1173338&r1=1173337&r2=1173338&view=diff
==============================================================================
--- 
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
 (original)
+++ 
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
 Tue Sep 20 20:27:33 2011
@@ -105,6 +105,7 @@ public class Store implements HeapSize {
   final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
   private final String storeNameStr;
   private final boolean inMemory;
+  private final int compactionKVMax;
 
   /*
    * List of store files inside this store. This is an immutable list that
@@ -200,6 +201,7 @@ public class Store implements HeapSize {
     this.minCompactSize = conf.getLong("hbase.hstore.compaction.min.size",
         this.region.memstoreFlushSize);
     this.compactRatio = conf.getFloat("hbase.hstore.compaction.ratio", 1.2F);
+    this.compactionKVMax = conf.getInt("hbase.hstore.compaction.kv.max", 10);
 
     if (Store.closeCheckInterval == 0) {
       Store.closeCheckInterval = conf.getInt(
@@ -930,7 +932,8 @@ public class Store implements HeapSize {
         // since scanner.next() can return 'false' but still be delivering 
data,
         // we have to use a do/while loop.
         ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
-        while (scanner.next(kvs)) {
+        // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
+        while (scanner.next(kvs,this.compactionKVMax)) {
           if (writer == null && !kvs.isEmpty()) {
             writer = createWriterInTmp(maxKeyCount,
               this.compactionCompression);