svn commit: r1173336 - in /hbase/branches/0.92: CHANGES.txt src/main/java/org/apache/hadoop/hbase/regionserver/Store.java

[tedyu]

Author: tedyu
Date: Tue Sep 20 20:22:48 2011
New Revision: 1173336

URL: http://svn.apache.org/viewvc?rev=1173336&view=rev
Log:
HBASE-3421  Very wide rows -- 30M plus -- cause us OOME (Nate Putnam)

Modified:
    hbase/branches/0.92/CHANGES.txt
    
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java

Modified: hbase/branches/0.92/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hbase/branches/0.92/CHANGES.txt?rev=1173336&r1=1173335&r2=1173336&view=diff
==============================================================================
--- hbase/branches/0.92/CHANGES.txt (original)
+++ hbase/branches/0.92/CHANGES.txt Tue Sep 20 20:22:48 2011
@@ -288,6 +288,7 @@ Release 0.91.0 - Unreleased
    HBASE-4428  Two methods in CacheTestUtils don't call setDaemon() on the 
threads
    HBASE-4400  .META. getting stuck if RS hosting it is dead and znode state 
is in
                RS_ZK_REGION_OPENED (Ramkrishna)
+   HBASE-3421  Very wide rows -- 30M plus -- cause us OOME (Nate Putnam)
 
   IMPROVEMENTS
    HBASE-3290  Max Compaction Size (Nicolas Spiegelberg via Stack)  

Modified: 
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: 
http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1173336&r1=1173335&r2=1173336&view=diff
==============================================================================
--- 
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
 (original)
+++ 
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
 Tue Sep 20 20:22:48 2011
@@ -118,6 +118,7 @@ public class Store implements HeapSize {
   private final String storeNameStr;
   private final boolean inMemory;
   private CompactionProgress progress;
+  private final int compactionKVMax;
 
   /*
    * List of store files inside this store. This is an immutable list that
@@ -210,6 +211,7 @@ public class Store implements HeapSize {
     this.maxCompactSize
       = conf.getLong("hbase.hstore.compaction.max.size", Long.MAX_VALUE);
     this.compactRatio = conf.getFloat("hbase.hstore.compaction.ratio", 1.2F);
+    this.compactionKVMax = conf.getInt("hbase.hstore.compaction.kv.max", 10);
 
     if (Store.closeCheckInterval == 0) {
       Store.closeCheckInterval = conf.getInt(
@@ -1146,7 +1148,8 @@ public class Store implements HeapSize {
         // since scanner.next() can return 'false' but still be delivering 
data,
         // we have to use a do/while loop.
         ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
-        while (scanner.next(kvs)) {
+        // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
+        while (scanner.next(kvs,this.compactionKVMax)) {
           if (writer == null && !kvs.isEmpty()) {
             writer = createWriterInTmp(maxKeyCount,
               this.compactionCompression);