svn commit: r1103894 - in /cassandra/branches/cassandra-0.7: CHANGES.txt src/java/org/apache/cassandra/db/HintedHandOffManager.java

[jbellis]

Author: jbellis
Date: Mon May 16 21:15:12 2011
New Revision: 1103894

URL: http://svn.apache.org/viewvc?rev=1103894&view=rev
Log:
adjust hinted handoff page size to avoid OOM with large columns
patch by jbellis; reviewed by brandonwilliams for CASSANDRA-2652

Modified:
    cassandra/branches/cassandra-0.7/CHANGES.txt
    
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/HintedHandOffManager.java

Modified: cassandra/branches/cassandra-0.7/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/CHANGES.txt?rev=1103894&r1=1103893&r2=1103894&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.7/CHANGES.txt Mon May 16 21:15:12 2011
@@ -1,3 +1,8 @@
+0.7.7
+ * adjust hinted handoff page size to avoid OOM with large columns 
+   (CASSANDRA-2652)
+
+
 0.7.6
  * force GC to reclaim disk space on flush, if necessary (CASSANDRA-2404)
  * move gossip heartbeat back to its own thread (CASSANDRA-2554)
@@ -24,7 +29,7 @@
  * initialize local ep state prior to gossip startup if needed (CASSANDRA-2638)
  * fix empty Result with secondary index when limit=1 (CASSANDRA-2628)
  * add quote-escaping via backslash to CLI (CASSANDRA-2623)
- * fig pig example script (CASSANDRA-2487)
+ * fix pig example script (CASSANDRA-2487)
  * fix dynamic snitch race in adding latencies (CASSANDRA-2618)
  * Start/stop cassandra after more important services such as mdadm in
    debian packaging (CASSANDRA-2481)

Modified: 
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/HintedHandOffManager.java
URL: 
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/HintedHandOffManager.java?rev=1103894&r1=1103893&r2=1103894&view=diff
==============================================================================
--- 
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/HintedHandOffManager.java
 (original)
+++ 
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/HintedHandOffManager.java
 Mon May 16 21:15:12 2011
@@ -116,7 +116,7 @@ public class HintedHandOffManager implem
         logger_.debug("Created HHOM instance, registered MBean.");
     }
 
-    private static boolean sendMessage(InetAddress endpoint, String tableName, 
String cfName, ByteBuffer key) throws IOException
+    private static boolean sendRow(InetAddress endpoint, String tableName, 
String cfName, ByteBuffer key) throws IOException
     {
         if (!Gossiper.instance.isKnownEndpoint(endpoint))
         {
@@ -131,10 +131,21 @@ public class HintedHandOffManager implem
         Table table = Table.open(tableName);
         DecoratedKey dkey = StorageService.getPartitioner().decorateKey(key);
         ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
+
+        int pageSize = PAGE_SIZE;
+        // send less columns per page if they are very large
+        if (cfs.getMeanColumns() > 0)
+        {
+            int averageColumnSize = (int) (cfs.getMeanRowSize() / 
cfs.getMeanColumns());
+            pageSize = Math.min(PAGE_SIZE, 
DatabaseDescriptor.getInMemoryCompactionLimit() / averageColumnSize);
+            pageSize = Math.max(2, pageSize); // page size of 1 does not allow 
actual paging b/c of >= behavior on startColumn
+            logger_.debug("average hinted-row column size is {}; using 
pageSize of {}", averageColumnSize, pageSize);
+        }
+
         ByteBuffer startColumn = ByteBufferUtil.EMPTY_BYTE_BUFFER;
         while (true)
         {
-            QueryFilter filter = QueryFilter.getSliceFilter(dkey, new 
QueryPath(cfs.getColumnFamilyName()), startColumn, 
ByteBufferUtil.EMPTY_BYTE_BUFFER, false, PAGE_SIZE);
+            QueryFilter filter = QueryFilter.getSliceFilter(dkey, new 
QueryPath(cfs.getColumnFamilyName()), startColumn, 
ByteBufferUtil.EMPTY_BYTE_BUFFER, false, pageSize);
             ColumnFamily cf = cfs.getColumnFamily(filter);
             if (pagingFinished(cf, startColumn))
                 break;
@@ -328,7 +339,7 @@ public class HintedHandOffManager implem
                 for (IColumn tableCF : tableCFs)
                 {
                     String[] parts = getTableAndCFNames(tableCF.name());
-                    if (sendMessage(endpoint, parts[0], parts[1], 
keyColumn.name()))
+                    if (sendRow(endpoint, parts[0], parts[1], 
keyColumn.name()))
                     {
                         deleteHintKey(endpointAsUTF8, keyColumn.name(), 
tableCF.name(), tableCF.timestamp());
                         rowsReplayed++;