svn commit: r918524 - in /incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra: config/DatabaseDescriptor.java db/CompactionManager.java io/SSTableWriter.java

[jbellis]

Author: jbellis
Date: Wed Mar  3 15:43:20 2010
New Revision: 918524

URL: http://svn.apache.org/viewvc?rev=918524&view=rev
Log:
warn when exceptionally large rows are seen during compaction.  patch by 
Brandon Williams; reviewed by jbellis for CASSANDRA-843

Modified:
    
incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
    
incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java
    
incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableWriter.java

Modified: 
incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: 
http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=918524&r1=918523&r2=918524&view=diff
==============================================================================
--- 
incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
 (original)
+++ 
incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
 Wed Mar  3 15:43:20 2010
@@ -91,6 +91,8 @@
 
     static Map<String, KSMetaData> tables = new HashMap<String, KSMetaData>();
     private static int bmtThreshold = 256;
+    /* if this a row exceeds this threshold, we issue warnings during 
compaction */
+    private static long rowWarningThreshold = 512 * 1024 * 1024;
 
     /* Hashing strategy Random or OPHF */
     private static IPartitioner partitioner;
@@ -436,6 +438,13 @@
                 columnIndexSizeInKB = Integer.parseInt(columnIndexSize);
             }
 
+            String rowWarning = 
xmlUtils.getNodeValue("/Storage/RowWarningThresholdInMB");
+            if (rowWarning != null)
+            {
+                rowWarningThreshold = Integer.parseInt(rowWarning) * 1024 * 
1024;
+                if (rowWarningThreshold <= 0)
+                    throw new ConfigurationException("Row warning threshold 
must be a positive integer");
+            }
             /* data file and commit log directories. they get created later, 
when they're needed. */
             dataFileDirectories = 
xmlUtils.getNodeValues("/Storage/DataFileDirectories/DataFileDirectory");
             logFileDirectory = 
xmlUtils.getNodeValue("/Storage/CommitLogDirectory");
@@ -982,6 +991,11 @@
         return concurrentWriters;
     }
 
+    public static long getRowWarningThreshold()
+    {
+        return rowWarningThreshold;
+    }
+    
     public static String[] getAllDataFileLocations()
     {
         return dataFileDirectories;

Modified: 
incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java
URL: 
http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java?rev=918524&r1=918523&r2=918524&view=diff
==============================================================================
--- 
incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java
 (original)
+++ 
incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java
 Wed Mar  3 15:43:20 2010
@@ -299,9 +299,15 @@
             while (nni.hasNext())
             {
                 CompactionIterator.CompactedRow row = nni.next();
+                long prevpos = writer.getFilePointer();
+
                 writer.append(row.key, row.buffer);
                 validator.add(row);
                 totalkeysWritten++;
+
+                long rowsize = writer.getFilePointer() - prevpos;
+                if (rowsize > DatabaseDescriptor.getRowWarningThreshold())
+                    logger.warn("Large row " + row.key.key + " in " + 
cfs.getColumnFamilyName() + " " + rowsize + " bytes");
             }
             validator.complete();
         }

Modified: 
incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableWriter.java
URL: 
http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableWriter.java?rev=918524&r1=918523&r2=918524&view=diff
==============================================================================
--- 
incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableWriter.java
 (original)
+++ 
incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableWriter.java
 Wed Mar  3 15:43:20 2010
@@ -170,6 +170,11 @@
         return filename;
     }
 
+    public long getFilePointer()
+    {
+        return dataFile.getFilePointer();
+    }
+    
     public static SSTableReader renameAndOpen(String dataFileName) throws 
IOException
     {
         SSTableWriter.rename(indexFilename(dataFileName));