Author: jbellis
Date: Wed Aug 17 05:52:45 2011
New Revision: 1158528
URL: http://svn.apache.org/viewvc?rev=1158528&view=rev
Log:
merge from 0.8
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/CHANGES.txt
cassandra/trunk/contrib/ (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
(props changed)
cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g
cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Aug 17 05:52:45 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7:1026516-1151306
/cassandra/branches/cassandra-0.7.0:1053690-1055654
-/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1157377
+/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1158501
/cassandra/branches/cassandra-0.8.0:1125021-1130369
/cassandra/branches/cassandra-0.8.1:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
Modified: cassandra/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1158528&r1=1158527&r2=1158528&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Wed Aug 17 05:52:45 2011
@@ -45,6 +45,10 @@
in a commitlog segment (CASSANDRA-3021)
* fix cassandra.bat when CASSANDRA_HOME contains spaces (CASSANDRA-2952)
* fix to SSTableSimpleUnsortedWriter bufferSize calculation (CASSANDRA-3027)
+ * make cleanup and normal compaction able to skip empty rows
+ (rows containing nothing but expired tombstones) (CASSANDRA-3039)
+ * work around native memory leak in com.sun.management.GarbageCollectorMXBean
+ (CASSANDRA-2868)
0.8.4
Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Aug 17 05:52:45 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
/cassandra/branches/cassandra-0.7/contrib:1026516-1151306
/cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
-/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1157377
+/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1158501
/cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369
/cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Aug 17 05:52:45 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1151306
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1157377
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1158501
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Aug 17 05:52:45 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1151306
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1157377
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1158501
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Aug 17 05:52:45 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1151306
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1157377
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1158501
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Aug 17 05:52:45 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1151306
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1157377
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1158501
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Aug 17 05:52:45 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1151306
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1157377
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1158501
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
Modified: cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g?rev=1158528&r1=1158527&r2=1158528&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g Wed Aug 17 05:52:45
2011
@@ -207,7 +207,6 @@ whereClause returns [WhereClause clause]
insertStatement returns [UpdateStatement expr]
: {
Attributes attrs = new Attributes();
- Map<Term, Term> columns = new HashMap<Term, Term>();
List<Term> columnNames = new ArrayList<Term>();
List<Term> columnValues = new ArrayList<Term>();
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java?rev=1158528&r1=1158527&r2=1158528&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
Wed Aug 17 05:52:45 2011
@@ -199,6 +199,7 @@ public class CommitLog implements Commit
}
final ReplayPosition globalPosition =
Ordering.from(ReplayPosition.comparator).min(cfPositions.values());
+ Checksum checksum = new CRC32();
for (final File file : clogs)
{
final long segment =
CommitLogSegment.idFromFilename(file.getName());
@@ -237,7 +238,6 @@ public class CommitLog implements Commit
logger.debug("Reading mutation at " +
reader.getFilePointer());
long claimedCRC32;
- Checksum checksum = new CRC32();
int serializedSize;
try
{
@@ -250,6 +250,7 @@ public class CommitLog implements Commit
if (serializedSize < 10)
break;
long claimedSizeChecksum = reader.readLong();
+ checksum.reset();
checksum.update(serializedSize);
if (checksum.getValue() != claimedSizeChecksum)
break; // entry wasn't synced correctly/fully.
that's ok.
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1158528&r1=1158527&r2=1158528&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
Wed Aug 17 05:52:45 2011
@@ -700,8 +700,11 @@ public class CompactionManager implement
SSTableIdentityIterator row = (SSTableIdentityIterator)
scanner.next();
if (Range.isTokenInRanges(row.getKey().token, ranges))
{
+ AbstractCompactedRow compactedRow =
controller.getCompactedRow(row);
+ if (compactedRow.isEmpty())
+ continue;
writer = maybeCreateWriter(cfs,
compactionFileLocation, expectedBloomFilterSize, writer,
Collections.singletonList(sstable));
- writer.append(controller.getCompactedRow(row));
+ writer.append(compactedRow);
totalkeysWritten++;
}
else
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionTask.java?rev=1158528&r1=1158527&r2=1158528&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
Wed Aug 17 05:52:45 2011
@@ -152,6 +152,9 @@ public class CompactionTask extends Abst
while (nni.hasNext())
{
AbstractCompactedRow row = nni.next();
+ if (row.isEmpty())
+ continue;
+
long position = writer.append(row);
totalkeysWritten++;
Modified: cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java?rev=1158528&r1=1158527&r2=1158528&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java Wed
Aug 17 05:52:45 2011
@@ -20,11 +20,13 @@ package org.apache.cassandra.service;
*
*/
+import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
import java.util.concurrent.TimeUnit;
import javax.management.MBeanServer;
import javax.management.ObjectName;
@@ -46,32 +48,22 @@ public class GCInspector
public static final GCInspector instance = new GCInspector();
private HashMap<String, Long> gctimes = new HashMap<String, Long>();
+ private HashMap<String, Long> gccounts = new HashMap<String, Long>();
+
+ List<GarbageCollectorMXBean> beans = new
ArrayList<GarbageCollectorMXBean>();
+ MemoryMXBean membean = ManagementFactory.getMemoryMXBean();
- List<Object> beans = new ArrayList<Object>(); // these are instances of
com.sun.management.GarbageCollectorMXBean
private volatile boolean cacheSizesReduced;
public GCInspector()
{
- // we only want this class to do its thing on sun jdks, or when the
sun classes are present.
- Class gcBeanClass = null;
- try
- {
- gcBeanClass =
Class.forName("com.sun.management.GarbageCollectorMXBean");
- Class.forName("com.sun.management.GcInfo");
- }
- catch (ClassNotFoundException ex)
- {
- // this happens when using a non-sun jdk.
- logger.warn("Cannot load sun GC monitoring classes. GCInspector is
disabled.");
- }
-
MBeanServer server = ManagementFactory.getPlatformMBeanServer();
try
{
ObjectName gcName = new
ObjectName(ManagementFactory.GARBAGE_COLLECTOR_MXBEAN_DOMAIN_TYPE + ",*");
for (ObjectName name : server.queryNames(gcName, null))
{
- Object gc = ManagementFactory.newPlatformMXBeanProxy(server,
name.getCanonicalName(), gcBeanClass);
+ GarbageCollectorMXBean gc =
ManagementFactory.newPlatformMXBeanProxy(server, name.getCanonicalName(),
GarbageCollectorMXBean.class);
beans.add(gc);
}
}
@@ -98,43 +90,42 @@ public class GCInspector
private void logGCResults()
{
- for (Object gc : beans)
+ for (GarbageCollectorMXBean gc : beans)
{
- SunGcWrapper gcw = new SunGcWrapper(gc);
- if (gcw.isLastGcInfoNull())
+ Long previousTotal = gctimes.get(gc.getName());
+ Long total = gc.getCollectionTime();
+ if (previousTotal == null)
+ previousTotal = 0L;
+ if (previousTotal.equals(total))
continue;
-
- Long previous = gctimes.get(gcw.getName());
- if (previous != null && previous.longValue() ==
gcw.getCollectionTime().longValue())
- continue;
- gctimes.put(gcw.getName(), gcw.getCollectionTime());
-
- long previousMemoryUsed = 0;
- long memoryUsed = 0;
- long memoryMax = 0;
- for (Map.Entry<String, MemoryUsage> entry :
gcw.getMemoryUsageBeforeGc().entrySet())
- {
- previousMemoryUsed += entry.getValue().getUsed();
- }
- for (Map.Entry<String, MemoryUsage> entry :
gcw.getMemoryUsageAfterGc().entrySet())
- {
- MemoryUsage mu = entry.getValue();
- memoryUsed += mu.getUsed();
- memoryMax += mu.getMax();
- }
-
- String st = String.format("GC for %s: %s ms, %s reclaimed leaving
%s used; max is %s",
- gcw.getName(), gcw.getDuration(),
previousMemoryUsed - memoryUsed, memoryUsed, memoryMax);
- if (gcw.getDuration() > MIN_DURATION)
+ gctimes.put(gc.getName(), total);
+ Long duration = total - previousTotal;
+ assert duration > 0;
+
+ Long previousCount = gccounts.get(gc.getName());
+ Long count = gc.getCollectionCount();
+ if (previousCount == null)
+ previousCount = 0L;
+ gccounts.put(gc.getName(), count);
+ assert count > previousCount;
+
+ MemoryUsage mu = membean.getHeapMemoryUsage();
+ long memoryUsed = mu.getUsed();
+ long memoryMax = mu.getMax();
+
+ String st = String.format("GC for %s: %s ms for %s collections, %s
used; max is %s",
+ gc.getName(), duration, count -
previousCount, memoryUsed, memoryMax);
+ long durationPerCollection = duration / (count - previousCount);
+ if (durationPerCollection > MIN_DURATION)
logger.info(st);
else if (logger.isDebugEnabled())
logger.debug(st);
- if (gcw.getDuration() > MIN_DURATION_TPSTATS)
+ if (durationPerCollection > MIN_DURATION_TPSTATS)
StatusLogger.log();
// if we just finished a full collection and we're still using a
lot of memory, try to reduce the pressure
- if (gcw.getName().equals("ConcurrentMarkSweep"))
+ if (gc.getName().equals("ConcurrentMarkSweep"))
{
SSTableDeletingTask.rescheduleFailedTasks();
@@ -155,82 +146,4 @@ public class GCInspector
}
}
}
-
- // wrapper for sun class. this enables other jdks to compile this class.
- private static final class SunGcWrapper
- {
-
- private Map<String, MemoryUsage> usageBeforeGc = null;
- private Map<String, MemoryUsage> usageAfterGc = null;
- private String name;
- private Long collectionTime;
- private Long duration;
-
- SunGcWrapper(Object gcMxBean)
- {
- // if we've gotten this far, we've already verified that the right
classes are in the CP. Now we just
- // need to check for boneheadedness.
- // grab everything we need here so that we don't have to deal with
try/catch everywhere.
- try
- {
- assert
Class.forName("com.sun.management.GarbageCollectorMXBean").isAssignableFrom(gcMxBean.getClass());
- Method getGcInfo =
gcMxBean.getClass().getDeclaredMethod("getLastGcInfo");
- Object lastGcInfo = getGcInfo.invoke(gcMxBean);
- if (lastGcInfo != null)
- {
- usageBeforeGc = (Map<String,
MemoryUsage>)lastGcInfo.getClass().getDeclaredMethod("getMemoryUsageBeforeGc").invoke(lastGcInfo);
- usageAfterGc = (Map<String,
MemoryUsage>)lastGcInfo.getClass().getDeclaredMethod("getMemoryUsageAfterGc").invoke(lastGcInfo);
- duration =
(Long)lastGcInfo.getClass().getDeclaredMethod("getDuration").invoke(lastGcInfo);
- name =
(String)gcMxBean.getClass().getDeclaredMethod("getName").invoke(gcMxBean);
- collectionTime =
(Long)gcMxBean.getClass().getDeclaredMethod("getCollectionTime").invoke(gcMxBean);
- }
- }
- catch (ClassNotFoundException e)
- {
- throw new RuntimeException(e);
- }
- catch (NoSuchMethodException e)
- {
- throw new RuntimeException(e);
- }
- catch (IllegalAccessException e)
- {
- throw new RuntimeException(e);
- }
- catch (InvocationTargetException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- String getName()
- {
- return name;
- }
-
- Long getCollectionTime()
- {
- return collectionTime;
- }
-
- Long getDuration()
- {
- return duration;
- }
-
- Map<String, MemoryUsage> getMemoryUsageAfterGc()
- {
- return usageAfterGc;
- }
-
- Map<String, MemoryUsage> getMemoryUsageBeforeGc()
- {
- return usageBeforeGc;
- }
-
- boolean isLastGcInfoNull()
- {
- return usageBeforeGc == null;
- }
- }
}
