Updated Branches: refs/heads/cassandra-1.2 2980796a7 -> 2267c2094 refs/heads/cassandra-2.0 fb5584979 -> 52b4fc393 refs/heads/trunk 74e710a12 -> eb125ad78
Add tombstone debug threshold and histogram patch by Russell Spitzer and Lyuben Todorov; reviewed by jbellis for CASSANDRA-6042 and CASSANDRA-6057 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2267c209 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2267c209 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2267c209 Branch: refs/heads/cassandra-1.2 Commit: 2267c20942feb486fbbf1bd54d7a0fbbe468ae10 Parents: 2980796 Author: Jonathan Ellis <[email protected]> Authored: Mon Sep 23 09:30:22 2013 -0500 Committer: Jonathan Ellis <[email protected]> Committed: Mon Sep 23 12:39:46 2013 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + conf/cassandra.yaml | 5 ++++ .../org/apache/cassandra/config/Config.java | 2 ++ .../cassandra/config/DatabaseDescriptor.java | 14 ++++++++++ .../apache/cassandra/db/ColumnFamilyStore.java | 28 +++++++++++++++++--- .../cassandra/db/ColumnFamilyStoreMBean.java | 9 +++++++ .../cassandra/db/filter/SliceQueryFilter.java | 19 +++++++++++-- .../cassandra/metrics/ColumnFamilyMetrics.java | 8 ++++++ .../cassandra/service/StorageService.java | 10 +++++++ .../cassandra/service/StorageServiceMBean.java | 5 ++++ 10 files changed, 95 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 124b3e4..9ed00e2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,7 @@ * Allow where clause conditions to be in parenthesis (CASSANDRA-6037) * Do not open non-ssl storage port if encryption option is all (CASSANDRA-3916) * Improve memory usage of metadata min/max column names (CASSANDRA-6077) + * Add tombstone debug threshold and histogram (CASSANDRA-6042, 6057) 1.2.10 http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/conf/cassandra.yaml ---------------------------------------------------------------------- diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 27ac09b..a3cdec7 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -440,6 +440,11 @@ snapshot_before_compaction: false # lose data on truncation or drop. auto_snapshot: true +# Log a debug message if more than this many tombstones are scanned +# in a single-partition query. Set the threshold on SliceQueryFilter +# to debug to enable. +tombstone_debug_threshold: 10000 + # Add column indexes to a row after its contents reach this size. # Increase if your column values are large, or if you have a very large # number of columns. The competing causes are, Cassandra has to http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/config/Config.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index a924a4c..292161b 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -176,6 +176,8 @@ public class Config private static boolean loadYaml = true; private static boolean outboundBindAny = false; + public volatile int tombstone_debug_threshold = 10000; + public static boolean getOutboundBindAny() { return outboundBindAny; http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/config/DatabaseDescriptor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 22de2d6..2c999ef 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -930,6 +930,20 @@ public class DatabaseDescriptor } /** + * How many tombstones need to be scanned before we log a + * debug message + */ + public static int getTombstoneDebugThreshold() + { + return conf.tombstone_debug_threshold; + } + + public static void setTombstoneDebugThreshold(int tombstoneDebugThreshold) + { + conf.tombstone_debug_threshold = tombstoneDebugThreshold; + } + + /** * size of commitlog segments to allocate */ public static int getCommitLogSegmentSize() http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index c646461..a41c157 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -49,10 +49,7 @@ import org.apache.cassandra.db.compaction.AbstractCompactionStrategy; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.LeveledCompactionStrategy; import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.filter.ExtendedFilter; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.filter.QueryPath; +import org.apache.cassandra.db.filter.*; import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.db.marshal.AbstractType; @@ -1221,6 +1218,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean result = cf.isSuper() ? removeDeleted(cf, gcBefore) : removeDeletedCF(cf, gcBefore); } + + if (filter.filter instanceof SliceQueryFilter) + { + // Log the number of tombstones scanned on single key queries + metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastIgnored()); + metric.liveScannedHistogram.update(((SliceQueryFilter) filter.filter).lastLive()); + } } finally { @@ -1907,6 +1911,22 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return getMinimumCompactionThreshold() <= 0 || getMaximumCompactionThreshold() <= 0; } + public long getTombstonesPerLastRead() + { + return metric.tombstoneScannedHistogram.count(); + } + + public float getPercentageTombstonesPerLastRead() + { + long total = metric.tombstoneScannedHistogram.count() + metric.liveScannedHistogram.count(); + return (metric.tombstoneScannedHistogram.count() / total); + } + + public long getLiveCellsPerLastRead() + { + return metric.liveScannedHistogram.count(); + } + // End JMX get/set. public long estimateKeys() http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java index 554c204..f937032 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java @@ -286,6 +286,15 @@ public interface ColumnFamilyStoreMBean */ public void disableAutoCompaction(); + /** Number of tombstoned cells retreived during the last slicequery */ + public long getTombstonesPerLastRead(); + + /** Percentage of tombstoned cells retreived during the last slicequery */ + public float getPercentageTombstonesPerLastRead(); + + /** Number of live cells retreived during the last slicequery */ + public long getLiveCellsPerLastRead(); + public long estimateKeys(); /** http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java index 39d1ca1..7e1ec6b 100644 --- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java +++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java @@ -17,7 +17,9 @@ */ package org.apache.cassandra.db.filter; -import java.io.*; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; @@ -26,9 +28,10 @@ import com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; import org.apache.cassandra.db.columniterator.ISSTableColumnIterator; +import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; import org.apache.cassandra.db.columniterator.SSTableSliceIterator; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.CompositeType; @@ -172,6 +175,8 @@ public class SliceQueryFilter implements IDiskAtomFilter } Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored()); + if (columnCounter.ignored() > DatabaseDescriptor.getTombstoneDebugThreshold()) + logger.debug("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored()); } public int getLiveCount(ColumnFamily cf) @@ -249,6 +254,16 @@ public class SliceQueryFilter implements IDiskAtomFilter return columnCounter == null ? 0 : columnCounter.live(); } + public int lastIgnored() + { + return columnCounter == null ? 0 : columnCounter.ignored(); + } + + public int lastLive() + { + return columnCounter == null ? 0 : columnCounter.live(); + } + @Override public String toString() { http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java index d111667..9bd90a9 100644 --- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java +++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java @@ -79,6 +79,10 @@ public class ColumnFamilyMetrics public final Gauge<Long> bloomFilterDiskSpaceUsed; /** Key cache hit rate for this CF */ public final Gauge<Double> keyCacheHitRate; + /** Tombstones scanned in queries on this CF */ + public final Histogram tombstoneScannedHistogram; + /** Live cells scanned in queries on this CF */ + public final Histogram liveScannedHistogram; private final MetricNameFactory factory; @@ -295,6 +299,8 @@ public class ColumnFamilyMetrics return Math.max(requests, 1); // to avoid NaN. } }); + tombstoneScannedHistogram = Metrics.newHistogram(factory.createMetricName("TombstoneScannedHistogram")); + liveScannedHistogram = Metrics.newHistogram(factory.createMetricName("LiveScannedHistogram")); } public void updateSSTableIterated(int count) @@ -331,6 +337,8 @@ public class ColumnFamilyMetrics Metrics.defaultRegistry().removeMetric(factory.createMetricName("RecentBloomFilterFalseRatio")); Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed")); Metrics.defaultRegistry().removeMetric(factory.createMetricName("KeyCacheHitRate")); + Metrics.defaultRegistry().removeMetric(factory.createMetricName("TombstoneScannedHistogram")); + Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram")); } class ColumnFamilyMetricNameFactory implements MetricNameFactory http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/service/StorageService.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 7967dee..50719fd 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -3934,4 +3934,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { return DatabaseDescriptor.getPartitionerName(); } + + public int getTombstoneDebugThreshold() + { + return DatabaseDescriptor.getTombstoneDebugThreshold(); + } + + public void setTombstoneDebugThreshold(int tombstoneDebugThreshold) + { + DatabaseDescriptor.setTombstoneDebugThreshold(tombstoneDebugThreshold); + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/service/StorageServiceMBean.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 4fbed9c..63add12 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -476,4 +476,9 @@ public interface StorageServiceMBean extends NotificationEmitter public String getClusterName(); /** Returns the cluster partitioner */ public String getPartitionerName(); + + /** Returns the threshold for returning debugging queries with many tombstones */ + public int getTombstoneDebugThreshold(); + /** Sets the threshold for returning debugging queries with many tombstones */ + public void setTombstoneDebugThreshold(int tombstoneDebugThreshold); }
