Repository: cassandra Updated Branches: refs/heads/trunk d32ae4a60 -> 52df6a58d
Added compaction metrics to track failures when running low on disk space patch by John Haddad; reviewed by jasobrown for CASSANDRA-13015 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/52df6a58 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/52df6a58 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/52df6a58 Branch: refs/heads/trunk Commit: 52df6a58d18200c11d2e9d88fc9276b5ef1ee06a Parents: d32ae4a Author: Jon Haddad <[email protected]> Authored: Tue Jan 10 22:32:34 2017 -0800 Committer: Jason Brown <[email protected]> Committed: Thu Jan 19 04:29:44 2017 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/db/compaction/CompactionManager.java | 16 ++++++++++++++++ .../cassandra/db/compaction/CompactionTask.java | 15 +++++++++++++-- .../apache/cassandra/metrics/CompactionMetrics.java | 15 +++++++++++++++ 4 files changed, 45 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/52df6a58/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index a67cf43..8f68618 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0 + * Improved compactions metrics (CASSANDRA-13015) * Speed-up start-up sequence by avoiding un-needed flushes (CASSANDRA-13031) * Use Caffeine (W-TinyLFU) for on-heap caches (CASSANDRA-10855) * Thrift removal (CASSANDRA-11115) http://git-wip-us.apache.org/repos/asf/cassandra/blob/52df6a58/src/java/org/apache/cassandra/db/compaction/CompactionManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index fb11bad..9c74f62 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -1838,6 +1838,22 @@ public class CompactionManager implements CompactionManagerMBean void finishCompaction(CompactionInfo.Holder ci); } + public void incrementAborted() + { + metrics.compactionsAborted.inc(); + } + + public void incrementCompactionsReduced() + { + metrics.compactionsReduced.inc(); + } + + public void incrementSstablesDropppedFromCompactions(long num) + { + metrics.sstablesDropppedFromCompactions.inc(num); + } + + public List<Map<String, String>> getCompactions() { List<Holder> compactionHolders = CompactionMetrics.getCompactions(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/52df6a58/src/java/org/apache/cassandra/db/compaction/CompactionTask.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java index b2e9b8c..62efa3d 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@ -27,7 +27,6 @@ import java.util.concurrent.TimeUnit; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; -import com.google.common.primitives.Ints; import com.google.common.util.concurrent.RateLimiter; import org.apache.cassandra.db.Directories; @@ -333,23 +332,35 @@ public class CompactionTask extends AbstractCompactionTask } CompactionStrategyManager strategy = cfs.getCompactionStrategyManager(); - + int sstablesRemoved = 0; while(true) { long expectedWriteSize = cfs.getExpectedCompactedFileSize(transaction.originals(), compactionType); long estimatedSSTables = Math.max(1, expectedWriteSize / strategy.getMaxSSTableBytes()); if(cfs.getDirectories().hasAvailableDiskSpace(estimatedSSTables, expectedWriteSize)) + { + // we're ok now on space so now we track the failures, if any + if(sstablesRemoved > 0) + { + CompactionManager.instance.incrementCompactionsReduced(); + CompactionManager.instance.incrementSstablesDropppedFromCompactions(sstablesRemoved); + } + break; + } if (!reduceScopeForLimitedSpace(expectedWriteSize)) { // we end up here if we can't take any more sstables out of the compaction. // usually means we've run out of disk space String msg = String.format("Not enough space for compaction, estimated sstables = %d, expected write size = %d", estimatedSSTables, expectedWriteSize); + logger.warn(msg); + CompactionManager.instance.incrementAborted(); throw new RuntimeException(msg); } + sstablesRemoved++; logger.warn("Not enough space for compaction, {}MB estimated. Reducing scope.", (float) expectedWriteSize / 1024 / 1024); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/52df6a58/src/java/org/apache/cassandra/metrics/CompactionMetrics.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java index 9aef0f8..96ca38c 100644 --- a/src/java/org/apache/cassandra/metrics/CompactionMetrics.java +++ b/src/java/org/apache/cassandra/metrics/CompactionMetrics.java @@ -55,6 +55,16 @@ public class CompactionMetrics implements CompactionManager.CompactionExecutorSt /** Total number of bytes compacted since server [re]start */ public final Counter bytesCompacted; + + /** Total number of compactions that have had sstables drop out of them */ + public final Counter compactionsReduced; + + /** Total number of sstables that have been dropped out */ + public final Counter sstablesDropppedFromCompactions; + + /** Total number of compactions which have outright failed due to lack of disk space */ + public final Counter compactionsAborted; + public CompactionMetrics(final ThreadPoolExecutor... collectors) { pendingTasks = Metrics.register(factory.createMetricName("PendingTasks"), new Gauge<Integer>() @@ -137,6 +147,11 @@ public class CompactionMetrics implements CompactionManager.CompactionExecutorSt }); totalCompactionsCompleted = Metrics.meter(factory.createMetricName("TotalCompactionsCompleted")); bytesCompacted = Metrics.counter(factory.createMetricName("BytesCompacted")); + + // compaction failure metrics + compactionsReduced = Metrics.counter(factory.createMetricName("CompactionsReduced")); + sstablesDropppedFromCompactions = Metrics.counter(factory.createMetricName("SSTablesDroppedFromCompaction")); + compactionsAborted = Metrics.counter(factory.createMetricName("CompactionsAborted")); } public void beginCompaction(CompactionInfo.Holder ci)
