reduce log spam from invalid counter shards patch by slebresne and jbellis for CASSANDRA-5026
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5b511b62 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5b511b62 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5b511b62 Branch: refs/heads/cassandra-1.2.0 Commit: 5b511b62136d3bcb230a282fa433a122d061ee22 Parents: 5e46297 Author: Jonathan Ellis <jbel...@apache.org> Authored: Sat Dec 8 09:18:10 2012 -0600 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Sat Dec 8 09:18:10 2012 -0600 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/db/compaction/CompactionManager.java | 19 +++++++++++++- .../cassandra/db/context/CounterContext.java | 12 ++++----- 3 files changed, 23 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/5b511b62/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 874dbd4..15d63f0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 1.1.8 + * reduce log spam from invalid counter shards (CASSANDRA-5026) * Improve schema propagation performance (CASSANDRA-5025) * Fall back to old describe_splits if d_s_ex is not available (CASSANDRA-4803) * Improve error reporting when streaming ranges fail (CASSANDRA-5009) http://git-wip-us.apache.org/repos/asf/cassandra/blob/5b511b62/src/java/org/apache/cassandra/db/compaction/CompactionManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index edfea0a..ed1fb7e 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -19,7 +19,6 @@ package org.apache.cassandra.db.compaction; import java.io.File; -import java.io.IOError; import java.io.IOException; import java.lang.management.ManagementFactory; import java.nio.ByteBuffer; @@ -47,7 +46,6 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.*; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.service.AntiEntropyService; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.service.StorageService; @@ -77,6 +75,16 @@ public class CompactionManager implements CompactionManagerMBean public static final int NO_GC = Integer.MIN_VALUE; public static final int GC_ALL = Integer.MAX_VALUE; + // A thread local that tells us if the current thread is owned by the compaction manager. Used + // by CounterContext to figure out if it should log a warning for invalid counter shards. + public static final ThreadLocal<Boolean> isCompactionManager = new ThreadLocal<Boolean>() { + @Override + protected Boolean initialValue() + { + return false; + } + }; + /** * compactionLock has two purposes: * - "Special" compactions will acquire writelock instead of readlock to make sure that all @@ -931,6 +939,13 @@ public class CompactionManager implements CompactionManagerMBean return totalCompactionsCompleted; } + protected void beforeExecute(Thread t, Runnable r) + { + // can't set this in Thread factory, so we do it redundantly here + isCompactionManager.set(true); + super.beforeExecute(t, r); + } + // modified from DebuggableThreadPoolExecutor so that CompactionInterruptedExceptions are not logged @Override public void afterExecute(Runnable r, Throwable t) http://git-wip-us.apache.org/repos/asf/cassandra/blob/5b511b62/src/java/org/apache/cassandra/db/context/CounterContext.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/context/CounterContext.java b/src/java/org/apache/cassandra/db/context/CounterContext.java index d466caa..bbab603 100644 --- a/src/java/org/apache/cassandra/db/context/CounterContext.java +++ b/src/java/org/apache/cassandra/db/context/CounterContext.java @@ -27,6 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.DBConstants; +import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.marshal.MarshalException; import org.apache.cassandra.utils.Allocator; import org.apache.cassandra.utils.ByteBufferUtil; @@ -376,14 +377,11 @@ public class CounterContext implements IContext long leftCount = leftState.getCount(); long rightCount = rightState.getCount(); - if (leftCount != rightCount) + if (leftCount != rightCount && CompactionManager.isCompactionManager.get()) { - logger.error("invalid counter shard detected; ({}, {}, {}) and ({}, {}, {}) differ only in " - + "count; will pick highest to self-heal; this indicates a bug or corruption generated a bad counter shard", - new Object[] { - leftState.getNodeId(), leftClock, leftCount, - rightState.getNodeId(), rightClock, rightCount, - }); + logger.warn("invalid counter shard detected; ({}, {}, {}) and ({}, {}, {}) differ only in " + + "count; will pick highest to self-heal on compaction", + new Object[] { leftState.getNodeId(), leftClock, leftCount, rightState.getNodeId(), rightClock, rightCount, }); } if (leftCount > rightCount)