Repository: cassandra Updated Branches: refs/heads/trunk 860790045 -> 1e66e97ca
Add configurable warning threshold for GC duration Patch by jmckenzie and achowdhery for CASSANDRA-8907 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/488db6f5 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/488db6f5 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/488db6f5 Branch: refs/heads/trunk Commit: 488db6f50662e5e63c382c793d2c7ad69b8c74fc Parents: e9f3604 Author: Joshua McKenzie <jmcken...@apache.org> Authored: Fri Sep 11 12:54:30 2015 -0400 Committer: Joshua McKenzie <jmcken...@apache.org> Committed: Fri Sep 11 12:54:30 2015 -0400 ---------------------------------------------------------------------- CHANGES.txt | 1 + conf/cassandra.yaml | 5 +++++ src/java/org/apache/cassandra/config/Config.java | 2 ++ .../apache/cassandra/config/DatabaseDescriptor.java | 11 +++++++++++ .../org/apache/cassandra/service/GCInspector.java | 16 +++++++++++----- 5 files changed, 30 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/488db6f5/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 2b96949..4e0df42 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 2.1.10 + * Added configurable warning threshold for GC duration (CASSANDRA-8907) * (cqlsh) Make cqlsh PEP8 compliant (CASSANDRA-10066) * (cqlsh) Fix error when starting cqlsh with --debug (CASSANDRA-10282) * Scrub, Cleanup and Upgrade do not unmark compacting until all operations http://git-wip-us.apache.org/repos/asf/cassandra/blob/488db6f5/conf/cassandra.yaml ---------------------------------------------------------------------- diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 4e0111d..7e266d1 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -802,3 +802,8 @@ internode_compression: all # reducing overhead from the TCP protocol itself, at the cost of increasing # latency if you block for cross-datacenter responses. inter_dc_tcp_nodelay: false + +# GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level +# Adjust the threshold based on your application throughput requirement +# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level +# gc_warn_threshold_in_ms: 1000 http://git-wip-us.apache.org/repos/asf/cassandra/blob/488db6f5/src/java/org/apache/cassandra/config/Config.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index d960463..911dd73 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -228,6 +228,8 @@ public class Config public volatile Long index_summary_capacity_in_mb; public volatile int index_summary_resize_interval_in_minutes = 60; + public int gc_warn_threshold_in_ms = 0; + private static final CsvPreference STANDARD_SURROUNDING_SPACES_NEED_QUOTES = new CsvPreference.Builder(CsvPreference.STANDARD_PREFERENCE) .surroundingSpacesNeedQuotes(true).build(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/488db6f5/src/java/org/apache/cassandra/config/DatabaseDescriptor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index bbecc6b..3a6a8fd 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -389,6 +389,11 @@ public class DatabaseDescriptor } paritionerName = partitioner.getClass().getCanonicalName(); + if (conf.gc_warn_threshold_in_ms < 0) + { + throw new ConfigurationException("gc_warn_threshold_in_ms must be a positive integer"); + } + if (conf.max_hint_window_in_ms == null) { throw new ConfigurationException("max_hint_window_in_ms cannot be set to null"); @@ -1707,4 +1712,10 @@ public class DatabaseDescriptor { return conf.otc_coalescing_window_us; } + + public static long getGCWarnThreshold() + { + return conf.gc_warn_threshold_in_ms; + } + } http://git-wip-us.apache.org/repos/asf/cassandra/blob/488db6f5/src/java/org/apache/cassandra/service/GCInspector.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/GCInspector.java b/src/java/org/apache/cassandra/service/GCInspector.java index df0527f..cf17a34 100644 --- a/src/java/org/apache/cassandra/service/GCInspector.java +++ b/src/java/org/apache/cassandra/service/GCInspector.java @@ -25,17 +25,20 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; + import javax.management.MBeanServer; import javax.management.Notification; import javax.management.NotificationListener; import javax.management.ObjectName; import javax.management.openmbean.CompositeData; +import com.sun.management.GarbageCollectionNotificationInfo; +import com.sun.management.GcInfo; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.sun.management.GarbageCollectionNotificationInfo; -import com.sun.management.GcInfo; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.io.sstable.SSTableDeletingTask; import org.apache.cassandra.utils.StatusLogger; @@ -44,7 +47,8 @@ public class GCInspector implements NotificationListener, GCInspectorMXBean public static final String MBEAN_NAME = "org.apache.cassandra.service:type=GCInspector"; private static final Logger logger = LoggerFactory.getLogger(GCInspector.class); final static long MIN_LOG_DURATION = 200; - final static long MIN_LOG_DURATION_TPSTATS = 1000; + final static long GC_WARN_THRESHOLD_IN_MS = DatabaseDescriptor.getGCWarnThreshold(); + final static long STAT_THRESHOLD = Math.min(GC_WARN_THRESHOLD_IN_MS != 0 ? GC_WARN_THRESHOLD_IN_MS : MIN_LOG_DURATION, MIN_LOG_DURATION); static final class State { @@ -248,12 +252,14 @@ public class GCInspector implements NotificationListener, GCInspectorMXBean } String st = sb.toString(); - if (duration > MIN_LOG_DURATION) + if (GC_WARN_THRESHOLD_IN_MS != 0 && duration > GC_WARN_THRESHOLD_IN_MS) + logger.warn(st); + else if (duration > MIN_LOG_DURATION) logger.info(st); else if (logger.isDebugEnabled()) logger.debug(st); - if (duration > MIN_LOG_DURATION_TPSTATS) + if (duration > STAT_THRESHOLD) StatusLogger.log(); // if we just finished an old gen collection and we're still using a lot of memory, try to reduce the pressure