memtable_cleanup_threshold and memtable_flush_writers defaults patch by jbellis; reviewed by bes for CASSANDRA-7551
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3adf5948 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3adf5948 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3adf5948 Branch: refs/heads/cassandra-2.1.0 Commit: 3adf59486a4e55f409fcc62a4c942a070b98dd8f Parents: bd0eafd Author: Jonathan Ellis <jbel...@apache.org> Authored: Tue Jul 15 15:14:50 2014 -0500 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Tue Jul 15 15:14:56 2014 -0500 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ conf/cassandra.yaml | 23 +++++++++++++++----- .../org/apache/cassandra/config/Config.java | 6 ++--- .../cassandra/config/DatabaseDescriptor.java | 6 +++++ 4 files changed, 28 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/3adf5948/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index e6ded2a..b20c465 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,6 @@ 2.1.0-final + * Updated memtable_cleanup_threshold and memtable_flush_writers defaults + (CASSANDRA-7551) * (Windows) fix startup when WMI memory query fails (CASSANDRA-7505) * Anti-compaction proceeds if any part of the repair failed (CASANDRA-7521) * Add missing table name to DROP INDEX responses and notifications (CASSANDRA-7539) http://git-wip-us.apache.org/repos/asf/cassandra/blob/3adf5948/conf/cassandra.yaml ---------------------------------------------------------------------- diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index f1e5576..4ce69fe 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -282,8 +282,13 @@ concurrent_counter_writes: 32 # memtable_total_space_in_mb: 2048 # Ratio of occupied non-flushing memtable size to total permitted size -# that will trigger a flush of the largest memtable. -memtable_cleanup_threshold: 0.4 +# that will trigger a flush of the largest memtable. Lager mct will +# mean larger flushes and hence less compaction, but also less concurrent +# flush activity which can make it difficult to keep your disks fed +# under heavy write load. +# +# memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1) +#memtable_cleanup_threshold: 0.11 # Specify the way Cassandra allocates and manages memtable memory. # Options are: @@ -304,9 +309,14 @@ memtable_allocation_type: heap_buffers # This sets the amount of memtable flush writer threads. These will # be blocked by disk io, and each one will hold a memtable in memory -# while blocked. If your flush directory is backed by SSD, you may -# want to increase this. -memtable_flush_writers: 2 +# while blocked. +# +# memtable_flush_writers defaults to the smaller of (number of disks, +# number of cores), with a minimum of 2 and a maximum of 8. +# +# If your data directories are backed by SSD, you should increase this +# to the number of cores. +#memtable_flush_writers: 8 # A fixed memory pool size in MB for for SSTable index summaries. If left # empty, this will default to 5% of the heap size. If the memory usage of @@ -503,6 +513,9 @@ batch_size_warn_threshold_in_kb: 5 # # concurrent_compactors defaults to the smaller of (number of disks, # number of cores), with a minimum of 2 and a maximum of 8. +# +# If your data directories are backed by SSD, you should increase this +# to the number of cores. #concurrent_compactors: 1 # Throttles compaction to the given total throughput across the entire http://git-wip-us.apache.org/repos/asf/cassandra/blob/3adf5948/src/java/org/apache/cassandra/config/Config.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 4d4a95b..2438e09 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -90,12 +90,10 @@ public class Config @Deprecated public Integer concurrent_replicates = null; - // we don't want a lot of contention, but we also don't want to starve all other tables - // if a big one flushes. OS buffering should be able to minimize contention with 2 threads. - public int memtable_flush_writers = 2; + public Integer memtable_flush_writers = null; public Integer memtable_heap_space_in_mb; public Integer memtable_offheap_space_in_mb; - public float memtable_cleanup_threshold = 0.4f; + public Double memtable_cleanup_threshold = null; public Integer storage_port = 7000; public Integer ssl_storage_port = 7001; http://git-wip-us.apache.org/repos/asf/cassandra/blob/3adf5948/src/java/org/apache/cassandra/config/DatabaseDescriptor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 5a2f26f..6730cb5 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -483,6 +483,12 @@ public class DatabaseDescriptor if (conf.commitlog_directory.equals(conf.saved_caches_directory)) throw new ConfigurationException("saved_caches_directory must not be the same as the commitlog_directory"); + if (conf.memtable_flush_writers == null) + conf.memtable_flush_writers = Math.min(8, Math.max(2, Math.min(FBUtilities.getAvailableProcessors(), conf.data_file_directories.length))); + + if (conf.memtable_cleanup_threshold == null) + conf.memtable_cleanup_threshold = 1.0 / (1 + conf.memtable_flush_writers); + if (conf.concurrent_compactors == null) conf.concurrent_compactors = Math.min(8, Math.max(2, Math.min(FBUtilities.getAvailableProcessors(), conf.data_file_directories.length)));