Author: jbellis
Date: Mon Sep 27 22:10:42 2010
New Revision: 1001929
URL: http://svn.apache.org/viewvc?rev=1001929&view=rev
Log:
sanity checks for compaction thresholds.
patch by jhermes; reviewed by jbellis for CASSANDRA-1527
Modified:
cassandra/trunk/conf/cassandra.yaml
cassandra/trunk/src/java/org/apache/cassandra/avro/CassandraServer.java
cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java
Modified: cassandra/trunk/conf/cassandra.yaml
URL:
http://svn.apache.org/viewvc/cassandra/trunk/conf/cassandra.yaml?rev=1001929&r1=1001928&r2=1001929&view=diff
==============================================================================
--- cassandra/trunk/conf/cassandra.yaml (original)
+++ cassandra/trunk/conf/cassandra.yaml Mon Sep 27 22:10:42 2010
@@ -1,6 +1,9 @@
# Cassandra storage config YAML
+
+#NOTE !!!!!!!! NOTE
# See http://wiki.apache.org/cassandra/StorageConfiguration for
-# explanations of configuration directives.
+# full explanations of configuration directives
+#NOTE !!!!!!!! NOTE
# The name of the cluster. This is mainly used to prevent machines in
# one logical cluster from joining another.
@@ -290,38 +293,48 @@ index_interval: 128
# - replication_factor: Number of replicas of each row
# - column_families: column families associated with this keyspace
#
-# ColumnFamily required parameters:
-# - name: name of the ColumnFamily. Must not contain the character "-".
-# - compare_with: tells Cassandra how to sort the columns for slicing
-# operations. The default is BytesType, which is a straightforward
-# lexical comparison of the bytes in each column. Other options are
-# AsciiType, UTF8Type, LexicalUUIDType, TimeUUIDType, LongType,
-# and IntegerType (a generic variable-length integer type).
-# You can also specify the fully-qualified class name to a class of
-# your choice extending org.apache.cassandra.db.marshal.AbstractType.
-#
-# ColumnFamily optional parameters:
-# - keys_cached: specifies the number of keys per sstable whose
-# locations we keep in memory in "mostly LRU" order. (JUST the key
-# locations, NOT any column values.) Specify a fraction (value less
-# than 1) or an absolute number of keys to cache. Defaults to 200000
-# keys.
-# - rows_cached: specifies the number of rows whose entire contents we
-# cache in memory. Do not use this on ColumnFamilies with large rows,
-# or ColumnFamilies with high write:read ratios. Specify a fraction
-# (value less than 1) or an absolute number of rows to cache.
-# Defaults to 0. (i.e. row caching is off by default)
-# - comment: used to attach additional human-readable information about
-# the column family to its definition.
-# - read_repair_chance: specifies the probability with which read
-# repairs should be invoked on non-quorum reads. must be between 0
-# and 1. defaults to 1.0 (always read repair).
-# - preload_row_cache: If true, will populate row cache on startup.
-# Defaults to false.
-# - gc_grace_seconds: specifies the time to wait before garbage
-# collecting tombstones (deletion markers). defaults to 864000 (10
-# days). See http://wiki.apache.org/cassandra/DistributedDeletes
-#
+# ColumnFamily required parameters:
+# - name: name of the ColumnFamily. Must not contain the character "-".
+# - compare_with: tells Cassandra how to sort the columns for slicing
+# operations. The default is BytesType, which is a straightforward
+# lexical comparison of the bytes in each column. Other options are
+# AsciiType, UTF8Type, LexicalUUIDType, TimeUUIDType, LongType,
+# and IntegerType (a generic variable-length integer type).
+# You can also specify the fully-qualified class name to a class of
+# your choice extending org.apache.cassandra.db.marshal.AbstractType.
+#
+# ColumnFamily optional parameters:
+# - keys_cached: specifies the number of keys per sstable whose
+# locations we keep in memory in "mostly LRU" order. (JUST the key
+# locations, NOT any column values.) Specify a fraction (value less
+# than 1) or an absolute number of keys to cache. Defaults to 200000
+# keys.
+# - rows_cached: specifies the number of rows whose entire contents we
+# cache in memory. Do not use this on ColumnFamilies with large rows,
+# or ColumnFamilies with high write:read ratios. Specify a fraction
+# (value less than 1) or an absolute number of rows to cache.
+# Defaults to 0. (i.e. row caching is off by default)
+# - comment: used to attach additional human-readable information about
+# the column family to its definition.
+# - read_repair_chance: specifies the probability with which read
+# repairs should be invoked on non-quorum reads. must be between 0
+# and 1. defaults to 1.0 (always read repair).
+# - preload_row_cache: If true, will populate row cache on startup.
+# Defaults to false.
+# - gc_grace_seconds: specifies the time to wait before garbage
+# collecting tombstones (deletion markers). defaults to 864000 (10
+# days). See http://wiki.apache.org/cassandra/DistributedDeletes
+# - default_validation_class: specifies a validator class to use for
+# validating all the column values in the CF.
+# - min_compaction_threshold: the minimum number of SSTables needed
+# to start a minor compaction. increasing this will cause minor
+# compactions to start less frequently and be more intensive. setting
+# this to 0 disables minor compactions. defaults to 4.
+# - max_compaction_threshold: the maximum number of SSTables allowed
+# before a minor compaction is forced. decreasing this will cause
+# minor compactions to start more frequently and be less intensive.
+# setting this to 0 disables minor compactions. defaults to 32.
+#
# NOTE: this keyspace definition is for demonstration purposes only.
# Cassandra will not load these definitions during startup. See
# http://wiki.apache.org/cassandra/FAQ#no_keyspaces for an explanation.
Modified:
cassandra/trunk/src/java/org/apache/cassandra/avro/CassandraServer.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/avro/CassandraServer.java?rev=1001929&r1=1001928&r2=1001929&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/avro/CassandraServer.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/avro/CassandraServer.java Mon
Sep 27 22:10:42 2010
@@ -840,6 +840,8 @@ public class CassandraServer implements
String validate = cf_def.default_validation_class == null ?
D_CF_COMPTYPE : cf_def.default_validation_class.toString();
String subCompare = cf_def.subcomparator_type == null ?
D_CF_SUBCOMPTYPE : cf_def.subcomparator_type.toString();
+ CFMetaData.validateMinMaxCompactionThresholds(cf_def);
+
return new CFMetaData(cf_def.keyspace.toString(),
cf_def.name.toString(),
ColumnFamilyType.create(cfType),
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java?rev=1001929&r1=1001928&r2=1001929&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java Mon
Sep 27 22:10:42 2010
@@ -480,6 +480,8 @@ public final class CFMetaData
}
else if (subcolumnComparator !=
DatabaseDescriptor.getComparator(cf_def.subcomparator_type.toString()))
throw new ConfigurationException("subcolumncomparators do not
match.");
+
+ validateMinMaxCompactionThresholds(cf_def);
return new CFMetaData(tableName,
cfName,
@@ -523,7 +525,9 @@ public final class CFMetaData
}
else if (subcolumnComparator !=
DatabaseDescriptor.getComparator(cf_def.subcomparator_type))
throw new ConfigurationException("subcolumncomparators do not
match.");
-
+
+ validateMinMaxCompactionThresholds(cf_def);
+
return new CFMetaData(tableName,
cfName,
cfType,
@@ -615,4 +619,64 @@ public final class CFMetaData
def.column_metadata = column_meta;
return def;
}
+
+ public static void
validateMinMaxCompactionThresholds(org.apache.cassandra.thrift.CfDef cf_def)
throws ConfigurationException
+ {
+ if (cf_def.isSetMin_compaction_threshold() &&
cf_def.isSetMax_compaction_threshold())
+ {
+ if ((cf_def.min_compaction_threshold >
cf_def.max_compaction_threshold) &&
+ cf_def.max_compaction_threshold != 0)
+ {
+ throw new ConfigurationException("min_compaction_threshold
cannot be greater than max_compaction_threshold");
+ }
+ }
+ else if (cf_def.isSetMin_compaction_threshold())
+ {
+ if (cf_def.min_compaction_threshold >
DEFAULT_MAX_COMPACTION_THRESHOLD)
+ {
+ throw new ConfigurationException("min_compaction_threshold
cannot be greather than max_compaction_threshold (default " +
+
DEFAULT_MAX_COMPACTION_THRESHOLD + ")");
+ }
+ }
+ else if (cf_def.isSetMax_compaction_threshold())
+ {
+ if (cf_def.max_compaction_threshold <
DEFAULT_MIN_COMPACTION_THRESHOLD && cf_def.max_compaction_threshold != 0) {
+ throw new ConfigurationException("max_compaction_threshold
cannot be less than min_compaction_threshold");
+ }
+ }
+ else
+ {
+ //Defaults are valid.
+ }
+ }
+
+ public static void
validateMinMaxCompactionThresholds(org.apache.cassandra.avro.CfDef cf_def)
throws ConfigurationException
+ {
+ if (cf_def.min_compaction_threshold != null &&
cf_def.max_compaction_threshold != null)
+ {
+ if ((cf_def.min_compaction_threshold >
cf_def.max_compaction_threshold) &&
+ cf_def.max_compaction_threshold != 0)
+ {
+ throw new ConfigurationException("min_compaction_threshold
cannot be greater than max_compaction_threshold");
+ }
+ }
+ else if (cf_def.min_compaction_threshold != null)
+ {
+ if (cf_def.min_compaction_threshold >
DEFAULT_MAX_COMPACTION_THRESHOLD)
+ {
+ throw new ConfigurationException("min_compaction_threshold
cannot be greather than max_compaction_threshold (default " +
+
DEFAULT_MAX_COMPACTION_THRESHOLD + ")");
+ }
+ }
+ else if (cf_def.max_compaction_threshold != null)
+ {
+ if (cf_def.max_compaction_threshold <
DEFAULT_MIN_COMPACTION_THRESHOLD && cf_def.max_compaction_threshold != 0) {
+ throw new ConfigurationException("max_compaction_threshold
cannot be less than min_compaction_threshold");
+ }
+ }
+ else
+ {
+ //Defaults are valid.
+ }
+ }
}
Modified:
cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=1001929&r1=1001928&r2=1001929&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
Mon Sep 27 22:10:42 2010
@@ -548,7 +548,16 @@ public class DatabaseDescriptor
{
throw new ConfigurationException("read_repair_chance must
be between 0.0 and 1.0");
}
-
+
+ if (cf.min_compaction_threshold < 0 ||
cf.max_compaction_threshold < 0)
+ {
+ throw new
ConfigurationException("min/max_compaction_thresholds must be non-negative
integers.");
+ }
+ if ((cf.min_compaction_threshold >
cf.max_compaction_threshold) && cf.max_compaction_threshold != 0)
+ {
+ throw new ConfigurationException("min_compaction_threshold
must be smaller than max_compaction_threshold, or either must be 0 (disabled)");
+ }
+
Map<byte[], ColumnDefinition> metadata = new TreeMap<byte[],
ColumnDefinition>(FBUtilities.byteArrayComparator);
for (RawColumnDefinition rcd : cf.column_metadata)
{
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1001929&r1=1001928&r2=1001929&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Mon
Sep 27 22:10:42 2010
@@ -1637,6 +1637,10 @@ public class ColumnFamilyStore implement
public void setMinimumCompactionThreshold(int minCompactionThreshold)
{
+ //TODO: If someone complains this is too rude, make it more friendly.
+ if ((minCompactionThreshold > this.maxCompactionThreshold) &&
this.maxCompactionThreshold != 0) {
+ throw new RuntimeException("The min_compaction_threshold cannot be
larger than the max.");
+ }
this.minCompactionThreshold = minCompactionThreshold;
}
@@ -1647,6 +1651,10 @@ public class ColumnFamilyStore implement
public void setMaximumCompactionThreshold(int maxCompactionThreshold)
{
+ //TODO: If someone complains this is too rude, make it more friendly.
+ if (maxCompactionThreshold < this.minCompactionThreshold) {
+ throw new RuntimeException("The max_compaction_threshold cannot be
smaller than the min.");
+ }
this.maxCompactionThreshold = maxCompactionThreshold;
}
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java?rev=1001929&r1=1001928&r2=1001929&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
Mon Sep 27 22:10:42 2010
@@ -185,4 +185,9 @@ public interface ColumnFamilyStoreMBean
* Sets the maximum number of sstables in queue before compaction kicks off
*/
public void setMaximumCompactionThreshold(int threshold);
+
+ /**
+ * Disable automatic compaction.
+ */
+ public void disableAutoCompaction();
}
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java?rev=1001929&r1=1001928&r2=1001929&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java Mon
Sep 27 22:10:42 2010
@@ -87,7 +87,7 @@ public class CompactionManager implement
Integer minThreshold = cfs.getMinimumCompactionThreshold();
Integer maxThreshold = cfs.getMaximumCompactionThreshold();
- if (minThreshold <= 0 || maxThreshold <= 0)
+ if (minThreshold == 0 || maxThreshold == 0)
{
logger.debug("Compaction is currently disabled.");
return 0;
@@ -114,18 +114,25 @@ public class CompactionManager implement
private void updateEstimateFor(ColumnFamilyStore cfs,
Set<List<SSTableReader>> buckets)
{
- Integer minct = cfs.getMinimumCompactionThreshold();
- Integer maxct = cfs.getMaximumCompactionThreshold();
+ Integer minThreshold = cfs.getMinimumCompactionThreshold();
+ Integer maxThreshold = cfs.getMaximumCompactionThreshold();
- int n = 0;
- for (List<SSTableReader> sstables : buckets)
+ if (minThreshold > 0 && maxThreshold > 0)
{
- if (sstables.size() >= minct)
+ int n = 0;
+ for (List<SSTableReader> sstables : buckets)
{
- n += 1 + sstables.size() / (maxct - minct);
+ if (sstables.size() >= minThreshold)
+ {
+ n += Math.ceil((double)sstables.size() / maxThreshold);
+ }
}
+ estimatedCompactions.put(cfs, n);
+ }
+ else
+ {
+ logger.debug("Compaction is currently disabled.");
}
- estimatedCompactions.put(cfs, n);
}
public Future<Object> submitCleanup(final ColumnFamilyStore cfStore)
Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java?rev=1001929&r1=1001928&r2=1001929&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java Mon Sep 27
22:10:42 2010
@@ -539,6 +539,8 @@ public class DefsTest extends CleanupHel
cf_def.setRow_cache_size(43.3);
cf_def.setColumn_metadata(new ArrayList<ColumnDef>());
cf_def.setDefault_validation_class("BytesType");
+ cf_def.setMin_compaction_threshold(5);
+ cf_def.setMax_compaction_threshold(31);
// test valid operations.
cf_def.setComment("Modified comment");
@@ -575,7 +577,17 @@ public class DefsTest extends CleanupHel
updateCfm = cf.apply(cf_def);
new UpdateColumnFamily(cf, updateCfm).apply();
cf = updateCfm;
-
+
+ cf_def.setMin_compaction_threshold(3);
+ updateCfm = cf.apply(cf_def);
+ new UpdateColumnFamily(cf, updateCfm).apply();
+ cf = updateCfm;
+
+ cf_def.setMax_compaction_threshold(33);
+ updateCfm = cf.apply(cf_def);
+ new UpdateColumnFamily(cf, updateCfm).apply();
+ cf = updateCfm;
+
// can't test changing the reconciler because there is only one impl.
// check the cumulative affect.
@@ -646,6 +658,28 @@ public class DefsTest extends CleanupHel
{
cf_def.setComparator_type(UTF8Type.class.getSimpleName());
}
+
+ try
+ {
+ cf_def.setMin_compaction_threshold(34);
+ updateCfm = cf.apply(cf_def);
+ throw new AssertionError("Should have blown up when min > max.");
+ }
+ catch (ConfigurationException expected)
+ {
+ cf_def.setMin_compaction_threshold(3);
+ }
+
+ try
+ {
+ cf_def.setMax_compaction_threshold(2);
+ updateCfm = cf.apply(cf_def);
+ throw new AssertionError("Should have blown up when max > min.");
+ }
+ catch (ConfigurationException expected)
+ {
+ cf_def.setMax_compaction_threshold(33);
+ }
}
private CFMetaData addTestCF(String ks, String cf, String comment)
