Author: jbellis
Date: Sat Sep 25 01:31:12 2010
New Revision: 1001119
URL: http://svn.apache.org/viewvc?rev=1001119&view=rev
Log:
make IndexInterval configurable. patch by jhermes; reviewed by jbellis for
CASSANDRA-1488
Modified:
cassandra/trunk/CHANGES.txt
cassandra/trunk/conf/cassandra.yaml
cassandra/trunk/src/java/org/apache/cassandra/config/Config.java
cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
cassandra/trunk/test/unit/org/apache/cassandra/db/CompactionsTest.java
cassandra/trunk/test/unit/org/apache/cassandra/io/LazilyCompactedRowTest.java
Modified: cassandra/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1001119&r1=1001118&r2=1001119&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Sat Sep 25 01:31:12 2010
@@ -88,6 +88,7 @@
* RPM packaging spec file (CASSANDRA-786)
* weighted request scheduler (CASSANDRA-1485)
* treat expired columns as deleted (CASSANDRA-1539)
+ * make IndexInterval configurable (CASSANDRA-1488)
0.7-beta1
Modified: cassandra/trunk/conf/cassandra.yaml
URL:
http://svn.apache.org/viewvc/cassandra/trunk/conf/cassandra.yaml?rev=1001119&r1=1001118&r2=1001119&view=diff
==============================================================================
--- cassandra/trunk/conf/cassandra.yaml (original)
+++ cassandra/trunk/conf/cassandra.yaml Sat Sep 25 01:31:12 2010
@@ -242,6 +242,11 @@ request_scheduler: org.apache.cassandra.
# the request scheduling. Currently the only valid option is keyspace.
# request_scheduler_id: keyspace
+# The Index Interval determines how large the sampling of row keys
+# is for a given SSTable. The larger the sampling, the more effective
+# the index is at the cost of space.
+index_interval: 128
+
# A ColumnFamily is the Cassandra concept closest to a relational table.
#
# Keyspaces are separate groups of ColumnFamilies. Except in very
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/Config.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/Config.java?rev=1001119&r1=1001118&r2=1001119&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/Config.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/Config.java Sat Sep 25
01:31:12 2010
@@ -100,6 +100,8 @@ public class Config
public RequestSchedulerId request_scheduler_id;
public RequestSchedulerOptions request_scheduler_options;
+ public Integer index_interval = 128;
+
public List<RawKeyspace> keyspaces;
public static enum CommitLogSync {
Modified:
cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=1001119&r1=1001118&r2=1001119&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
Sat Sep 25 01:31:12 2010
@@ -1102,4 +1102,9 @@ public class DatabaseDescriptor
{
return getCFMetaData(desc.ksname, desc.cfname);
}
+
+ public static Integer getIndexInterval()
+ {
+ return conf.index_interval;
+ }
}
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java?rev=1001119&r1=1001118&r2=1001119&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java Sat
Sep 25 01:31:12 2010
@@ -250,7 +250,7 @@ public class CompactionManager implement
long totalkeysWritten = 0;
// TODO the int cast here is potentially buggy
- int expectedBloomFilterSize = Math.max(SSTableReader.indexInterval(),
(int)SSTableReader.getApproximateKeyCount(sstables));
+ int expectedBloomFilterSize =
Math.max(DatabaseDescriptor.getIndexInterval(),
(int)SSTableReader.getApproximateKeyCount(sstables));
if (logger.isDebugEnabled())
logger.debug("Expected bloom filter size : " +
expectedBloomFilterSize);
@@ -338,7 +338,7 @@ public class CompactionManager implement
long startTime = System.currentTimeMillis();
long totalkeysWritten = 0;
- int expectedBloomFilterSize = Math.max(SSTableReader.indexInterval(),
(int)(SSTableReader.getApproximateKeyCount(sstables) / 2));
+ int expectedBloomFilterSize =
Math.max(DatabaseDescriptor.getIndexInterval(),
(int)(SSTableReader.getApproximateKeyCount(sstables) / 2));
if (logger.isDebugEnabled())
logger.debug("Expected bloom filter size : " +
expectedBloomFilterSize);
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java?rev=1001119&r1=1001118&r2=1001119&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
Sat Sep 25 01:31:12 2010
@@ -32,16 +32,13 @@ import org.apache.cassandra.db.Decorated
public class IndexSummary
{
- /** Every 128th index entry is loaded into memory so we know where to
start looking for the actual key w/o seeking */
- public static final int INDEX_INTERVAL = 128;/* Required extension for
temporary files created during compactions. */
-
private ArrayList<KeyPosition> indexPositions;
private int keysWritten = 0;
private long lastIndexPosition;
public void maybeAddEntry(DecoratedKey decoratedKey, long indexPosition)
{
- if (keysWritten++ % INDEX_INTERVAL == 0)
+ if (keysWritten++ % DatabaseDescriptor.getIndexInterval() == 0)
{
if (indexPositions == null)
{
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1001119&r1=1001118&r2=1001119&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Sat Sep 25 01:31:12 2010
@@ -57,7 +57,7 @@ public class SSTableReader extends SSTab
private static final Logger logger =
LoggerFactory.getLogger(SSTableReader.class);
// guesstimated size of INDEX_INTERVAL index entries
- private static final int INDEX_FILE_BUFFER_BYTES = 16 *
IndexSummary.INDEX_INTERVAL;
+ private static final int INDEX_FILE_BUFFER_BYTES = 16 *
DatabaseDescriptor.getIndexInterval();
// `finalizers` is required to keep the PhantomReferences alive after the
enclosing SSTR is itself
// unreferenced. otherwise they will never get enqueued.
@@ -121,11 +121,6 @@ public class SSTableReader extends SSTab
private volatile SSTableDeletingReference phantomReference;
- public static int indexInterval()
- {
- return IndexSummary.INDEX_INTERVAL;
- }
-
public static long getApproximateKeyCount(Iterable<SSTableReader> sstables)
{
long count = 0;
@@ -133,7 +128,7 @@ public class SSTableReader extends SSTab
for (SSTableReader sstable : sstables)
{
int indexKeyCount = sstable.getKeySamples().size();
- count = count + (indexKeyCount + 1) * IndexSummary.INDEX_INTERVAL;
+ count = count + (indexKeyCount + 1) *
DatabaseDescriptor.getIndexInterval();
if (logger.isDebugEnabled())
logger.debug("index size for bloom filter calc for file : " +
sstable.getFilename() + " : " + count);
}
@@ -331,7 +326,7 @@ public class SSTableReader extends SSTab
*/
public long estimatedKeys()
{
- return indexSummary.getIndexPositions().size() *
IndexSummary.INDEX_INTERVAL;
+ return indexSummary.getIndexPositions().size() *
DatabaseDescriptor.getIndexInterval();
}
/**
Modified:
cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java?rev=1001119&r1=1001118&r2=1001119&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
Sat Sep 25 01:31:12 2010
@@ -1382,7 +1382,7 @@ public class StorageService implements I
keys.add(sample);
}
FBUtilities.sortSampledKeys(keys, range);
- int splits = keys.size() * SSTableReader.indexInterval() /
keysPerSplit;
+ int splits = keys.size() * DatabaseDescriptor.getIndexInterval() /
keysPerSplit;
if (keys.size() >= splits)
{
Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/CompactionsTest.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/CompactionsTest.java?rev=1001119&r1=1001118&r2=1001119&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/CompactionsTest.java
(original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/CompactionsTest.java Sat
Sep 25 01:31:12 2010
@@ -31,6 +31,7 @@ import org.apache.cassandra.Util;
import org.junit.Test;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.CleanupHelper;
import org.apache.cassandra.db.filter.QueryPath;
@@ -55,7 +56,7 @@ public class CompactionsTest extends Cle
final int ROWS_PER_SSTABLE = 10;
Set<DecoratedKey> inserted = new HashSet<DecoratedKey>();
- for (int j = 0; j < (SSTableReader.indexInterval() * 3) /
ROWS_PER_SSTABLE; j++) {
+ for (int j = 0; j < (DatabaseDescriptor.getIndexInterval() * 3) /
ROWS_PER_SSTABLE; j++) {
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
DecoratedKey key = Util.dk(String.valueOf(i % 2));
RowMutation rm = new RowMutation(TABLE1, key.key);
Modified:
cassandra/trunk/test/unit/org/apache/cassandra/io/LazilyCompactedRowTest.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/io/LazilyCompactedRowTest.java?rev=1001119&r1=1001118&r2=1001119&view=diff
==============================================================================
---
cassandra/trunk/test/unit/org/apache/cassandra/io/LazilyCompactedRowTest.java
(original)
+++
cassandra/trunk/test/unit/org/apache/cassandra/io/LazilyCompactedRowTest.java
Sat Sep 25 01:31:12 2010
@@ -35,6 +35,7 @@ import org.junit.Test;
import org.apache.cassandra.CleanupHelper;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.dht.CollatingOrderPreservingPartitioner;
@@ -200,7 +201,7 @@ public class LazilyCompactedRowTest exte
ColumnFamilyStore cfs = table.getColumnFamilyStore("Standard1");
final int ROWS_PER_SSTABLE = 10;
- for (int j = 0; j < (SSTableReader.indexInterval() * 3) /
ROWS_PER_SSTABLE; j++) {
+ for (int j = 0; j < (DatabaseDescriptor.getIndexInterval() * 3) /
ROWS_PER_SSTABLE; j++) {
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
byte[] key = String.valueOf(i % 2).getBytes();
RowMutation rm = new RowMutation("Keyspace1", key);
