Author: jbellis
Date: Sat Sep 18 07:19:15 2010
New Revision: 998419
URL: http://svn.apache.org/viewvc?rev=998419&view=rev
Log:
make IndexInterval configurable. patch by Jon Hermes; reviewed by jbellis for
CASSANDRA-1488
Modified:
cassandra/branches/cassandra-0.6/CHANGES.txt
cassandra/branches/cassandra-0.6/conf/storage-conf.xml
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/service/StorageService.java
cassandra/branches/cassandra-0.6/test/unit/org/apache/cassandra/db/CompactionsTest.java
Modified: cassandra/branches/cassandra-0.6/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/CHANGES.txt?rev=998419&r1=998418&r2=998419&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.6/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.6/CHANGES.txt Sat Sep 18 07:19:15 2010
@@ -21,6 +21,7 @@
* nodes that coordinated a loadbalance in the past could not be seen by
newly added nodes (CASSANDRA-1467)
* use JNA, if present, to take snapshots (CASSANDRA-1371)
+ * make IndexInterval configurable (CASSANDRA-1488)
0.6.5
Modified: cassandra/branches/cassandra-0.6/conf/storage-conf.xml
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/conf/storage-conf.xml?rev=998419&r1=998418&r2=998419&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.6/conf/storage-conf.xml (original)
+++ cassandra/branches/cassandra-0.6/conf/storage-conf.xml Sat Sep 18 07:19:15
2010
@@ -51,6 +51,15 @@
<HintedHandoffEnabled>true</HintedHandoffEnabled>
<!--
+ ~ The Index Interval determines the rate of sampling of row keys
+ ~ is for a given SSTable: 1/IndexInterval keys are held in memory
+ ~ for the lifetime of the sstable for use during key lookup.
+ ~ (This is separate from the KeyCache.) Larger intervals will result
+ ~ in lower memory usage at the cost of slower row lookup at read time.
+ -->
+ <IndexInterval>128</IndexInterval>
+
+ <!--
~ Keyspaces and ColumnFamilies:
~ A ColumnFamily is the Cassandra concept closest to a relational
~ table. Keyspaces are separate groups of ColumnFamilies. Except in
Modified:
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=998419&r1=998418&r2=998419&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
(original)
+++
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
Sat Sep 18 07:19:15 2010
@@ -138,6 +138,8 @@ public class DatabaseDescriptor
private static IAuthenticator authenticator = new AllowAllAuthenticator();
+ private static int indexinterval = 128;
+
private final static String STORAGE_CONF_FILE = "storage-conf.xml";
/**
@@ -494,6 +496,14 @@ public class DatabaseDescriptor
if (logger.isDebugEnabled())
logger.debug("setting hintedHandoffEnabled to " +
hintedHandoffEnabled);
+ String indexIntervalStr =
xmlUtils.getNodeValue("/Storage/IndexInterval");
+ if (indexIntervalStr != null)
+ {
+ indexinterval = Integer.parseInt(indexIntervalStr);
+ if (indexinterval <= 0)
+ throw new ConfigurationException("Index Interval must be a
positive, non-zero integer.");
+ }
+
readTablesFromXml();
if (tables.isEmpty())
throw new ConfigurationException("No keyspaces configured");
@@ -1206,4 +1216,9 @@ public class DatabaseDescriptor
{
return hintedHandoffEnabled;
}
+
+ public static int getIndexInterval()
+ {
+ return indexinterval;
+ }
}
Modified:
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java?rev=998419&r1=998418&r2=998419&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java
(original)
+++
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java
Sat Sep 18 07:19:15 2010
@@ -270,7 +270,7 @@ public class CompactionManager implement
long totalkeysWritten = 0;
// TODO the int cast here is potentially buggy
- int expectedBloomFilterSize = Math.max(SSTableReader.indexInterval(),
(int)SSTableReader.getApproximateKeyCount(sstables));
+ int expectedBloomFilterSize =
Math.max(DatabaseDescriptor.getIndexInterval(),
(int)SSTableReader.getApproximateKeyCount(sstables));
if (logger.isDebugEnabled())
logger.debug("Expected bloom filter size : " +
expectedBloomFilterSize);
@@ -354,7 +354,7 @@ public class CompactionManager implement
long startTime = System.currentTimeMillis();
long totalkeysWritten = 0;
- int expectedBloomFilterSize = Math.max(SSTableReader.indexInterval(),
(int)(SSTableReader.getApproximateKeyCount(sstables) / 2));
+ int expectedBloomFilterSize =
Math.max(DatabaseDescriptor.getIndexInterval(),
(int)(SSTableReader.getApproximateKeyCount(sstables) / 2));
if (logger.isDebugEnabled())
logger.debug("Expected bloom filter size : " +
expectedBloomFilterSize);
Modified:
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java?rev=998419&r1=998418&r2=998419&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java
(original)
+++
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java
Sat Sep 18 07:19:15 2010
@@ -31,9 +31,6 @@ import org.apache.cassandra.db.Decorated
public class IndexSummary
{
- /** Every 128th index entry is loaded into memory so we know where to
start looking for the actual key w/o seeking */
- public static final int INDEX_INTERVAL = 128;/* Required extension for
temporary files created during compactions. */
-
private ArrayList<KeyPosition> indexPositions;
private Map<KeyPosition, SSTable.PositionSize> spannedIndexDataPositions;
private Map<Long, KeyPosition> spannedIndexPositions;
@@ -44,7 +41,7 @@ public class IndexSummary
{
boolean spannedIndexEntry = DatabaseDescriptor.getIndexAccessMode() ==
DatabaseDescriptor.DiskAccessMode.mmap
&&
SSTableReader.bufferIndex(indexPosition) !=
SSTableReader.bufferIndex(nextIndexPosition);
- if ((keysWritten++ % INDEX_INTERVAL == 0) || spannedIndexEntry)
+ if ((keysWritten++ % DatabaseDescriptor.getIndexInterval() == 0) ||
spannedIndexEntry)
{
if (indexPositions == null)
{
Modified:
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java?rev=998419&r1=998418&r2=998419&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java
(original)
+++
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java
Sat Sep 18 07:19:15 2010
@@ -85,11 +85,6 @@ public class SSTableReader extends SSTab
// in a perfect world, BUFFER_SIZE would be final, but we need to test
with a smaller size to stay sane.
static long BUFFER_SIZE = Integer.MAX_VALUE;
- public static int indexInterval()
- {
- return IndexSummary.INDEX_INTERVAL;
- }
-
public static long getApproximateKeyCount(Iterable<SSTableReader> sstables)
{
long count = 0;
@@ -97,7 +92,7 @@ public class SSTableReader extends SSTab
for (SSTableReader sstable : sstables)
{
int indexKeyCount = sstable.getIndexPositions().size();
- count = count + (indexKeyCount + 1) * IndexSummary.INDEX_INTERVAL;
+ count = count + (indexKeyCount + 1) *
DatabaseDescriptor.getIndexInterval();
if (logger.isDebugEnabled())
logger.debug("index size for bloom filter calc for file : " +
sstable.getFilename() + " : " + count);
}
@@ -223,7 +218,7 @@ public class SSTableReader extends SSTab
public long estimatedKeys()
{
- return indexSummary.getIndexPositions().size() *
IndexSummary.INDEX_INTERVAL;
+ return indexSummary.getIndexPositions().size() *
DatabaseDescriptor.getIndexInterval();
}
void loadBloomFilter() throws IOException
@@ -345,6 +340,7 @@ public class SSTableReader extends SSTab
// scan the on-disk index, starting at the nearest sampled position
try
{
+ int interval = DatabaseDescriptor.getIndexInterval();
int i = 0;
do
{
@@ -398,7 +394,7 @@ public class SSTableReader extends SSTab
bloomFilterTracker.addFalsePositive();
return null;
}
- } while (++i < IndexSummary.INDEX_INTERVAL);
+ } while (++i < interval);
}
finally
{
Modified:
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/service/StorageService.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/service/StorageService.java?rev=998419&r1=998418&r2=998419&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/service/StorageService.java
(original)
+++
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/service/StorageService.java
Sat Sep 18 07:19:15 2010
@@ -42,7 +42,6 @@ import org.apache.cassandra.dht.*;
import org.apache.cassandra.gms.*;
import org.apache.cassandra.io.DeletionService;
import org.apache.cassandra.io.IndexSummary;
-import org.apache.cassandra.io.SSTableReader;
import org.apache.cassandra.locator.*;
import org.apache.cassandra.net.*;
import org.apache.cassandra.service.AntiEntropyService.TreeRequestVerbHandler;
@@ -1301,7 +1300,7 @@ public class StorageService implements I
}
}
FBUtilities.sortSampledKeys(keys, range);
- int splits = keys.size() * SSTableReader.indexInterval() /
keysPerSplit;
+ int splits = keys.size() * DatabaseDescriptor.getIndexInterval() /
keysPerSplit;
if (keys.size() >= splits)
{
Modified:
cassandra/branches/cassandra-0.6/test/unit/org/apache/cassandra/db/CompactionsTest.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/test/unit/org/apache/cassandra/db/CompactionsTest.java?rev=998419&r1=998418&r2=998419&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.6/test/unit/org/apache/cassandra/db/CompactionsTest.java
(original)
+++
cassandra/branches/cassandra-0.6/test/unit/org/apache/cassandra/db/CompactionsTest.java
Sat Sep 18 07:19:15 2010
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.net.InetAddress;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
-import java.util.ArrayList;
import java.util.Set;
import java.util.HashSet;
@@ -30,7 +29,7 @@ import org.apache.cassandra.Util;
import org.junit.Test;
-import org.apache.cassandra.io.SSTableReader;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.CleanupHelper;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.utils.FBUtilities;
@@ -53,7 +52,7 @@ public class CompactionsTest extends Cle
final int ROWS_PER_SSTABLE = 10;
Set<String> inserted = new HashSet<String>();
- for (int j = 0; j < (SSTableReader.indexInterval() * 3) /
ROWS_PER_SSTABLE; j++) {
+ for (int j = 0; j < (DatabaseDescriptor.getIndexInterval() * 3) /
ROWS_PER_SSTABLE; j++) {
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
String key = String.valueOf(i % 2);
RowMutation rm = new RowMutation(TABLE1, key);
