Author: jbellis
Date: Sat Mar 20 02:50:50 2010
New Revision: 925536
URL: http://svn.apache.org/viewvc?rev=925536&view=rev
Log:
merge from 0.6
Added:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
- copied, changed from r925514,
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/CHANGES.txt
cassandra/trunk/contrib/ (props changed)
cassandra/trunk/contrib/client_only/ (props changed)
cassandra/trunk/interface/ (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
(props changed)
cassandra/trunk/src/ (props changed)
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java
cassandra/trunk/src/java/org/apache/cassandra/io/util/FileDataInput.java
cassandra/trunk/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java
cassandra/trunk/src/java/org/apache/cassandra/service/AntiEntropyService.java
cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
cassandra/trunk/test/ (props changed)
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Mar 20 02:50:50 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6:922689-925440
+/cassandra/branches/cassandra-0.6:922689-925514
/incubator/cassandra/branches/cassandra-0.3:774578-796573
/incubator/cassandra/branches/cassandra-0.4:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5:888872-915439
Modified: cassandra/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=925536&r1=925535&r2=925536&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Sat Mar 20 02:50:50 2010
@@ -10,6 +10,7 @@ dev
* Bootstrapping can skip ranges under the right conditions (CASSANDRA-902)
* fix merging row versions in range_slice for CL > ONE (CASSANDRA-884)
* default write ConsistencyLeven chaned from ZERO to ONE
+ * fix for index entries spanning mmap buffer boundaries (CASSANDRA-857)
* use lexical comparison if time part of TimeUUIDs are the same
(CASSANDRA-907)
* bound read, mutation, and response stages to fix possible OOM
Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Sat Mar 20 02:50:50 2010
@@ -0,0 +1 @@
+*.iml
Propchange: cassandra/trunk/contrib/client_only/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Sat Mar 20 02:50:50 2010
@@ -0,0 +1 @@
+*.iml
Propchange: cassandra/trunk/interface/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Sat Mar 20 02:50:50 2010
@@ -1 +1,2 @@
avro
+*.iml
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Mar 20 02:50:50 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-925440
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-925514
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Cassandra.java:888872-903502
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Mar 20 02:50:50 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-925440
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-925514
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Column.java:888872-903502
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Mar 20 02:50:50 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-925440
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-925514
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:888872-903502
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Mar 20 02:50:50 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-925440
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-925514
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:888872-903502
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Mar 20 02:50:50 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-925440
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-925514
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:888872-903502
Propchange: cassandra/trunk/src/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Sat Mar 20 02:50:50 2010
@@ -1,2 +1,2 @@
gen-java
-
+*.iml
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=925536&r1=925535&r2=925536&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Sat
Mar 20 02:50:50 2010
@@ -49,10 +49,7 @@ import org.apache.cassandra.dht.Abstract
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableScanner;
-import org.apache.cassandra.io.sstable.SSTableTracker;
+import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.thrift.SliceRange;
Copied:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
(from r925514,
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java)
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java?p2=cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java&p1=cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java&r1=925514&r2=925536&rev=925536&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
Sat Mar 20 02:50:50 2010
@@ -1,4 +1,4 @@
-package org.apache.cassandra.io;
+package org.apache.cassandra.io.sstable;
import java.util.ArrayList;
import java.util.HashMap;
@@ -19,7 +19,7 @@ public class IndexSummary
public void maybeAddEntry(DecoratedKey decoratedKey, long dataPosition,
long dataSize, long indexPosition, long nextIndexPosition)
{
- boolean spannedIndexEntry = SSTableReader.bufferIndex(indexPosition)
!= SSTableReader.bufferIndex(nextIndexPosition);
+ boolean spannedIndexEntry =
RowIndexedReader.bufferIndex(indexPosition) !=
RowIndexedReader.bufferIndex(nextIndexPosition);
if (keysWritten++ % INDEX_INTERVAL == 0 || spannedIndexEntry)
{
if (indexPositions == null)
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java?rev=925536&r1=925535&r2=925536&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java
Sat Mar 20 02:50:50 2010
@@ -65,8 +65,7 @@ class RowIndexedReader extends SSTableRe
RowIndexedReader(Descriptor desc,
IPartitioner partitioner,
- List<KeyPosition> indexPositions,
- Map<KeyPosition, PositionSize> spannedIndexDataPositions,
+ IndexSummary indexSummary,
BloomFilter bloomFilter)
throws IOException
{
@@ -107,14 +106,13 @@ class RowIndexedReader extends SSTableRe
buffers = null;
}
- this.indexPositions = indexPositions;
- this.spannedIndexDataPositions = spannedIndexDataPositions;
+ this.indexSummary = indexSummary;
this.bf = bloomFilter;
}
RowIndexedReader(Descriptor desc, IPartitioner partitioner) throws
IOException
{
- this(desc, partitioner, null, null, null);
+ this(desc, partitioner, null, null);
}
public static RowIndexedReader open(Descriptor desc, IPartitioner
partitioner) throws IOException
@@ -128,14 +126,14 @@ class RowIndexedReader extends SSTableRe
public long estimatedKeys()
{
- return (indexPositions.size() + 1) * INDEX_INTERVAL;
+ return (indexSummary.getIndexPositions().size() + 1) *
IndexSummary.INDEX_INTERVAL;
}
public Collection<DecoratedKey> getKeySamples()
{
- return Collections2.transform(indexPositions,
- new Function<KeyPosition,DecoratedKey>(){
- public DecoratedKey
apply(KeyPosition kp)
+ return Collections2.transform(indexSummary.getIndexPositions(),
+ new Function<IndexSummary.KeyPosition,
DecoratedKey>(){
+ public DecoratedKey
apply(IndexSummary.KeyPosition kp)
{
return kp.key;
}
@@ -157,14 +155,13 @@ class RowIndexedReader extends SSTableRe
void loadIndexFile() throws IOException
{
- indexPositions = new ArrayList<KeyPosition>();
// we read the positions in a BRAF so we don't have to worry about an
entry spanning a mmap boundary.
// any entries that do, we force into the in-memory sample so key
lookup can always bsearch within
// a single mmapped segment.
+ indexSummary = new IndexSummary();
BufferedRandomAccessFile input = new
BufferedRandomAccessFile(indexFilename(), "r");
try
{
- int i = 0;
long indexSize = input.length();
while (true)
{
@@ -176,27 +173,21 @@ class RowIndexedReader extends SSTableRe
DecoratedKey decoratedKey =
partitioner.convertFromDiskFormat(input.readUTF());
long dataPosition = input.readLong();
long nextIndexPosition = input.getFilePointer();
- boolean spannedEntry = bufferIndex(indexPosition) !=
bufferIndex(nextIndexPosition);
- if (i++ % INDEX_INTERVAL == 0 || spannedEntry)
+ // read the next index entry to see how big the row is
+ long nextDataPosition;
+ if (input.isEOF())
+ {
+ nextDataPosition = length();
+ }
+ else
{
- KeyPosition info;
- info = new KeyPosition(decoratedKey, indexPosition);
- indexPositions.add(info);
-
- if (spannedEntry)
- {
- if (spannedIndexDataPositions == null)
- {
- spannedIndexDataPositions = new
HashMap<KeyPosition, PositionSize>();
- }
- // read the next index entry to see how big the row is
corresponding to the current, mmap-segment-spanning one
- input.readUTF();
- long nextDataPosition = input.readLong();
- input.seek(nextIndexPosition);
- spannedIndexDataPositions.put(info, new
PositionSize(dataPosition, nextDataPosition - dataPosition));
- }
+ input.readUTF();
+ nextDataPosition = input.readLong();
+ input.seek(nextIndexPosition);
}
+ indexSummary.maybeAddEntry(decoratedKey, dataPosition,
nextDataPosition - dataPosition, indexPosition, nextIndexPosition);
}
+ indexSummary.complete();
}
finally
{
@@ -212,10 +203,10 @@ class RowIndexedReader extends SSTableRe
}
/** get the position in the index file to start scanning to find the given
key (at most indexInterval keys away) */
- private KeyPosition getIndexScanPosition(DecoratedKey decoratedKey)
+ private IndexSummary.KeyPosition getIndexScanPosition(DecoratedKey
decoratedKey)
{
- assert indexPositions != null && indexPositions.size() > 0;
- int index = Collections.binarySearch(indexPositions, new
KeyPosition(decoratedKey, -1));
+ assert indexSummary.getIndexPositions() != null &&
indexSummary.getIndexPositions().size() > 0;
+ int index = Collections.binarySearch(indexSummary.getIndexPositions(),
new IndexSummary.KeyPosition(decoratedKey, -1));
if (index < 0)
{
// binary search gives us the first index _greater_ than the key
searched for,
@@ -223,11 +214,11 @@ class RowIndexedReader extends SSTableRe
int greaterThan = (index + 1) * -1;
if (greaterThan == 0)
return null;
- return indexPositions.get(greaterThan - 1);
+ return indexSummary.getIndexPositions().get(greaterThan - 1);
}
else
{
- return indexPositions.get(index);
+ return indexSummary.getIndexPositions().get(index);
}
}
@@ -252,23 +243,19 @@ class RowIndexedReader extends SSTableRe
}
// next, see if the sampled index says it's impossible for the key to
be present
- KeyPosition sampledPosition = getIndexScanPosition(decoratedKey);
+ IndexSummary.KeyPosition sampledPosition =
getIndexScanPosition(decoratedKey);
if (sampledPosition == null)
- {
return null;
- }
// handle exact sampled index hit
- if (spannedIndexDataPositions != null)
- {
- PositionSize info = spannedIndexDataPositions.get(sampledPosition);
- if (info != null)
- return info;
- }
-
- // scan the on-disk index, starting at the nearest sampled position
- long p = sampledPosition.position;
- FileDataInput input = null;
+ PositionSize info = indexSummary.getSpannedPosition(sampledPosition);
+ if (info != null)
+ return info;
+
+ // get either a buffered or a mmap'd input for the on-disk index
+ long p = sampledPosition.indexPosition;
+ FileDataInput input;
+ int bufferIndex = bufferIndex(p);
try
{
if (indexBuffers == null)
@@ -278,43 +265,44 @@ class RowIndexedReader extends SSTableRe
}
else
{
- input = new MappedFileDataInput(indexBuffers[bufferIndex(p)],
indexFilename(), (int)(p % BUFFER_SIZE));
+ input = new MappedFileDataInput(indexBuffers[bufferIndex],
indexFilename(), BUFFER_SIZE * bufferIndex, (int)(p % BUFFER_SIZE));
}
+ }
+ catch (IOException e)
+ {
+ throw new IOError(e);
+ }
+
+ // scan the on-disk index, starting at the nearest sampled position
+ try
+ {
int i = 0;
do
{
- DecoratedKey indexDecoratedKey;
- try
- {
- indexDecoratedKey =
partitioner.convertFromDiskFormat(input.readUTF());
- }
- catch (EOFException e)
+ // if using mmapped i/o, skip to the next mmap buffer if
necessary
+ if (input.isEOF() ||
indexSummary.getSpannedPosition(input.getAbsolutePosition()) != null)
{
- return null;
+ if (indexBuffers == null || ++bufferIndex ==
indexBuffers.length)
+ break;
+ input = new MappedFileDataInput(indexBuffers[bufferIndex],
indexFilename(), BUFFER_SIZE * bufferIndex, 0);
+ continue;
}
- long position = input.readLong();
+
+ // read key & data position from index entry
+ DecoratedKey indexDecoratedKey =
partitioner.convertFromDiskFormat(input.readUTF());
+ long dataPosition = input.readLong();
+
int v = indexDecoratedKey.compareTo(decoratedKey);
if (v == 0)
{
- PositionSize info;
- if (!input.isEOF())
- {
- int utflen = input.readUnsignedShort();
- if (utflen != input.skipBytes(utflen))
- throw new EOFException();
- info = new PositionSize(position, input.readLong() -
position);
- }
- else
- {
- info = new PositionSize(position, length() - position);
- }
+ info = getDataPositionSize(input, dataPosition);
if (keyCache != null && keyCache.getCapacity() > 0)
keyCache.put(unifiedKey, info);
return info;
}
if (v > 0)
return null;
- } while (++i < INDEX_INTERVAL);
+ } while (++i < IndexSummary.INDEX_INTERVAL);
}
catch (IOException e)
{
@@ -335,10 +323,30 @@ class RowIndexedReader extends SSTableRe
return null;
}
+ private PositionSize getDataPositionSize(FileDataInput input, long
dataPosition) throws IOException
+ {
+ // if we've reached the end of the index, then the row size is "the
rest of the data file"
+ if (input.isEOF())
+ return new PositionSize(dataPosition, length() - dataPosition);
+
+ // otherwise, row size is the start of the next row (in next index
entry), minus the start of this one.
+ long nextIndexPosition = input.getAbsolutePosition();
+ // if next index entry would span mmap boundary, get the next row
position from the summary instead
+ PositionSize nextPositionSize =
indexSummary.getSpannedPosition(nextIndexPosition);
+ if (nextPositionSize != null)
+ return new PositionSize(dataPosition, nextPositionSize.position -
dataPosition);
+
+ // read next entry directly
+ int utflen = input.readUnsignedShort();
+ if (utflen != input.skipBytes(utflen))
+ throw new EOFException();
+ return new PositionSize(dataPosition, input.readLong() - dataPosition);
+ }
+
/** like getPosition, but if key is not found will return the location of
the first key _greater_ than the desired one, or -1 if no such key exists. */
public long getNearestPosition(DecoratedKey decoratedKey) throws
IOException
{
- KeyPosition sampledPosition = getIndexScanPosition(decoratedKey);
+ IndexSummary.KeyPosition sampledPosition =
getIndexScanPosition(decoratedKey);
if (sampledPosition == null)
{
return 0;
@@ -346,7 +354,7 @@ class RowIndexedReader extends SSTableRe
// can't use a MappedFileDataInput here, since we might cross a
segment boundary while scanning
BufferedRandomAccessFile input = new
BufferedRandomAccessFile(indexFilename(), "r");
- input.seek(sampledPosition.position);
+ input.seek(sampledPosition.indexPosition);
try
{
while (true)
@@ -411,7 +419,7 @@ class RowIndexedReader extends SSTableRe
throw new IOError(e);
}
}
- return new MappedFileDataInput(buffers[bufferIndex(info.position)],
getFilename(), (int) (info.position % BUFFER_SIZE));
+ return new MappedFileDataInput(buffers[bufferIndex(info.position)],
getFilename(), BUFFER_SIZE * (info.position / BUFFER_SIZE), (int)
(info.position % BUFFER_SIZE));
}
static int bufferIndex(long position)
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=925536&r1=925535&r2=925536&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java Sat
Mar 20 02:50:50 2010
@@ -24,7 +24,6 @@ import java.io.IOError;
import java.io.IOException;
import java.util.List;
import java.util.Arrays;
-import java.util.Map;
import java.util.StringTokenizer;
import org.slf4j.Logger;
@@ -64,11 +63,8 @@ public abstract class SSTable
protected Descriptor desc;
protected IPartitioner partitioner;
protected BloomFilter bf;
- protected List<KeyPosition> indexPositions;
- protected Map<KeyPosition, PositionSize> spannedIndexDataPositions; // map
of index position, to data position, for index entries spanning mmap segments
+ protected IndexSummary indexSummary;
- /* Every 128th index entry is loaded into memory so we know where to start
looking for the actual key w/o seeking */
- public static final int INDEX_INTERVAL = 128;/* Required extension for
temporary files created during compactions. */
public static final String TEMPFILE_MARKER = "tmp";
protected SSTable(String filename, IPartitioner partitioner)
@@ -196,33 +192,6 @@ public abstract class SSTable
return sum;
}
- /**
- * This is a simple container for the index Key and its corresponding
position
- * in the data file. Binary search is performed on a list of these objects
- * to lookup keys within the SSTable data file.
- */
- public static class KeyPosition implements Comparable<KeyPosition>
- {
- public final DecoratedKey key;
- public final long position;
-
- public KeyPosition(DecoratedKey key, long position)
- {
- this.key = key;
- this.position = position;
- }
-
- public int compareTo(KeyPosition kp)
- {
- return key.compareTo(kp.key);
- }
-
- public String toString()
- {
- return key + ":" + position;
- }
- }
-
public long bytesOnDisk()
{
long bytes = 0;
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=925536&r1=925535&r2=925536&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Sat Mar 20 02:50:50 2010
@@ -89,7 +89,7 @@ public abstract class SSTableReader exte
public static int indexInterval()
{
- return INDEX_INTERVAL;
+ return IndexSummary.INDEX_INTERVAL;
}
public static long getApproximateKeyCount(Iterable<SSTableReader> sstables)
@@ -99,7 +99,7 @@ public abstract class SSTableReader exte
for (SSTableReader sstable : sstables)
{
int indexKeyCount = sstable.getKeySamples().size();
- count = count + (indexKeyCount + 1) * INDEX_INTERVAL;
+ count = count + (indexKeyCount + 1) * IndexSummary.INDEX_INTERVAL;
if (logger.isDebugEnabled())
logger.debug("index size for bloom filter calc for file : " +
sstable.getFilename() + " : " + count);
}
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=925536&r1=925535&r2=925536&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
Sat Mar 20 02:50:50 2010
@@ -34,7 +34,6 @@ import org.apache.cassandra.db.Decorated
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.util.BufferedRandomAccessFile;
import org.apache.cassandra.io.util.DataOutputBuffer;
-import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.BloomFilter;
import org.apache.cassandra.utils.FBUtilities;
@@ -51,6 +50,7 @@ public class SSTableWriter extends SSTab
public SSTableWriter(String filename, long keyCount, IPartitioner
partitioner) throws IOException
{
super(filename, partitioner);
+ indexSummary = new IndexSummary();
dataFile = new BufferedRandomAccessFile(getFilename(), "rw",
(int)(DatabaseDescriptor.getFlushDataBufferSizeInMB() * 1024 * 1024));
indexFile = new BufferedRandomAccessFile(indexFilename(), "rw",
(int)(DatabaseDescriptor.getFlushIndexBufferSizeInMB() * 1024 * 1024));
bf = BloomFilter.getFilter(keyCount, 15);
@@ -85,25 +85,7 @@ public class SSTableWriter extends SSTab
if (logger.isTraceEnabled())
logger.trace("wrote index of " + decoratedKey + " at " +
indexPosition);
- boolean spannedIndexEntry =
RowIndexedReader.bufferIndex(indexPosition) !=
RowIndexedReader.bufferIndex(indexFile.getFilePointer());
- if (keysWritten++ % INDEX_INTERVAL == 0 || spannedIndexEntry)
- {
- if (indexPositions == null)
- {
- indexPositions = new ArrayList<KeyPosition>();
- }
- KeyPosition info = new KeyPosition(decoratedKey, indexPosition);
- indexPositions.add(info);
-
- if (spannedIndexEntry)
- {
- if (spannedIndexDataPositions == null)
- {
- spannedIndexDataPositions = new HashMap<KeyPosition,
PositionSize>();
- }
- spannedIndexDataPositions.put(info, new
PositionSize(dataPosition, dataSize));
- }
- }
+ indexSummary.maybeAddEntry(decoratedKey, dataPosition, dataSize,
indexPosition, indexFile.getFilePointer());
}
// TODO make this take a DataOutputStream and wrap the byte[] version to
combine them
@@ -150,7 +132,8 @@ public class SSTableWriter extends SSTab
rename(filterFilename());
rename(getFilename());
- return new RowIndexedReader(newdesc, partitioner, indexPositions,
spannedIndexDataPositions, bf);
+ indexSummary.complete();
+ return new RowIndexedReader(newdesc, partitioner, indexSummary, bf);
}
static String rename(String tmpFilename)
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java?rev=925536&r1=925535&r2=925536&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java
Sat Mar 20 02:50:50 2010
@@ -253,7 +253,12 @@ public class BufferedRandomAccessFile ex
}
this.curr_ = pos;
}
-
+
+ public long getAbsolutePosition()
+ {
+ return getFilePointer();
+ }
+
public long getFilePointer()
{
return this.curr_;
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/util/FileDataInput.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/util/FileDataInput.java?rev=925536&r1=925535&r2=925536&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/util/FileDataInput.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/util/FileDataInput.java
Sat Mar 20 02:50:50 2010
@@ -36,4 +36,6 @@ public interface FileDataInput extends D
public void reset() throws IOException;
public int bytesPastMark();
+
+ long getAbsolutePosition();
}
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java?rev=925536&r1=925535&r2=925536&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java
Sat Mar 20 02:50:50 2010
@@ -30,21 +30,28 @@ public class MappedFileDataInput extends
private final String filename;
private int position;
private int markedPosition;
+ private final long absoluteStartPosition;
- public MappedFileDataInput(MappedByteBuffer buffer, String filename)
+ public MappedFileDataInput(MappedByteBuffer buffer, String filename, long
absoluteStartPosition)
{
- this(buffer, filename, 0);
+ this(buffer, filename, absoluteStartPosition, 0);
}
- public MappedFileDataInput(MappedByteBuffer buffer, String filename, int
position)
+ public MappedFileDataInput(MappedByteBuffer buffer, String filename, long
absoluteStartPosition, int position)
{
assert buffer != null;
+ this.absoluteStartPosition = absoluteStartPosition;
this.buffer = buffer;
this.filename = filename;
this.position = position;
}
- // don't make this public, this is only for seeking WITHIN the current
mapped segment
+ public long getAbsolutePosition()
+ {
+ return absoluteStartPosition + position;
+ }
+
+// don't make this public, this is only for seeking WITHIN the current mapped
segment
private void seekInternal(int pos) throws IOException
{
position = pos;
Modified:
cassandra/trunk/src/java/org/apache/cassandra/service/AntiEntropyService.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/AntiEntropyService.java?rev=925536&r1=925535&r2=925536&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/service/AntiEntropyService.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/service/AntiEntropyService.java
Sat Mar 20 02:50:50 2010
@@ -35,6 +35,7 @@ import org.apache.cassandra.io.Compactio
import org.apache.cassandra.io.ICompactSerializer;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.IndexSummary;
import org.apache.cassandra.streaming.StreamOut;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
@@ -45,7 +46,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Collections2;
-import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
/**
Modified:
cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java?rev=925536&r1=925535&r2=925536&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
Sat Mar 20 02:50:50 2010
@@ -27,7 +27,6 @@ import java.util.concurrent.CountDownLat
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
-import java.util.concurrent.FutureTask;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.net.InetAddress;
@@ -43,6 +42,7 @@ import org.apache.cassandra.gms.*;
import org.apache.cassandra.io.DeletionService;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.IndexSummary;
import org.apache.cassandra.locator.*;
import org.apache.cassandra.net.*;
import org.apache.cassandra.service.AntiEntropyService.TreeRequestVerbHandler;
Propchange: cassandra/trunk/test/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Sat Mar 20 02:50:50 2010
@@ -0,0 +1 @@
+*.iml
