Author: jbellis
Date: Tue Apr 13 03:59:27 2010
New Revision: 933475
URL: http://svn.apache.org/viewvc?rev=933475&view=rev
Log:
merge from 0.6
Added:
cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
- copied, changed from r933466,
cassandra/branches/cassandra-0.6/test/unit/org/apache/cassandra/io/SSTableReaderTest.java
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/CHANGES.txt
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
(props changed)
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java
cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java
cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 13 03:59:27 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6:922689-933419
+/cassandra/branches/cassandra-0.6:922689-933466
/incubator/cassandra/branches/cassandra-0.3:774578-796573
/incubator/cassandra/branches/cassandra-0.4:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5:888872-915439
Modified: cassandra/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=933475&r1=933474&r2=933475&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Tue Apr 13 03:59:27 2010
@@ -24,6 +24,8 @@ dev
(CASSANDRA-969)
* Retrieve the correct number of undeleted columns, if any, from
a supercolumn in a row that had been deleted previously (CASSANDRA-920)
+ * fix index scans that cross the 2GB mmap boundaries for both mmap
+ and standard i/o modes (CASSANDRA-866)
0.6.0-RC1
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 13 03:59:27 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-933419
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-933466
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Cassandra.java:888872-903502
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 13 03:59:27 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-933419
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-933466
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Column.java:888872-903502
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 13 03:59:27 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-933419
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-933466
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:888872-903502
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 13 03:59:27 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-933419
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-933466
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:888872-903502
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 13 03:59:27 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-933419
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-933466
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:888872-903502
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java?rev=933475&r1=933474&r2=933475&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
Tue Apr 13 03:59:27 2010
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
public class IndexSummary
@@ -36,11 +37,13 @@ public class IndexSummary
private ArrayList<KeyPosition> indexPositions;
private Map<KeyPosition, SSTable.PositionSize> spannedIndexDataPositions;
private Map<Long, KeyPosition> spannedIndexPositions;
- int keysWritten = 0;
+ private int keysWritten = 0;
+ private long lastIndexPosition;
public void maybeAddEntry(DecoratedKey decoratedKey, long dataPosition,
long dataSize, long indexPosition, long nextIndexPosition)
{
- boolean spannedIndexEntry =
RowIndexedReader.bufferIndex(indexPosition) !=
RowIndexedReader.bufferIndex(nextIndexPosition);
+ boolean spannedIndexEntry = DatabaseDescriptor.getIndexAccessMode() ==
DatabaseDescriptor.DiskAccessMode.mmap
+ &&
RowIndexedReader.bufferIndex(indexPosition) !=
RowIndexedReader.bufferIndex(nextIndexPosition);
if (keysWritten++ % INDEX_INTERVAL == 0 || spannedIndexEntry)
{
if (indexPositions == null)
@@ -61,6 +64,7 @@ public class IndexSummary
spannedIndexPositions.put(info.indexPosition, info);
}
}
+ lastIndexPosition = indexPosition;
}
public List<KeyPosition> getIndexPositions()
@@ -73,14 +77,19 @@ public class IndexSummary
indexPositions.trimToSize();
}
- public SSTable.PositionSize getSpannedPosition(KeyPosition sampledPosition)
+ public SSTable.PositionSize getSpannedDataPosition(KeyPosition
sampledPosition)
{
if (spannedIndexDataPositions == null)
return null;
return spannedIndexDataPositions.get(sampledPosition);
}
- public SSTable.PositionSize getSpannedPosition(long nextIndexPosition)
+ public KeyPosition getSpannedIndexPosition(long nextIndexPosition)
+ {
+ return spannedIndexPositions == null ? null :
spannedIndexPositions.get(nextIndexPosition);
+ }
+
+ public SSTable.PositionSize getSpannedDataPosition(long nextIndexPosition)
{
if (spannedIndexDataPositions == null)
return null;
@@ -92,6 +101,12 @@ public class IndexSummary
return spannedIndexDataPositions.get(info);
}
+ public long getLastIndexPosition()
+ {
+ return lastIndexPosition;
+ }
+
+
/**
* This is a simple container for the index Key and its corresponding
position
* in the index file. Binary search is performed on a list of these objects
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java?rev=933475&r1=933474&r2=933475&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/RowIndexedReader.java
Tue Apr 13 03:59:27 2010
@@ -33,6 +33,7 @@ import org.apache.cassandra.cache.Instru
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.utils.BloomFilter;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -56,7 +57,8 @@ class RowIndexedReader extends SSTableRe
{
private static final Logger logger =
LoggerFactory.getLogger(RowIndexedReader.class);
- private static final long BUFFER_SIZE = Integer.MAX_VALUE;
+ // in a perfect world, BUFFER_SIZE would be final, but we need to test
with a smaller size to stay sane.
+ static long BUFFER_SIZE = Integer.MAX_VALUE;
// jvm can only map up to 2GB at a time, so we split index/data into
segments of that size when using mmap i/o
private final MappedByteBuffer[] indexBuffers;
@@ -248,15 +250,9 @@ class RowIndexedReader extends SSTableRe
if (sampledPosition == null)
return null;
- // handle exact sampled index hit
- PositionSize info = indexSummary.getSpannedPosition(sampledPosition);
- if (info != null)
- return info;
-
// get either a buffered or a mmap'd input for the on-disk index
long p = sampledPosition.indexPosition;
FileDataInput input;
- int bufferIndex = bufferIndex(p);
try
{
if (indexBuffers == null)
@@ -266,7 +262,7 @@ class RowIndexedReader extends SSTableRe
}
else
{
- input = new MappedFileDataInput(indexBuffers[bufferIndex],
indexFilename(), BUFFER_SIZE * bufferIndex, (int)(p % BUFFER_SIZE));
+ input = indexInputAt(p);
}
}
catch (IOException e)
@@ -280,12 +276,33 @@ class RowIndexedReader extends SSTableRe
int i = 0;
do
{
+ // handle exact sampled index hit
+ IndexSummary.KeyPosition kp =
indexSummary.getSpannedIndexPosition(input.getAbsolutePosition());
+ if (kp != null && kp.key.equals(decoratedKey))
+ return indexSummary.getSpannedDataPosition(kp);
+
// if using mmapped i/o, skip to the next mmap buffer if
necessary
- if (input.isEOF() ||
indexSummary.getSpannedPosition(input.getAbsolutePosition()) != null)
+ if (input.isEOF() || kp != null)
{
- if (indexBuffers == null || ++bufferIndex ==
indexBuffers.length)
+ if (indexBuffers == null) // not mmap-ing, just one index
input
+ break;
+
+ FileDataInput oldInput = input;
+ if (kp == null)
+ {
+ input = indexInputAt(input.getAbsolutePosition());
+ }
+ else
+ {
+ long nextUnspannedPostion = input.getAbsolutePosition()
+ + 2 +
FBUtilities.encodedUTF8Length(StorageService.getPartitioner().convertToDiskFormat(kp.key))
+ + 8;
+ input = indexInputAt(nextUnspannedPostion);
+ }
+ oldInput.close();
+ if (input == null)
break;
- input = new MappedFileDataInput(indexBuffers[bufferIndex],
indexFilename(), BUFFER_SIZE * bufferIndex, 0);
+
continue;
}
@@ -296,7 +313,7 @@ class RowIndexedReader extends SSTableRe
int v = indexDecoratedKey.compareTo(decoratedKey);
if (v == 0)
{
- info = getDataPositionSize(input, dataPosition);
+ PositionSize info = getDataPositionSize(input,
dataPosition);
if (keyCache != null && keyCache.getCapacity() > 0)
keyCache.put(unifiedKey, info);
return info;
@@ -324,6 +341,14 @@ class RowIndexedReader extends SSTableRe
return null;
}
+ private FileDataInput indexInputAt(long indexPosition)
+ {
+ if (indexPosition > indexSummary.getLastIndexPosition())
+ return null;
+ int bufferIndex = bufferIndex(indexPosition);
+ return new MappedFileDataInput(indexBuffers[bufferIndex],
indexFilename(), BUFFER_SIZE * bufferIndex, (int)(indexPosition % BUFFER_SIZE));
+ }
+
private PositionSize getDataPositionSize(FileDataInput input, long
dataPosition) throws IOException
{
// if we've reached the end of the index, then the row size is "the
rest of the data file"
@@ -333,7 +358,7 @@ class RowIndexedReader extends SSTableRe
// otherwise, row size is the start of the next row (in next index
entry), minus the start of this one.
long nextIndexPosition = input.getAbsolutePosition();
// if next index entry would span mmap boundary, get the next row
position from the summary instead
- PositionSize nextPositionSize =
indexSummary.getSpannedPosition(nextIndexPosition);
+ PositionSize nextPositionSize =
indexSummary.getSpannedDataPosition(nextIndexPosition);
if (nextPositionSize != null)
return new PositionSize(dataPosition, nextPositionSize.position -
dataPosition);
Modified:
cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java?rev=933475&r1=933474&r2=933475&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java
Tue Apr 13 03:59:27 2010
@@ -34,6 +34,7 @@ import org.apache.cassandra.dht.IPartiti
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
public class ThriftValidation
{
@@ -44,18 +45,7 @@ public class ThriftValidation
throw new InvalidRequestException("Key may not be empty");
}
// check that writeUTF will be able to handle it -- encoded length
must fit in 2 bytes
- int strlen = key.length();
- int utflen = 0;
- for (int i = 0; i < strlen; i++)
- {
- int c = key.charAt(i);
- if ((c >= 0x0001) && (c <= 0x007F))
- utflen++;
- else if (c > 0x07FF)
- utflen += 3;
- else
- utflen += 2;
- }
+ int utflen = FBUtilities.encodedUTF8Length(key);
if (utflen > 65535)
throw new InvalidRequestException("Encoded key length of " +
utflen + " is longer than maximum of 65535");
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java?rev=933475&r1=933474&r2=933475&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java Tue
Apr 13 03:59:27 2010
@@ -453,4 +453,20 @@ public class FBUtilities
}
}
+ public static int encodedUTF8Length(String st)
+ {
+ int strlen = st.length();
+ int utflen = 0;
+ for (int i = 0; i < strlen; i++)
+ {
+ int c = st.charAt(i);
+ if ((c >= 0x0001) && (c <= 0x007F))
+ utflen++;
+ else if (c > 0x07FF)
+ utflen += 3;
+ else
+ utflen += 2;
+ }
+ return utflen;
+ }
}
Copied:
cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
(from r933466,
cassandra/branches/cassandra-0.6/test/unit/org/apache/cassandra/io/SSTableReaderTest.java)
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java?p2=cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java&p1=cassandra/branches/cassandra-0.6/test/unit/org/apache/cassandra/io/SSTableReaderTest.java&r1=933466&r2=933475&rev=933475&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.6/test/unit/org/apache/cassandra/io/SSTableReaderTest.java
(original)
+++
cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
Tue Apr 13 03:59:27 2010
@@ -1,4 +1,4 @@
-package org.apache.cassandra.io;
+package org.apache.cassandra.io.sstable;
import java.io.IOException;
import java.util.HashSet;
@@ -7,6 +7,7 @@ import java.util.concurrent.ExecutionExc
import org.junit.Test;
+import org.apache.cassandra.CleanupHelper;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.QueryPath;
@@ -14,12 +15,12 @@ import org.apache.cassandra.io.util.File
import org.apache.cassandra.service.StorageService;
-public class SSTableReaderTest
+public class SSTableReaderTest extends CleanupHelper
{
@Test
public void testSpannedIndexPositions() throws IOException,
ExecutionException, InterruptedException
{
- SSTableReader.BUFFER_SIZE = 40;
+ RowIndexedReader.BUFFER_SIZE = 40;
Table table = Table.open("Keyspace1");
ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
