Author: jbellis
Date: Mon Dec 20 04:56:59 2010
New Revision: 1050991
URL: http://svn.apache.org/viewvc?rev=1050991&view=rev
Log:
merge from 0.7
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/CHANGES.txt
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
(props changed)
cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
cassandra/trunk/test/unit/org/apache/cassandra/db/CompactionsTest.java
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Dec 20 04:56:59 2010
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6:922689-1050132,1050753-1050754
-/cassandra/branches/cassandra-0.7:1026517-1050755
+/cassandra/branches/cassandra-0.6:922689-1050987
+/cassandra/branches/cassandra-0.7:1026517-1050989
/incubator/cassandra/branches/cassandra-0.3:774578-796573
/incubator/cassandra/branches/cassandra-0.4:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5:888872-915439
Modified: cassandra/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1050991&r1=1050990&r2=1050991&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Mon Dec 20 04:56:59 2010
@@ -21,6 +21,8 @@
* return InvalidRequest when remove of subcolumn without supercolumn
is requested (CASSANDRA-1866)
* flush before repair (CASSANDRA-1748)
+ * SSTableExport validates key order (CASSANDRA-1884)
+ * Re-cache hot keys post-compaction without hitting disk (CASSANDRA-1878)
0.7.0-rc2
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Dec 20 04:56:59 2010
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1050132,1050753-1050754
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026517-1050755
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1050987
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026517-1050989
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Cassandra.java:888872-903502
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Dec 20 04:56:59 2010
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1050132,1050753-1050754
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026517-1050755
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1050987
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026517-1050989
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Column.java:888872-903502
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Dec 20 04:56:59 2010
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1050132,1050753-1050754
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026517-1050755
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1050987
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026517-1050989
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:888872-903502
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Dec 20 04:56:59 2010
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1050132,1050753-1050754
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026517-1050755
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1050987
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026517-1050989
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:888872-903502
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Dec 20 04:56:59 2010
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1050132,1050753-1050754
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026517-1050755
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1050987
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026517-1050989
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:810145-834239,834349-834350
/incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:888872-903502
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java?rev=1050991&r1=1050990&r2=1050991&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java Mon
Dec 20 04:56:59 2010
@@ -305,6 +305,8 @@ public class CompactionManager implement
Iterator<AbstractCompactedRow> nni = new FilterIterator(ci,
PredicateUtils.notNullPredicate());
executor.beginCompaction(cfs, ci);
+ Map<DecoratedKey, Long> cachedKeys = new HashMap<DecoratedKey, Long>();
+
try
{
if (!nni.hasNext())
@@ -321,8 +323,17 @@ public class CompactionManager implement
while (nni.hasNext())
{
AbstractCompactedRow row = nni.next();
- writer.append(row);
+ long position = writer.append(row);
totalkeysWritten++;
+
+ for (SSTableReader sstable : sstables)
+ {
+ if (sstable.getCachedPosition(row.key) != null)
+ {
+ cachedKeys.put(row.key, position);
+ break;
+ }
+ }
}
}
finally
@@ -332,6 +343,8 @@ public class CompactionManager implement
SSTableReader ssTable =
writer.closeAndOpenReader(getMaxDataAge(sstables));
cfs.replaceCompactedSSTables(sstables, Arrays.asList(ssTable));
+ for (Entry<DecoratedKey, Long> entry : cachedKeys.entrySet())
+ ssTable.cacheKey(entry.getKey(), entry.getValue());
submitMinorIfNeeded(cfs);
long dTime = System.currentTimeMillis() - startTime;
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1050991&r1=1050990&r2=1050991&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Mon Dec 20 04:56:59 2010
@@ -413,6 +413,23 @@ public class SSTableReader extends SSTab
return positions;
}
+ public void cacheKey(DecoratedKey key, Long info)
+ {
+ keyCache.put(new Pair<Descriptor, DecoratedKey>(descriptor, key),
info);
+ }
+
+ public Long getCachedPosition(DecoratedKey key)
+ {
+ return getCachedPosition(new Pair<Descriptor,
DecoratedKey>(descriptor, key));
+ }
+
+ private Long getCachedPosition(Pair<Descriptor, DecoratedKey> unifiedKey)
+ {
+ if (keyCache != null && keyCache.getCapacity() > 0)
+ return keyCache.get(unifiedKey);
+ return null;
+ }
+
/**
* @param decoratedKey The key to apply as the rhs to the given Operator.
* @param op The Operator defining matching keys: the nearest key to the
target matching the operator wins.
@@ -426,14 +443,9 @@ public class SSTableReader extends SSTab
// next, the key cache
Pair<Descriptor, DecoratedKey> unifiedKey = new Pair<Descriptor,
DecoratedKey>(descriptor, decoratedKey);
- if (keyCache != null && keyCache.getCapacity() > 0)
- {
- Long cachedPosition = keyCache.get(unifiedKey);
- if (cachedPosition != null)
- {
- return cachedPosition;
- }
- }
+ Long cachedPosition = getCachedPosition(unifiedKey);
+ if (cachedPosition != null)
+ return cachedPosition;
// next, see if the sampled index says it's impossible for the key to
be present
IndexSummary.KeyPosition sampledPosition =
getIndexScanPosition(decoratedKey);
Modified:
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=1050991&r1=1050990&r2=1050991&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
Mon Dec 20 04:56:59 2010
@@ -122,7 +122,7 @@ public class SSTableWriter extends SSTab
dbuilder.addPotentialBoundary(dataPosition);
}
- public void append(AbstractCompactedRow row) throws IOException
+ public long append(AbstractCompactedRow row) throws IOException
{
long currentPosition = beforeAppend(row.key);
FBUtilities.writeShortByteArray(row.key.key, dataFile);
@@ -130,6 +130,7 @@ public class SSTableWriter extends SSTab
estimatedRowSize.add(dataFile.getFilePointer() - currentPosition);
estimatedColumnCount.add(row.columnCount());
afterAppend(row.key, currentPosition);
+ return currentPosition;
}
public void append(DecoratedKey decoratedKey, ColumnFamily cf) throws
IOException
Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=1050991&r1=1050990&r2=1050991&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java Mon
Dec 20 04:56:59 2010
@@ -157,9 +157,16 @@ public class SSTableExport
{
Descriptor desc = Descriptor.fromFilename(ssTableFile);
KeyIterator iter = new KeyIterator(desc);
+ DecoratedKey lastKey = null;
while (iter.hasNext())
{
DecoratedKey key = iter.next();
+
+ // validate order of the keys in the sstable
+ if (lastKey != null && lastKey.compareTo(key) > 0 )
+ throw new IOException("Key out of order! " + lastKey + " > " +
key);
+ lastKey = key;
+
outs.println(bytesToHex(key.key));
}
iter.close();
@@ -187,12 +194,21 @@ public class SSTableExport
excludeSet = new HashSet<String>(Arrays.asList(excludes));
outs.println("{");
+
+ // last key to compare order
+ DecoratedKey lastKey = null;
for (String key : keys)
{
if (excludeSet.contains(key))
continue;
DecoratedKey<?> dk =
partitioner.decorateKey(ByteBuffer.wrap(hexToBytes(key)));
+
+ // validate order of the keys in the sstable
+ if (lastKey != null && lastKey.compareTo(dk) > 0 )
+ throw new IOException("Key out of order! " + lastKey + " > " +
dk);
+ lastKey = dk;
+
scanner.seekTo(dk);
i++;
Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/CompactionsTest.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/CompactionsTest.java?rev=1050991&r1=1050990&r2=1050991&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/CompactionsTest.java
(original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/CompactionsTest.java Mon
Dec 20 04:56:59 2010
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.ArrayList;
import java.util.Set;
import java.util.HashSet;
+import java.util.List;
+import java.util.ArrayList;
import org.apache.cassandra.Util;
