Author: slebresne
Date: Wed Jul 6 12:17:23 2011
New Revision: 1143372
URL: http://svn.apache.org/viewvc?rev=1143372&view=rev
Log:
merge from 0.8
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/CHANGES.txt
cassandra/trunk/contrib/ (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
(props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
(props changed)
cassandra/trunk/src/java/org/apache/cassandra/db/EchoedRow.java
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionController.java
cassandra/trunk/test/unit/org/apache/cassandra/Util.java
cassandra/trunk/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 6 12:17:23 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7:1026516-1140567,1141129,1141213,1141217
/cassandra/branches/cassandra-0.7.0:1053690-1055654
-/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220
+/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220,1143352
/cassandra/branches/cassandra-0.8.0:1125021-1130369
/cassandra/branches/cassandra-0.8.1:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
Modified: cassandra/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1143372&r1=1143371&r2=1143372&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Wed Jul 6 12:17:23 2011
@@ -26,6 +26,7 @@
(CASSANDRA-2823)
* Fix race in SystemTable.getCurrentLocalNodeId (CASSANDRA-2824)
* Correctly set default for replicate_on_write (CASSANDRA-2835)
+ * handle row tombstones correctly in EchoedRow (CASSANDRA-2786)
0.8.1
Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 6 12:17:23 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
/cassandra/branches/cassandra-0.7/contrib:1026516-1140567,1141129,1141213,1141217
/cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
-/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220
+/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220,1143352
/cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369
/cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 6 12:17:23 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1140567,1141129,1141213,1141217
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220,1143352
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 6 12:17:23 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1140567,1141129,1141213,1141217
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220,1143352
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 6 12:17:23 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1140567,1141129,1141213,1141217
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220,1143352
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 6 12:17:23 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1140567,1141129,1141213,1141217
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220,1143352
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
Propchange:
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 6 12:17:23 2011
@@ -1,7 +1,7 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1140567,1141129,1141213,1141217
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1140755,1140760,1141134,1141214,1141220,1143352
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/EchoedRow.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/EchoedRow.java?rev=1143372&r1=1143371&r2=1143372&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/EchoedRow.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/EchoedRow.java Wed Jul 6
12:17:23 2011
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.security.MessageDigest;
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
+import org.apache.cassandra.db.compaction.CompactionController;
import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
/**
@@ -35,11 +36,13 @@ import org.apache.cassandra.io.sstable.S
public class EchoedRow extends AbstractCompactedRow
{
private final SSTableIdentityIterator row;
+ private final int gcBefore;
- public EchoedRow(SSTableIdentityIterator row)
+ public EchoedRow(CompactionController controller, SSTableIdentityIterator
row)
{
super(row.getKey());
this.row = row;
+ this.gcBefore = controller.gcBefore;
// Reset SSTableIdentityIterator because we have not guarantee the
filePointer hasn't moved since the Iterator was built
row.reset();
}
@@ -59,7 +62,7 @@ public class EchoedRow extends AbstractC
public boolean isEmpty()
{
- return !row.hasNext();
+ return !row.hasNext() &&
ColumnFamilyStore.removeDeletedCF(row.getColumnFamily(), gcBefore) == null;
}
public int columnCount()
Modified:
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionController.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionController.java?rev=1143372&r1=1143371&r2=1143372&view=diff
==============================================================================
---
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionController.java
(original)
+++
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionController.java
Wed Jul 6 12:17:23 2011
@@ -113,7 +113,7 @@ public class CompactionController
public AbstractCompactedRow getCompactedRow(List<SSTableIdentityIterator>
rows)
{
if (rows.size() == 1 && !needDeserialize() &&
!shouldPurge(rows.get(0).getKey()))
- return new EchoedRow(rows.get(0));
+ return new EchoedRow(this, rows.get(0));
long rowSize = 0;
for (SSTableIdentityIterator row : rows)
Modified: cassandra/trunk/test/unit/org/apache/cassandra/Util.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/Util.java?rev=1143372&r1=1143371&r2=1143372&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/Util.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/Util.java Wed Jul 6
12:17:23 2011
@@ -32,6 +32,7 @@ import java.util.concurrent.Future;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.CompactionTask;
import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.db.filter.QueryPath;
@@ -200,4 +201,11 @@ public class Util
return CompactionManager.instance.submitUserDefined(cfs, descriptors,
Integer.MAX_VALUE);
}
+ public static void compact(ColumnFamilyStore cfs,
Collection<SSTableReader> sstables, boolean forceDeserialize) throws IOException
+ {
+ CompactionTask task = new CompactionTask(cfs, sstables, (int)
(System.currentTimeMillis() / 1000) - cfs.metadata.getGcGraceSeconds());
+
task.isUserDefined(forceDeserialize).compactionFileLocation(cfs.table.getDataFileLocation(1));
+ task.execute(null);
+ }
+
}
Modified:
cassandra/trunk/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
URL:
http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java?rev=1143372&r1=1143371&r2=1143372&view=diff
==============================================================================
---
cassandra/trunk/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
(original)
+++
cassandra/trunk/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
Wed Jul 6 12:17:23 2011
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Collection;
+import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -116,27 +117,54 @@ public class CompactionsTest extends Cle
if (i % 2 == 0)
store.forceBlockingFlush();
}
+ Collection<SSTableReader> toCompact = store.getSSTables();
- // Force compaction. Since each row is in only one sstable, we will be
using EchoedRow.
- CompactionManager.instance.performMaximal(store);
+ // Reinserting the same keys. We will compact only the previous
sstable, but we need those new ones
+ // to make sure we use EchoedRow, otherwise it won't be used because
purge can be done.
+ for (int i=1; i < 5; i++)
+ {
+ DecoratedKey key = Util.dk(String.valueOf(i));
+ RowMutation rm = new RowMutation(TABLE1, key.key);
+ rm.add(new QueryPath("Standard2", null,
ByteBufferUtil.bytes(String.valueOf(i))), ByteBufferUtil.EMPTY_BYTE_BUFFER, i);
+ rm.apply();
+ }
+ store.forceBlockingFlush();
+ SSTableReader tmpSSTable = null;
+ for (SSTableReader sstable : store.getSSTables())
+ if (!toCompact.contains(sstable))
+ tmpSSTable = sstable;
+
+ // Force compaction on first sstables. Since each row is in only one
sstable, we will be using EchoedRow.
+ Util.compact(store, toCompact, false);
+
+ // Now, we remove the sstable that was just created to force the use
of EchoedRow (so that it doesn't hide the problem)
+ store.markCompacted(Collections.singleton(tmpSSTable));
- // Now assert we do have the two keys
+ // Now assert we do have the 4 keys
assertEquals(4, Util.getRangeSlice(store).size());
}
@Test
public void testDontPurgeAccidentaly() throws IOException,
ExecutionException, InterruptedException
{
+ // Testing with and without forcing deserialization. Without
deserialization, EchoedRow will be used.
+ testDontPurgeAccidentaly("test1", false);
+ testDontPurgeAccidentaly("test2", true);
+ }
+
+ private void testDontPurgeAccidentaly(String k, boolean forceDeserialize)
throws IOException, ExecutionException, InterruptedException
+ {
// This test catches the regression of CASSANDRA-2786
Table table = Table.open(TABLE1);
String cfname = "Super5";
ColumnFamilyStore store = table.getColumnFamilyStore(cfname);
// disable compaction while flushing
+ store.removeAllSSTables();
store.disableAutoCompaction();
// Add test row
- DecoratedKey key = Util.dk("test");
+ DecoratedKey key = Util.dk(k);
RowMutation rm = new RowMutation(TABLE1, key.key);
rm.add(new QueryPath(cfname, ByteBufferUtil.bytes("sc"),
ByteBufferUtil.bytes("c")), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0);
rm.apply();
@@ -145,11 +173,10 @@ public class CompactionsTest extends Cle
Collection<SSTableReader> sstablesBefore = store.getSSTables();
- QueryFilter filter = QueryFilter.getIdentityFilter(Util.dk("test"),
new QueryPath(cfname, null, null));
+ QueryFilter filter = QueryFilter.getIdentityFilter(key, new
QueryPath(cfname, null, null));
assert !store.getColumnFamily(filter).isEmpty();
// Remove key
- key = Util.dk("test");
rm = new RowMutation(TABLE1, key.key);
rm.delete(new QueryPath(cfname, null, null), 2);
rm.apply();
@@ -160,12 +187,12 @@ public class CompactionsTest extends Cle
store.forceBlockingFlush();
Collection<SSTableReader> sstablesAfter = store.getSSTables();
- Collection<Descriptor> toCompact = new ArrayList<Descriptor>();
+ Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>();
for (SSTableReader sstable : sstablesAfter)
if (!sstablesBefore.contains(sstable))
- toCompact.add(sstable.descriptor);
+ toCompact.add(sstable);
- CompactionManager.instance.submitUserDefined(store, toCompact, (int)
(System.currentTimeMillis() / 1000) - store.metadata.getGcGraceSeconds()).get();
+ Util.compact(store, toCompact, forceDeserialize);
cf = store.getColumnFamily(filter);
assert cf.isEmpty() : "should be empty: " + cf;
