svn commit: r1053890 - in /cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra: config/CFMetaData.java db/ColumnFamilyStore.java io/sstable/SSTable.java
Author: gdusbabek
Date: Thu Dec 30 13:52:57 2010
New Revision: 1053890
URL: http://svn.apache.org/viewvc?rev=1053890view=rev
Log:
CFS.scrubDataDirectories should also cleanup invalid secondary indexes. patch
by gdusbabek, reviewed by jbellis. CASSANDRA-1904
Modified:
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/config/CFMetaData.java
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
Modified:
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/config/CFMetaData.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/config/CFMetaData.java?rev=1053890r1=1053889r2=1053890view=diff
==
---
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/config/CFMetaData.java
(original)
+++
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/config/CFMetaData.java
Thu Dec 30 13:52:57 2010
@@ -267,11 +267,11 @@ public final class CFMetaData
nextId(),
column_metadata);
}
-
+
public static CFMetaData newIndexMetadata(String table, String parentCf,
ColumnDefinition info, AbstractType columnComparator)
{
return new CFMetaData(table,
- parentCf + . + (info.getIndexName() == null ?
FBUtilities.bytesToHex(info.name) : info.getIndexName()),
+ indexName(parentCf, info),
ColumnFamilyType.Standard,
columnComparator,
null,
@@ -346,6 +346,12 @@ public final class CFMetaData
{
cfIdMap.remove(new PairString, String(cfm.tableName, cfm.cfName));
}
+
+/** convention for nameing secondary indexes. */
+public static String indexName(String parentCf, ColumnDefinition info)
+{
+return parentCf + . + (info.getIndexName() == null ?
FBUtilities.bytesToHex(info.name) : info.getIndexName());
+}
public org.apache.cassandra.avro.CfDef deflate()
{
Modified:
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1053890r1=1053889r2=1053890view=diff
==
---
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Thu Dec 30 13:52:57 2010
@@ -498,6 +498,14 @@ public class ColumnFamilyStore implement
if (!file.delete())
logger.warn(could not delete +
file.getAbsolutePath());
}
+
+// also clean out any index leftovers.
+CFMetaData cfm = DatabaseDescriptor.getCFMetaData(table, columnFamily);
+if (cfm != null) // secondary indexes aren't stored in DD.
+{
+for (ColumnDefinition def : cfm.getColumn_metadata().values())
+scrubDataDirectories(table, CFMetaData.indexName(cfm.cfName,
def));
+}
}
// must be called after all sstables are loaded since row cache merges all
row versions
Modified:
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1053890r1=1053889r2=1053890view=diff
==
---
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
(original)
+++
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
Thu Dec 30 13:52:57 2010
@@ -230,6 +230,7 @@ public abstract class SSTable
ifile.skipBytes(8);
keys++;
}
+assert keys 0 ifile.getFilePointer() 0 ifile.length() 0;
long estimatedRows = ifile.length() / (ifile.getFilePointer() / keys);
ifile.seek(0);
return estimatedRows;
svn commit: r1053891 - /cassandra/branches/cassandra-0.7.0/CHANGES.txt
Author: gdusbabek Date: Thu Dec 30 13:53:29 2010 New Revision: 1053891 URL: http://svn.apache.org/viewvc?rev=1053891view=rev Log: CFS.scrubDataDirectories should also cleanup invalid secondary indexes. patch by gdusbabek, reviewed by jbellis. CASSANDRA-1904 Modified: cassandra/branches/cassandra-0.7.0/CHANGES.txt Modified: cassandra/branches/cassandra-0.7.0/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7.0/CHANGES.txt?rev=1053891r1=1053890r2=1053891view=diff == --- cassandra/branches/cassandra-0.7.0/CHANGES.txt (original) +++ cassandra/branches/cassandra-0.7.0/CHANGES.txt Thu Dec 30 13:53:29 2010 @@ -15,6 +15,8 @@ dev * change RandomPartitioner min token to -1 to avoid collision w/ tokens on actual nodes (CASSANDRA-1901) * examine the right nibble when validating TimeUUID (CASSANDRA-1910) + * CFS.scrubDataDirectories should also cleanup invalid secondary indexes + (CASSANDRA-1904) 0.7.0-rc3
svn commit: r1053893 - in /cassandra/branches/cassandra-0.7: ./ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/ src/java/
Author: gdusbabek Date: Thu Dec 30 13:55:14 2010 New Revision: 1053893 URL: http://svn.apache.org/viewvc?rev=1053893view=rev Log: merge from 0.7.0 Modified: cassandra/branches/cassandra-0.7/ (props changed) cassandra/branches/cassandra-0.7/CHANGES.txt cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/config/CFMetaData.java cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTable.java Propchange: cassandra/branches/cassandra-0.7/ -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 13:55:14 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6:922689-1053763 /cassandra/branches/cassandra-0.7:1026516,1035666,1050269 -/cassandra/branches/cassandra-0.7.0:1053690-1053764 +/cassandra/branches/cassandra-0.7.0:1053690-1053891 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689 /cassandra/trunk:1026516-1026734,1028929 /incubator/cassandra/branches/cassandra-0.3:774578-796573 Modified: cassandra/branches/cassandra-0.7/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/CHANGES.txt?rev=1053893r1=1053892r2=1053893view=diff == --- cassandra/branches/cassandra-0.7/CHANGES.txt (original) +++ cassandra/branches/cassandra-0.7/CHANGES.txt Thu Dec 30 13:55:14 2010 @@ -22,6 +22,8 @@ dev * change RandomPartitioner min token to -1 to avoid collision w/ tokens on actual nodes (CASSANDRA-1901) * examine the right nibble when validating TimeUUID (CASSANDRA-1910) + * CFS.scrubDataDirectories should also cleanup invalid secondary indexes + (CASSANDRA-1904) 0.7.0-rc3 Propchange: cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 13:55:14 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1053763 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516,1035666,1050269 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053764 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053891 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689 /cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1026734,1028929 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573 Propchange: cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 13:55:14 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1053763 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516,1035666,1050269 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053764 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053891 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689 /cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1026734,1028929 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198 Propchange: cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30
svn commit: r1053894 - in /cassandra/trunk: ./ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassan
Author: gdusbabek Date: Thu Dec 30 13:56:25 2010 New Revision: 1053894 URL: http://svn.apache.org/viewvc?rev=1053894view=rev Log: merge from 0.7 Modified: cassandra/trunk/ (props changed) cassandra/trunk/CHANGES.txt cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java Propchange: cassandra/trunk/ -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 13:56:25 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7:1026516-1053765 -/cassandra/branches/cassandra-0.7.0:1053690-1053764 +/cassandra/branches/cassandra-0.7:1026516-1053893 +/cassandra/branches/cassandra-0.7.0:1053690-1053891 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3:774578-796573 /incubator/cassandra/branches/cassandra-0.4:810145-834239,834349-834350 Modified: cassandra/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1053894r1=1053893r2=1053894view=diff == --- cassandra/trunk/CHANGES.txt (original) +++ cassandra/trunk/CHANGES.txt Thu Dec 30 13:56:25 2010 @@ -27,6 +27,8 @@ * change RandomPartitioner min token to -1 to avoid collision w/ tokens on actual nodes (CASSANDRA-1901) * examine the right nibble when validating TimeUUID (CASSANDRA-1910) + * CFS.scrubDataDirectories should also cleanup invalid secondary indexes + (CASSANDRA-1904) 0.7.0-rc3 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 13:56:25 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1053765 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053764 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1053893 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053891 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-834239,834349-834350 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 13:56:25 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1053765 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053764 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1053893 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053891 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-834239,834349-834350 Propchange:
[jira] Updated: (CASSANDRA-1916) Cleanup needs to remove secondary index entries
[ https://issues.apache.org/jira/browse/CASSANDRA-1916?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] T Jake Luciani updated CASSANDRA-1916: -- Attachment: 0003-add-unit-test.txt Added test case for this change. It verifies the tombstones are added on cleanup Cleanup needs to remove secondary index entries --- Key: CASSANDRA-1916 URL: https://issues.apache.org/jira/browse/CASSANDRA-1916 Project: Cassandra Issue Type: Bug Components: Core Reporter: Jonathan Ellis Assignee: Jonathan Ellis Fix For: 0.7.1 Attachments: 0001-merge-doCleanup-doAntiCompaction.txt, 0002-replace-AntiCompactionIterator-w-per-sstable-iteration.txt, 0003-add-unit-test.txt -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Resolved: (CASSANDRA-1452) Move applyMigrationOnStage to a static method in Migration
[ https://issues.apache.org/jira/browse/CASSANDRA-1452?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Gary Dusbabek resolved CASSANDRA-1452. -- Resolution: Not A Problem I can't remember why I wanted to do this. Move applyMigrationOnStage to a static method in Migration -- Key: CASSANDRA-1452 URL: https://issues.apache.org/jira/browse/CASSANDRA-1452 Project: Cassandra Issue Type: Improvement Components: Core Affects Versions: 0.7 beta 1 Reporter: Gary Dusbabek Assignee: Gary Dusbabek Priority: Trivial This is so avro methods can use it. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Commented: (CASSANDRA-1916) Cleanup needs to remove secondary index entries
[ https://issues.apache.org/jira/browse/CASSANDRA-1916?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976044#action_12976044 ] Jonathan Ellis commented on CASSANDRA-1916: --- simplified getRing in test to return StorageService.instance.getTokenMetadata(); :) committed w/ addition of post-cleanup flush of 2ary indexes Cleanup needs to remove secondary index entries --- Key: CASSANDRA-1916 URL: https://issues.apache.org/jira/browse/CASSANDRA-1916 Project: Cassandra Issue Type: Bug Components: Core Reporter: Jonathan Ellis Assignee: Jonathan Ellis Fix For: 0.7.1 Attachments: 0001-merge-doCleanup-doAntiCompaction.txt, 0002-replace-AntiCompactionIterator-w-per-sstable-iteration.txt, 0003-add-unit-test.txt -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Created: (CASSANDRA-1921) Add jmx password authentication to nodetool
Add jmx password authentication to nodetool --- Key: CASSANDRA-1921 URL: https://issues.apache.org/jira/browse/CASSANDRA-1921 Project: Cassandra Issue Type: Improvement Components: Tools Affects Versions: 0.7.0 rc 3 Reporter: Mck SembWever Assignee: Mck SembWever Fix For: 0.7.1 patch soon... -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1921) Add jmx password authentication to nodetool
[ https://issues.apache.org/jira/browse/CASSANDRA-1921?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Mck SembWever updated CASSANDRA-1921: - Attachment: CASSANDRA-1921.patch Add jmx password authentication to nodetool --- Key: CASSANDRA-1921 URL: https://issues.apache.org/jira/browse/CASSANDRA-1921 Project: Cassandra Issue Type: Improvement Components: Tools Affects Versions: 0.7.0 rc 3 Reporter: Mck SembWever Assignee: Mck SembWever Fix For: 0.7.1 Attachments: CASSANDRA-1921.patch patch soon... -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1921) Add jmx password authentication to nodetool
[ https://issues.apache.org/jira/browse/CASSANDRA-1921?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Mck SembWever updated CASSANDRA-1921: - Reviewer: jbellis Add jmx password authentication to nodetool --- Key: CASSANDRA-1921 URL: https://issues.apache.org/jira/browse/CASSANDRA-1921 Project: Cassandra Issue Type: Improvement Components: Tools Affects Versions: 0.7.0 rc 3 Reporter: Mck SembWever Assignee: Mck SembWever Fix For: 0.7.1 Attachments: CASSANDRA-1921.patch patch soon... -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
svn commit: r1053922 - in /cassandra/branches/cassandra-0.7.0: ./ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/ test/unit/org/apache/cassandra/db/
Author: jbellis
Date: Thu Dec 30 16:00:23 2010
New Revision: 1053922
URL: http://svn.apache.org/viewvc?rev=1053922view=rev
Log:
include secondary indexes in cleanup
patch by jbellis and tjake for CASSANDRA-1916
Added:
cassandra/branches/cassandra-0.7.0/test/unit/org/apache/cassandra/db/CleanupTest.java
Modified:
cassandra/branches/cassandra-0.7.0/CHANGES.txt
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/CompactionManager.java
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/Table.java
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/io/CompactionIterator.java
Modified: cassandra/branches/cassandra-0.7.0/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7.0/CHANGES.txt?rev=1053922r1=1053921r2=1053922view=diff
==
--- cassandra/branches/cassandra-0.7.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.7.0/CHANGES.txt Thu Dec 30 16:00:23 2010
@@ -15,6 +15,7 @@ dev
* change RandomPartitioner min token to -1 to avoid collision w/
tokens on actual nodes (CASSANDRA-1901)
* examine the right nibble when validating TimeUUID (CASSANDRA-1910)
+ * include secondary indexes in cleanup (CASSANDRA-1916)
* CFS.scrubDataDirectories should also cleanup invalid secondary indexes
(CASSANDRA-1904)
Modified:
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/CompactionManager.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/CompactionManager.java?rev=1053922r1=1053921r2=1053922view=diff
==
---
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/CompactionManager.java
(original)
+++
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/CompactionManager.java
Thu Dec 30 16:00:23 2010
@@ -18,10 +18,12 @@
package org.apache.cassandra.db;
+import java.io.DataOutput;
import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
-import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.*;
@@ -31,7 +33,6 @@ import javax.management.MBeanServer;
import javax.management.ObjectName;
import org.apache.commons.collections.PredicateUtils;
-import org.apache.commons.collections.iterators.CollatingIterator;
import org.apache.commons.collections.iterators.FilterIterator;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -47,7 +48,6 @@ import org.apache.cassandra.io.sstable.*
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.service.AntiEntropyService;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.cliffc.high_scale_lib.NonBlockingHashMap;
@@ -368,96 +368,109 @@ public class CompactionManager implement
}
/**
- * This function is used to do the anti compaction process , it spits out
the file which has keys that belong to a given range
- * If the target is not specified it spits out the file as a compacted
file with the unecessary ranges wiped out.
+ * This function goes over each file and removes the keys that the node is
not responsible for
+ * and only keeps keys that this node is responsible for.
*
- * @param cfs
- * @param sstables
- * @param ranges
- * @param target
- * @return
- * @throws java.io.IOException
+ * @throws IOException
*/
-private ListSSTableReader doAntiCompaction(ColumnFamilyStore cfs,
CollectionSSTableReader sstables, CollectionRange ranges, InetAddress
target)
-throws IOException
+private void doCleanupCompaction(ColumnFamilyStore cfs) throws IOException
{
+assert !cfs.isIndex();
Table table = cfs.table;
-logger.info(AntiCompacting [ + StringUtils.join(sstables, ,) +
]);
-// Calculate the expected compacted filesize
-long expectedRangeFileSize =
cfs.getExpectedCompactedFileSize(sstables) / 2;
-String compactionFileLocation =
table.getDataFileLocation(expectedRangeFileSize);
-if (compactionFileLocation == null)
-{
-throw new UnsupportedOperationException(disk full);
-}
+CollectionRange ranges =
StorageService.instance.getLocalRanges(table.name);
-ListSSTableReader results = new ArrayListSSTableReader();
-long startTime = System.currentTimeMillis();
-long totalkeysWritten = 0;
-
-int expectedBloomFilterSize =
Math.max(DatabaseDescriptor.getIndexInterval(),
(int)(SSTableReader.getApproximateKeyCount(sstables) / 2));
-if (logger.isDebugEnabled())
- logger.debug(Expected bloom
svn commit: r1053927 - in /cassandra/branches/cassandra-0.7: ./ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/ test/unit/org
Author: jbellis Date: Thu Dec 30 16:18:33 2010 New Revision: 1053927 URL: http://svn.apache.org/viewvc?rev=1053927view=rev Log: merge from 0.7.0 Added: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/CleanupTest.java Modified: cassandra/branches/cassandra-0.7/ (props changed) cassandra/branches/cassandra-0.7/CHANGES.txt cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/CompactionManager.java cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Table.java cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/CompactionIterator.java Propchange: cassandra/branches/cassandra-0.7/ -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 16:18:33 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6:922689-1053763 /cassandra/branches/cassandra-0.7:1026516,1035666,1050269 -/cassandra/branches/cassandra-0.7.0:1053690-1053891 +/cassandra/branches/cassandra-0.7.0:1053690-1053891,1053922 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689 /cassandra/trunk:1026516-1026734,1028929 /incubator/cassandra/branches/cassandra-0.3:774578-796573 Modified: cassandra/branches/cassandra-0.7/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/CHANGES.txt?rev=1053927r1=1053926r2=1053927view=diff == --- cassandra/branches/cassandra-0.7/CHANGES.txt (original) +++ cassandra/branches/cassandra-0.7/CHANGES.txt Thu Dec 30 16:18:33 2010 @@ -22,6 +22,7 @@ dev * change RandomPartitioner min token to -1 to avoid collision w/ tokens on actual nodes (CASSANDRA-1901) * examine the right nibble when validating TimeUUID (CASSANDRA-1910) + * include secondary indexes in cleanup (CASSANDRA-1916) * CFS.scrubDataDirectories should also cleanup invalid secondary indexes (CASSANDRA-1904) Propchange: cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 16:18:33 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1053763 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516,1035666,1050269 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053891 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053891,1053922 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689 /cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1026734,1028929 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573 Propchange: cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 16:18:33 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1053763 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516,1035666,1050269 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053891 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053891,1053922 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689 /cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1026734,1028929 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198 Propchange:
svn commit: r1053931 - /cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/CompactionManager.java
Author: jbellis
Date: Thu Dec 30 16:31:08 2010
New Revision: 1053931
URL: http://svn.apache.org/viewvc?rev=1053931view=rev
Log:
s/anticompacting/cleaning up/ in log messages
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/CompactionManager.java
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/CompactionManager.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/CompactionManager.java?rev=1053931r1=1053930r2=1053931view=diff
==
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/CompactionManager.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/CompactionManager.java
Thu Dec 30 16:31:08 2010
@@ -381,7 +381,7 @@ public class CompactionManager implement
for (SSTableReader sstable : cfs.getSSTables())
{
-logger.info(AntiCompacting + sstable);
+logger.info(Cleaning up + sstable);
// Calculate the expected compacted filesize
long expectedRangeFileSize =
cfs.getExpectedCompactedFileSize(Arrays.asList(sstable)) / 2;
String compactionFileLocation =
table.getDataFileLocation(expectedRangeFileSize);
@@ -433,7 +433,7 @@ public class CompactionManager implement
SSTableReader newSstable =
writer.closeAndOpenReader(sstable.maxDataAge);
results.add(newSstable);
-String format = AntiCompacted to %s. %,d to %,d (~%d%% of
original) bytes for %,d keys. Time: %,dms.;
+String format = Cleaned up to %s. %,d to %,d (~%d%% of
original) bytes for %,d keys. Time: %,dms.;
long dTime = System.currentTimeMillis() - startTime;
long startsize = sstable.length();
long endsize = newSstable.length();
svn commit: r1053933 - in /cassandra/trunk: ./ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/ test/unit/org/apache/cassandra
Author: jbellis Date: Thu Dec 30 16:33:17 2010 New Revision: 1053933 URL: http://svn.apache.org/viewvc?rev=1053933view=rev Log: merge from 0.7 Added: cassandra/trunk/test/unit/org/apache/cassandra/db/CleanupTest.java Modified: cassandra/trunk/ (props changed) cassandra/trunk/CHANGES.txt cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java cassandra/trunk/src/java/org/apache/cassandra/db/Table.java cassandra/trunk/src/java/org/apache/cassandra/io/CompactionIterator.java Propchange: cassandra/trunk/ -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 16:33:17 2010 @@ -1,5 +1,5 @@ /cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7:1026516-1053893 +/cassandra/branches/cassandra-0.7:1026516-1053893,1053928,1053931 /cassandra/branches/cassandra-0.7.0:1053690-1053891 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3:774578-796573 Modified: cassandra/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1053933r1=1053932r2=1053933view=diff == --- cassandra/trunk/CHANGES.txt (original) +++ cassandra/trunk/CHANGES.txt Thu Dec 30 16:33:17 2010 @@ -27,6 +27,7 @@ * change RandomPartitioner min token to -1 to avoid collision w/ tokens on actual nodes (CASSANDRA-1901) * examine the right nibble when validating TimeUUID (CASSANDRA-1910) + * include secondary indexes in cleanup (CASSANDRA-1916) * CFS.scrubDataDirectories should also cleanup invalid secondary indexes (CASSANDRA-1904) Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 16:33:17 2010 @@ -1,5 +1,5 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1053893 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1053893,1053928,1053931 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053891 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 16:33:17 2010 @@ -1,5 +1,5 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1053893 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1053893,1053928,1053931 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053891 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 16:33:17 2010 @@ -1,5 +1,5 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1053763
[jira] Created: (CASSANDRA-1922) exceptions after cleanup
exceptions after cleanup
Key: CASSANDRA-1922
URL: https://issues.apache.org/jira/browse/CASSANDRA-1922
Project: Cassandra
Issue Type: Bug
Affects Versions: 0.7.0
Reporter: Brandon Williams
Fix For: 0.7.0
It looks like CASSANDRA-1916 may have introduced a regression. After running a
cleanup, I get the following exception when trying to read:
{noformat}
ERROR 17:25:23,574 Fatal exception in thread Thread[ReadStage:99,5,main]
java.lang.AssertionError: skipping negative bytes is illegal: -1393754107
at
org.apache.cassandra.io.util.MappedFileDataInput.skipBytes(MappedFileDataInput.java:96)
at
org.apache.cassandra.io.sstable.IndexHelper.skipBloomFilter(IndexHelper.java:50)
at
org.apache.cassandra.db.columniterator.SimpleSliceReader.init(SimpleSliceReader.java:56)
at
org.apache.cassandra.db.columniterator.SSTableSliceIterator.createReader(SSTableSliceIterator.java:91)
at
org.apache.cassandra.db.columniterator.SSTableSliceIterator.init(SSTableSliceIterator.java:67)
at
org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:68)
at
org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:80)
at
org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1215)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1107)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1077)
at org.apache.cassandra.db.Table.getRow(Table.java:384)
at
org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:63)
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:68)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:63)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
{noformat}
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1922) exceptions after cleanup
[
https://issues.apache.org/jira/browse/CASSANDRA-1922?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Jonathan Ellis updated CASSANDRA-1922:
--
Attachment: 1922.txt
Patch includes test to reproduce the problem and a (one-line) fix in
CompactionManager. Also includes comments on AbstractCompactionIterator as to
what each method is supposed to include, to make it easier to avoid this
mistake in the future.
exceptions after cleanup
Key: CASSANDRA-1922
URL: https://issues.apache.org/jira/browse/CASSANDRA-1922
Project: Cassandra
Issue Type: Bug
Affects Versions: 0.7.0
Reporter: Brandon Williams
Fix For: 0.7.0
Attachments: 1922.txt
It looks like CASSANDRA-1916 may have introduced a regression. After running
a cleanup, I get the following exception when trying to read:
{noformat}
ERROR 17:25:23,574 Fatal exception in thread Thread[ReadStage:99,5,main]
java.lang.AssertionError: skipping negative bytes is illegal: -1393754107
at
org.apache.cassandra.io.util.MappedFileDataInput.skipBytes(MappedFileDataInput.java:96)
at
org.apache.cassandra.io.sstable.IndexHelper.skipBloomFilter(IndexHelper.java:50)
at
org.apache.cassandra.db.columniterator.SimpleSliceReader.init(SimpleSliceReader.java:56)
at
org.apache.cassandra.db.columniterator.SSTableSliceIterator.createReader(SSTableSliceIterator.java:91)
at
org.apache.cassandra.db.columniterator.SSTableSliceIterator.init(SSTableSliceIterator.java:67)
at
org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:68)
at
org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:80)
at
org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1215)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1107)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1077)
at org.apache.cassandra.db.Table.getRow(Table.java:384)
at
org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:63)
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:68)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:63)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
{noformat}
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
svn commit: r1053960 - in /cassandra/branches: cassandra-0.7.0/src/java/org/apache/cassandra/db/ cassandra-0.7.0/src/java/org/apache/cassandra/io/ cassandra-0.7.0/test/unit/org/apache/cassandra/db/ ca
Author: brandonwilliams
Date: Thu Dec 30 18:20:37 2010
New Revision: 1053960
URL: http://svn.apache.org/viewvc?rev=1053960view=rev
Log:
Fix CompactionManager regression from CASSANDRA-1916 and add a better
test and more docs to prevent in the future.
Patch by jbellis, reviewed by brandonwilliams for CASSANDRA-1922
Modified:
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/CompactionManager.java
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/io/AbstractCompactedRow.java
cassandra/branches/cassandra-0.7.0/test/unit/org/apache/cassandra/db/CleanupTest.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/CompactionManager.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/AbstractCompactedRow.java
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/CleanupTest.java
Modified:
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/CompactionManager.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/CompactionManager.java?rev=1053960r1=1053959r2=1053960view=diff
==
---
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/CompactionManager.java
(original)
+++
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/db/CompactionManager.java
Thu Dec 30 18:20:37 2010
@@ -798,6 +798,7 @@ public class CompactionManager implement
public void write(DataOutput out) throws IOException
{
+out.writeLong(row.dataSize);
row.echoData(out);
}
Modified:
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/io/AbstractCompactedRow.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/io/AbstractCompactedRow.java?rev=1053960r1=1053959r2=1053960view=diff
==
---
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/io/AbstractCompactedRow.java
(original)
+++
cassandra/branches/cassandra-0.7.0/src/java/org/apache/cassandra/io/AbstractCompactedRow.java
Thu Dec 30 18:20:37 2010
@@ -41,11 +41,23 @@ public abstract class AbstractCompactedR
this.key = key;
}
+/**
+ * write the row (size + column index + filter + column data, but NOT row
key) to @param out
+ */
public abstract void write(DataOutput out) throws IOException;
-
+
+/**
+ * update @param digest with the data bytes of the row (not including row
key or row size)
+ */
public abstract void update(MessageDigest digest);
+/**
+ * @return true if there are no columns in the row AND there are no
row-level tombstones to be preserved
+ */
public abstract boolean isEmpty();
+/**
+ * @return the number of columns in the row
+ */
public abstract int columnCount();
}
Modified:
cassandra/branches/cassandra-0.7.0/test/unit/org/apache/cassandra/db/CleanupTest.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7.0/test/unit/org/apache/cassandra/db/CleanupTest.java?rev=1053960r1=1053959r2=1053960view=diff
==
---
cassandra/branches/cassandra-0.7.0/test/unit/org/apache/cassandra/db/CleanupTest.java
(original)
+++
cassandra/branches/cassandra-0.7.0/test/unit/org/apache/cassandra/db/CleanupTest.java
Thu Dec 30 18:20:37 2010
@@ -33,6 +33,7 @@ import org.junit.Test;
import org.apache.cassandra.CleanupHelper;
import org.apache.cassandra.Util;
+import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
import org.apache.cassandra.db.filter.IFilter;
import org.apache.cassandra.db.filter.QueryFilter;
@@ -49,9 +50,10 @@ import org.apache.cassandra.utils.FBUtil
public class CleanupTest extends CleanupHelper
{
-public static final int LOOPS = 800;
+public static final int LOOPS = 200;
public static final String TABLE1 = Keyspace1;
public static final String CF1 = Indexed1;
+public static final String CF2 = Standard1;
public static final ByteBuffer COLUMN =
ByteBuffer.wrap(birthdate.getBytes());
public static final ByteBuffer VALUE = ByteBuffer.allocate(8);
static
@@ -61,34 +63,59 @@ public class CleanupTest extends Cleanup
}
@Test
-public void testCleanup() throws IOException, ExecutionException,
InterruptedException
+public void testCleanup() throws IOException, ExecutionException,
InterruptedException, ConfigurationException
{
+StorageService.instance.initServer();
+
Table table = Table.open(TABLE1);
+ColumnFamilyStore cfs = table.getColumnFamilyStore(CF2);
-ColumnFamilyStore cfs = table.getColumnFamilyStore(CF1);
+ListRow rows;
+
[jira] Resolved: (CASSANDRA-1922) exceptions after cleanup
[
https://issues.apache.org/jira/browse/CASSANDRA-1922?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Brandon Williams resolved CASSANDRA-1922.
-
Resolution: Fixed
Committed with unused import removed.
exceptions after cleanup
Key: CASSANDRA-1922
URL: https://issues.apache.org/jira/browse/CASSANDRA-1922
Project: Cassandra
Issue Type: Bug
Affects Versions: 0.7.0
Reporter: Brandon Williams
Fix For: 0.7.0
Attachments: 1922.txt
It looks like CASSANDRA-1916 may have introduced a regression. After running
a cleanup, I get the following exception when trying to read:
{noformat}
ERROR 17:25:23,574 Fatal exception in thread Thread[ReadStage:99,5,main]
java.lang.AssertionError: skipping negative bytes is illegal: -1393754107
at
org.apache.cassandra.io.util.MappedFileDataInput.skipBytes(MappedFileDataInput.java:96)
at
org.apache.cassandra.io.sstable.IndexHelper.skipBloomFilter(IndexHelper.java:50)
at
org.apache.cassandra.db.columniterator.SimpleSliceReader.init(SimpleSliceReader.java:56)
at
org.apache.cassandra.db.columniterator.SSTableSliceIterator.createReader(SSTableSliceIterator.java:91)
at
org.apache.cassandra.db.columniterator.SSTableSliceIterator.init(SSTableSliceIterator.java:67)
at
org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:68)
at
org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:80)
at
org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1215)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1107)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1077)
at org.apache.cassandra.db.Table.getRow(Table.java:384)
at
org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:63)
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:68)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:63)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
{noformat}
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
svn commit: r1053964 - in /cassandra/branches/cassandra-0.7: ./ interface/thrift/gen-java/org/apache/cassandra/thrift/
Author: jbellis Date: Thu Dec 30 18:22:30 2010 New Revision: 1053964 URL: http://svn.apache.org/viewvc?rev=1053964view=rev Log: merge from 0.7.0 Modified: cassandra/branches/cassandra-0.7/ (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) Propchange: cassandra/branches/cassandra-0.7/ -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 18:22:30 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6:922689-1053763 /cassandra/branches/cassandra-0.7:1026516,1035666,1050269 -/cassandra/branches/cassandra-0.7.0:1053690-1053891,1053922 +/cassandra/branches/cassandra-0.7.0:1053690-1053962 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689 /cassandra/trunk:1026516-1026734,1028929 /incubator/cassandra/branches/cassandra-0.3:774578-796573 Propchange: cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 18:22:30 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1053763 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516,1035666,1050269 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053891,1053922 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053962 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689 /cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1026734,1028929 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573 Propchange: cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 18:22:30 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1053763 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516,1035666,1050269 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053891,1053922 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053962 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689 /cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1026734,1028929 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198 Propchange: cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 18:22:30 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1053763 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516,1035666,1050269 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1053891,1053922 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1053962 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689 /cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1026734,1028929 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573 Propchange:
svn commit: r1053968 - in /cassandra/trunk: ./ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/db/ test/unit/org/apache/cassandra/db/
Author: jbellis Date: Thu Dec 30 18:33:52 2010 New Revision: 1053968 URL: http://svn.apache.org/viewvc?rev=1053968view=rev Log: merge from 0.7 Modified: cassandra/trunk/ (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java cassandra/trunk/test/unit/org/apache/cassandra/db/CleanupTest.java Propchange: cassandra/trunk/ -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 18:33:52 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7:1026516-1053893,1053928,1053931 -/cassandra/branches/cassandra-0.7.0:1053690-1053891 +/cassandra/branches/cassandra-0.7:1026516-1053928,1053931 +/cassandra/branches/cassandra-0.7.0:1053690-1053891,1053922 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3:774578-796573 /incubator/cassandra/branches/cassandra-0.4:810145-834239,834349-834350 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 18:33:52 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1053893,1053928,1053931 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053891 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1053928,1053931 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053891,1053922 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-834239,834349-834350 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 18:33:52 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1053893,1053928,1053931 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053891 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1053928,1053931 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053891,1053922 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-834239,834349-834350 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 18:33:52 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1053893,1053928,1053931 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1053891
svn commit: r1053969 - in /cassandra/trunk: ./ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/io/
Author: jbellis Date: Thu Dec 30 18:34:26 2010 New Revision: 1053969 URL: http://svn.apache.org/viewvc?rev=1053969view=rev Log: merge from 0.7 Modified: cassandra/trunk/ (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/trunk/src/java/org/apache/cassandra/io/AbstractCompactedRow.java Propchange: cassandra/trunk/ -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 18:34:26 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7:1026516-1053928,1053931 -/cassandra/branches/cassandra-0.7.0:1053690-1053891,1053922 +/cassandra/branches/cassandra-0.7:1026516-1053968 +/cassandra/branches/cassandra-0.7.0:1053690-1053962 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3:774578-796573 /incubator/cassandra/branches/cassandra-0.4:810145-834239,834349-834350 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 18:34:26 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1053928,1053931 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053891,1053922 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1053968 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1053962 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-834239,834349-834350 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 18:34:26 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1053928,1053931 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053891,1053922 +/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1053968 +/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1053962 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-834239,834349-834350 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java -- --- svn:mergeinfo (original) +++ svn:mergeinfo Thu Dec 30 18:34:26 2010 @@ -1,6 +1,6 @@ /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1053763 -/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1053928,1053931 -/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1053891,1053922
svn commit: r1053975 - in /cassandra/branches/cassandra-0.7.0: CHANGES.txt build.xml debian/changelog
Author: eevans
Date: Thu Dec 30 19:00:10 2010
New Revision: 1053975
URL: http://svn.apache.org/viewvc?rev=1053975view=rev
Log:
update versioning for 7.0 rc4 release
Modified:
cassandra/branches/cassandra-0.7.0/CHANGES.txt
cassandra/branches/cassandra-0.7.0/build.xml
cassandra/branches/cassandra-0.7.0/debian/changelog
Modified: cassandra/branches/cassandra-0.7.0/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7.0/CHANGES.txt?rev=1053975r1=1053974r2=1053975view=diff
==
--- cassandra/branches/cassandra-0.7.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.7.0/CHANGES.txt Thu Dec 30 19:00:10 2010
@@ -1,4 +1,4 @@
-dev
+0.7.0-rc4
* fix cli crash after backgrounding (CASSANDRA-1875)
* count timeouts in storageproxy latencies, and include latency
histograms in StorageProxyMBean (CASSANDRA-1893)
Modified: cassandra/branches/cassandra-0.7.0/build.xml
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7.0/build.xml?rev=1053975r1=1053974r2=1053975view=diff
==
--- cassandra/branches/cassandra-0.7.0/build.xml (original)
+++ cassandra/branches/cassandra-0.7.0/build.xml Thu Dec 30 19:00:10 2010
@@ -47,7 +47,7 @@
property name=test.unit.src value=${test.dir}/unit/
property name=test.long.src value=${test.dir}/long/
property name=dist.dir value=${build.dir}/dist/
-property name=base.version value=0.7.0-rc3/
+property name=base.version value=0.7.0-rc4/
condition property=version value=${base.version}
isset property=release/
/condition
Modified: cassandra/branches/cassandra-0.7.0/debian/changelog
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7.0/debian/changelog?rev=1053975r1=1053974r2=1053975view=diff
==
--- cassandra/branches/cassandra-0.7.0/debian/changelog (original)
+++ cassandra/branches/cassandra-0.7.0/debian/changelog Thu Dec 30 19:00:10 2010
@@ -1,3 +1,9 @@
+cassandra (0.7.0~rc4) unstable; urgency=low
+
+ * Release candidate release.
+
+ -- Eric Evans eev...@apache.org Thu, 30 Dec 2010 12:58:55 -0600
+
cassandra (0.7.0~rc3) unstable; urgency=low
* Release candidate release.
[jira] Commented: (CASSANDRA-1922) exceptions after cleanup
[
https://issues.apache.org/jira/browse/CASSANDRA-1922?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976093#action_12976093
]
Hudson commented on CASSANDRA-1922:
---
Integrated in Cassandra-0.7 #137 (See
[https://hudson.apache.org/hudson/job/Cassandra-0.7/137/])
Fix CompactionManager regression from CASSANDRA-1916 and add a better
test and more docs to prevent in the future.
Patch by jbellis, reviewed by brandonwilliams for CASSANDRA-1922
exceptions after cleanup
Key: CASSANDRA-1922
URL: https://issues.apache.org/jira/browse/CASSANDRA-1922
Project: Cassandra
Issue Type: Bug
Affects Versions: 0.7.0
Reporter: Brandon Williams
Fix For: 0.7.0
Attachments: 1922.txt
It looks like CASSANDRA-1916 may have introduced a regression. After running
a cleanup, I get the following exception when trying to read:
{noformat}
ERROR 17:25:23,574 Fatal exception in thread Thread[ReadStage:99,5,main]
java.lang.AssertionError: skipping negative bytes is illegal: -1393754107
at
org.apache.cassandra.io.util.MappedFileDataInput.skipBytes(MappedFileDataInput.java:96)
at
org.apache.cassandra.io.sstable.IndexHelper.skipBloomFilter(IndexHelper.java:50)
at
org.apache.cassandra.db.columniterator.SimpleSliceReader.init(SimpleSliceReader.java:56)
at
org.apache.cassandra.db.columniterator.SSTableSliceIterator.createReader(SSTableSliceIterator.java:91)
at
org.apache.cassandra.db.columniterator.SSTableSliceIterator.init(SSTableSliceIterator.java:67)
at
org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:68)
at
org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:80)
at
org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1215)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1107)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1077)
at org.apache.cassandra.db.Table.getRow(Table.java:384)
at
org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:63)
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:68)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:63)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
{noformat}
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1551) create tell me what nodes you have hints for jmx api
[ https://issues.apache.org/jira/browse/CASSANDRA-1551?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Jon Hermes updated CASSANDRA-1551: -- Attachment: 1551-nuke.txt bq. ability to nuke hints for a given node would also be useful. HHOM already has a nukeHintsFor(InetAddr), so the attached nuke patch exposes it via SS (and takes a string, which is a little nicer). create tell me what nodes you have hints for jmx api -- Key: CASSANDRA-1551 URL: https://issues.apache.org/jira/browse/CASSANDRA-1551 Project: Cassandra Issue Type: New Feature Components: Tools Reporter: Jonathan Ellis Assignee: Jon Hermes Priority: Minor Fix For: 0.7.1 Attachments: 1551-nuke.txt we can do this efficiently in 0.7 due to new HH schema. in 0.6 this would require scanning all hints so probably not worth it. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Created: (CASSANDRA-1923) unit tests that validate that message serialization isn't broken in the current version.
unit tests that validate that message serialization isn't broken in the current version. Key: CASSANDRA-1923 URL: https://issues.apache.org/jira/browse/CASSANDRA-1923 Project: Cassandra Issue Type: Sub-task Components: Core Reporter: Gary Dusbabek Assignee: Gary Dusbabek Priority: Minor Fix For: 0.7.1 There are two components to this. First, code that will generate the serialized messages. Second, code that will attempt to read the serialized messages. My plan is to commit this to 0.7.1 with generated serialized messages. Then I will merge that into trunk sans the generation code. A similar process will need to take place when we branch trunk to create 0.8, etc. On second thought, maybe it makes sense to keep the generation code and let it morph as the message formats change. If the tests ever break in the 0.7 branch, that means we've created a message incompatibility regression that needs to be fixed. If the tests ever break in trunk (post CASSANDRA-1015), it means that something in trunk has changed message serialization compatibility that will need to be restored (via whatever process is used for 1015). -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Commented: (CASSANDRA-1923) unit tests that validate that message serialization isn't broken in the current version.
[ https://issues.apache.org/jira/browse/CASSANDRA-1923?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976107#action_12976107 ] Jonathan Ellis commented on CASSANDRA-1923: --- I posted a (limited) test like this to CASSANDRA-1788, btw. unit tests that validate that message serialization isn't broken in the current version. Key: CASSANDRA-1923 URL: https://issues.apache.org/jira/browse/CASSANDRA-1923 Project: Cassandra Issue Type: Sub-task Components: Core Reporter: Gary Dusbabek Assignee: Gary Dusbabek Priority: Minor Fix For: 0.7.1 There are two components to this. First, code that will generate the serialized messages. Second, code that will attempt to read the serialized messages. My plan is to commit this to 0.7.1 with generated serialized messages. Then I will merge that into trunk sans the generation code. A similar process will need to take place when we branch trunk to create 0.8, etc. On second thought, maybe it makes sense to keep the generation code and let it morph as the message formats change. If the tests ever break in the 0.7 branch, that means we've created a message incompatibility regression that needs to be fixed. If the tests ever break in trunk (post CASSANDRA-1015), it means that something in trunk has changed message serialization compatibility that will need to be restored (via whatever process is used for 1015). -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Commented: (CASSANDRA-1015) Internal Messaging should be backwards compatible
[ https://issues.apache.org/jira/browse/CASSANDRA-1015?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976114#action_12976114 ] Gary Dusbabek commented on CASSANDRA-1015: -- I've swung back around to preferring that we use our own messages and serialization for this. Adopting a framework like thrift or avro would save us from the tedium of writing our own serialization methods, but would also take away control over the serialization, which I don't think we want. Here are a few examples of how I expect messages to change from version to version: 1. a field is added. every framework handles this fine. 2. a field is removed. every framework handles this fine. 3. a field type is changed (name stays the same). every framework handles this different. Thrift requires that we rename the field, which is the same as removing and then adding a field. We'd need to have translation routines to convert from the old field to the new one. Avro handles this a little bit better using a union. But then the avro object type becomes a union which would require ugly casting in our code. Using our own serialization would solve this problem at serialization/deserialization time. Given that a lot of our message format changes are of type 3, I think it makes sense to stick with rolling our own messages. The only debt we incur is maintaining the serialization methods, which hasn't been a substantial burden. Internal Messaging should be backwards compatible - Key: CASSANDRA-1015 URL: https://issues.apache.org/jira/browse/CASSANDRA-1015 Project: Cassandra Issue Type: Improvement Components: Core Reporter: Ryan King Assignee: Gary Dusbabek Priority: Critical Fix For: 0.8 Currently, incompatible changes in the node-to-node communication prevent rolling restarts of clusters. In order to fix this we should: 1) use a framework that makes doing compatible changes easy 2) have a policy of only making compatible changes between versions n and n+1* * Running multiple versions should only be supported for small periods of time. Running clusters of mixed version is not needed here. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Commented: (CASSANDRA-1895) Loadbalance during gossip issues leaves cluster in bad state
[ https://issues.apache.org/jira/browse/CASSANDRA-1895?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976119#action_12976119 ] Nick Bailey commented on CASSANDRA-1895: We have generations for node states. The problem is after 30 seconds we assume gossip has propagated and forget about nodes that have been removed. Then when we see the gossip again it looks like a brand new node. Loadbalance during gossip issues leaves cluster in bad state Key: CASSANDRA-1895 URL: https://issues.apache.org/jira/browse/CASSANDRA-1895 Project: Cassandra Issue Type: Bug Affects Versions: 0.8 Reporter: Stu Hood Fix For: 0.8 Attachments: logs.tgz, ring-views.txt Running loadbalance against a node in a 4 node cluster leaves gossip in a wonky state. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1895) Loadbalance during gossip issues leaves cluster in bad state
[ https://issues.apache.org/jira/browse/CASSANDRA-1895?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Jonathan Ellis updated CASSANDRA-1895: -- Component/s: Core Priority: Minor (was: Major) Affects Version/s: (was: 0.8) Fix Version/s: (was: 0.8) 0.7.1 Assignee: Brandon Williams sounds like the inverse of CASSANDRA-1730 Loadbalance during gossip issues leaves cluster in bad state Key: CASSANDRA-1895 URL: https://issues.apache.org/jira/browse/CASSANDRA-1895 Project: Cassandra Issue Type: Bug Components: Core Reporter: Stu Hood Assignee: Brandon Williams Priority: Minor Fix For: 0.7.1 Attachments: logs.tgz, ring-views.txt Running loadbalance against a node in a 4 node cluster leaves gossip in a wonky state. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
svn commit: r1054009 - /cassandra/branches/cassandra-0.7/CHANGES.txt
Author: jbellis Date: Thu Dec 30 22:12:30 2010 New Revision: 1054009 URL: http://svn.apache.org/viewvc?rev=1054009view=rev Log: split rc4 CHANGES out Modified: cassandra/branches/cassandra-0.7/CHANGES.txt Modified: cassandra/branches/cassandra-0.7/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/CHANGES.txt?rev=1054009r1=1054008r2=1054009view=diff == --- cassandra/branches/cassandra-0.7/CHANGES.txt (original) +++ cassandra/branches/cassandra-0.7/CHANGES.txt Thu Dec 30 22:12:30 2010 @@ -1,15 +1,16 @@ dev - * fix cli crash after backgrounding (CASSANDRA-1875) - * count timeouts in storageproxy latencies, and include latency - histograms in StorageProxyMBean (CASSANDRA-1893) - * check log4j configuration for changes every 10s (CASSANDRA-1525) + * check log4j configuration for changes every 10s (CASSANDRA-1525, 1907) * More-efficient cross-DC replication (CASSANDRA-1530) * upgrade to TFastFramedTransport (CASSANDRA-1743) - * fix CLI get recognition of supercolumns (CASSANDRA-1899) * avoid polluting page cache with commitlog or sstable writes and seq scan operations (CASSANDRA-1470) - * add OpenBitSet to support larger bloom filters (CASSANDRA-1555) - * handle URL-specified log4j regression (CASSANDRA-1907) + + +0.7.0-rc4 + * fix cli crash after backgrounding (CASSANDRA-1875) + * count timeouts in storageproxy latencies, and include latency + histograms in StorageProxyMBean (CASSANDRA-1893) + * fix CLI get recognition of supercolumns (CASSANDRA-1899) * enable keepalive on intra-cluster sockets (CASSANDRA-1766) * count timeouts towards dynamicsnitch latencies (CASSANDRA-1905) * Expose index-building status in JMX + cli schema description
[jira] Commented: (CASSANDRA-1896) Improve throughput by adding buffering to the inter-node TCP communication
[
https://issues.apache.org/jira/browse/CASSANDRA-1896?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976126#action_12976126
]
Tsiki commented on CASSANDRA-1896:
--
Followup on the buffering issue. I think I need to explain better the nature of
the change idea when it comes to writing to the socket.
The main problem with the write method is that it doesn't handle flushing
correctly. Since you disable the Nagle algorithm [ setTcpNoelay(true) ] the
socket is not buffered. Which means, every write() call flushes automatically.
Now, when you write an integer - it sends a packet. If you have small data
retrieved (small column values), you'll have a packet for each. this means a
lot of system and network overhead.
So there are two ways to go about it.
1. Leave the code as is and not disable Nagle. You can tune the socket send
buffer size to get optimal results.
2. Do application level caching which is what I did. My experience goes back a
long time with this. To the time system calls were much more expensive than
they are today, and when copying would be done regardless of the method you
would use. So, like most people, I did what I am used to.
In any case, the current implementation I think is flawed.
Improve throughput by adding buffering to the inter-node TCP communication
--
Key: CASSANDRA-1896
URL: https://issues.apache.org/jira/browse/CASSANDRA-1896
Project: Cassandra
Issue Type: Improvement
Components: Core
Affects Versions: 0.6
Reporter: Tsiki
Assignee: Brandon Williams
Fix For: 0.6.9, 0.7.1
Attachments: 1896.txt
The inbound and outbound TCP implementation under org.apache.cassandra.net
does not buffer the socket streams. A simple change in IncomingTcpConnection
and OutboundTcpConnection may give a rather big throughput increase. In my
tests, I got up t o 30% more out of my cluster. Below is the diff of these
two files with buffering included. The diff is over release 0.6.5 but can be
quite simply applied also to 0.7. I suggest perhaps to limit the buffered
input stream I added in IncomingTcpConnection to 4K. The Outbound
implementation can surely be implemented a bit better (remove some of the
code I duplicated there).
diff -r
apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
fix/apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
44c44
input = new DataInputStream(new
BufferedInputStream(socket.getInputStream()));
---
input = new DataInputStream(socket.getInputStream());
diff -r
apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
fix/apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
77d76
byte[] buf = new byte[4096];
80,112c79,89
int l = 0;
ByteBuffer bb;
while ((bb = queue.peek()) != null l+bb.limit()
buf.length) {
bb = take();
System.arraycopy(bb.array(), 0, buf, l, bb.limit());
l += bb.limit();
}
if (l == 0) {
bb = take();
if (bb == CLOSE_SENTINEL)
{
disconnect();
continue;
}
if (socket != null || connect())
writeConnected(bb);
else
// clear out the queue, else gossip messages back up.
queue.clear();
} else {
if (socket != null || connect()) {
try {
output.write(buf, 0, l);
if (queue.peek() == null)
output.flush();
} catch (IOException e) {
logger.info(error writing to + endpoint);
disconnect();
}
} else {
queue.clear();
}
}
---
ByteBuffer bb = take();
if (bb == CLOSE_SENTINEL)
{
disconnect();
continue;
}
if (socket != null || connect())
writeConnected(bb);
else
// clear out the queue, else gossip messages back up.
queue.clear();
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Commented: (CASSANDRA-1123) Allow tracing query details
[ https://issues.apache.org/jira/browse/CASSANDRA-1123?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976128#action_12976128 ] Jonathan Ellis commented on CASSANDRA-1123: --- still looks like the simplest solution is to add a thrift system_enable_query_details() method that would cause what we log at debug, to be logged at INFO _for queries from that connection_. We'd need to add this to the intra-cluster messages too, either as a part of the Message Header or (more lightweight) as a bit in the header computed by MessagingService.packIt. (Up until it hits MessagingService everything will be run on the same thread, so we can use a threadlocal like we do for keyspace in CassandraServer.) incidentally, i don't think we log anything about how many sstables are checked on a per-query basis (although we collect this globally in a histogram), so that would be good to add. Allow tracing query details --- Key: CASSANDRA-1123 URL: https://issues.apache.org/jira/browse/CASSANDRA-1123 Project: Cassandra Issue Type: New Feature Components: Core Reporter: Jonathan Ellis Assignee: Brandon Williams Fix For: 0.8 In the spirit of CASSANDRA-511, it would be useful to tracing on queries to see where latency is coming from: how long did row cache lookup take? key search in the index? merging the data from the sstables? etc. The main difference vs setting debug logging is that debug logging is too big of a hammer; by turning on the flood of logging for everyone, you actually distort the information you're looking for. This would be something you could set per-query (or more likely per connection). We don't need to be as sophisticated as the techniques discussed in the following papers but they are interesting reading: http://research.google.com/pubs/pub36356.html http://www.usenix.org/events/osdi04/tech/full_papers/barham/barham_html/ http://www.usenix.org/event/nsdi07/tech/fonseca.html -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Assigned: (CASSANDRA-1123) Allow tracing query details
[ https://issues.apache.org/jira/browse/CASSANDRA-1123?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Jonathan Ellis reassigned CASSANDRA-1123: - Assignee: Aaron Morton (was: Brandon Williams) Allow tracing query details --- Key: CASSANDRA-1123 URL: https://issues.apache.org/jira/browse/CASSANDRA-1123 Project: Cassandra Issue Type: New Feature Components: Core Reporter: Jonathan Ellis Assignee: Aaron Morton Fix For: 0.8 In the spirit of CASSANDRA-511, it would be useful to tracing on queries to see where latency is coming from: how long did row cache lookup take? key search in the index? merging the data from the sstables? etc. The main difference vs setting debug logging is that debug logging is too big of a hammer; by turning on the flood of logging for everyone, you actually distort the information you're looking for. This would be something you could set per-query (or more likely per connection). We don't need to be as sophisticated as the techniques discussed in the following papers but they are interesting reading: http://research.google.com/pubs/pub36356.html http://www.usenix.org/events/osdi04/tech/full_papers/barham/barham_html/ http://www.usenix.org/event/nsdi07/tech/fonseca.html -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Commented: (CASSANDRA-1896) Improve throughput by adding buffering to the inter-node TCP communication
[
https://issues.apache.org/jira/browse/CASSANDRA-1896?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976131#action_12976131
]
Jonathan Ellis commented on CASSANDRA-1896:
---
Wouldn't we expect 1) to be better for handling lots of small updates, since 2)
still forces a separate TCP message for each request?
Improve throughput by adding buffering to the inter-node TCP communication
--
Key: CASSANDRA-1896
URL: https://issues.apache.org/jira/browse/CASSANDRA-1896
Project: Cassandra
Issue Type: Improvement
Components: Core
Affects Versions: 0.6
Reporter: Tsiki
Assignee: Brandon Williams
Fix For: 0.6.9, 0.7.1
Attachments: 1896.txt
The inbound and outbound TCP implementation under org.apache.cassandra.net
does not buffer the socket streams. A simple change in IncomingTcpConnection
and OutboundTcpConnection may give a rather big throughput increase. In my
tests, I got up t o 30% more out of my cluster. Below is the diff of these
two files with buffering included. The diff is over release 0.6.5 but can be
quite simply applied also to 0.7. I suggest perhaps to limit the buffered
input stream I added in IncomingTcpConnection to 4K. The Outbound
implementation can surely be implemented a bit better (remove some of the
code I duplicated there).
diff -r
apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
fix/apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
44c44
input = new DataInputStream(new
BufferedInputStream(socket.getInputStream()));
---
input = new DataInputStream(socket.getInputStream());
diff -r
apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
fix/apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
77d76
byte[] buf = new byte[4096];
80,112c79,89
int l = 0;
ByteBuffer bb;
while ((bb = queue.peek()) != null l+bb.limit()
buf.length) {
bb = take();
System.arraycopy(bb.array(), 0, buf, l, bb.limit());
l += bb.limit();
}
if (l == 0) {
bb = take();
if (bb == CLOSE_SENTINEL)
{
disconnect();
continue;
}
if (socket != null || connect())
writeConnected(bb);
else
// clear out the queue, else gossip messages back up.
queue.clear();
} else {
if (socket != null || connect()) {
try {
output.write(buf, 0, l);
if (queue.peek() == null)
output.flush();
} catch (IOException e) {
logger.info(error writing to + endpoint);
disconnect();
}
} else {
queue.clear();
}
}
---
ByteBuffer bb = take();
if (bb == CLOSE_SENTINEL)
{
disconnect();
continue;
}
if (socket != null || connect())
writeConnected(bb);
else
// clear out the queue, else gossip messages back up.
queue.clear();
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Commented: (CASSANDRA-1896) Improve throughput by adding buffering to the inter-node TCP communication
[
https://issues.apache.org/jira/browse/CASSANDRA-1896?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976133#action_12976133
]
Tsiki commented on CASSANDRA-1896:
--
No. Since also the current implementation writes only when the buffer is full
or if there are no additional messages in the queue (hence the peek() call when
filling the buffer). Now if we are under load, which is the interesting case
here, then the queue will always be full since messages are always coming in
and so flushing will take place whenever the (4K or so) buffer is full and
we'll send only large packets.
Improve throughput by adding buffering to the inter-node TCP communication
--
Key: CASSANDRA-1896
URL: https://issues.apache.org/jira/browse/CASSANDRA-1896
Project: Cassandra
Issue Type: Improvement
Components: Core
Affects Versions: 0.6
Reporter: Tsiki
Assignee: Brandon Williams
Fix For: 0.6.9, 0.7.1
Attachments: 1896.txt
The inbound and outbound TCP implementation under org.apache.cassandra.net
does not buffer the socket streams. A simple change in IncomingTcpConnection
and OutboundTcpConnection may give a rather big throughput increase. In my
tests, I got up t o 30% more out of my cluster. Below is the diff of these
two files with buffering included. The diff is over release 0.6.5 but can be
quite simply applied also to 0.7. I suggest perhaps to limit the buffered
input stream I added in IncomingTcpConnection to 4K. The Outbound
implementation can surely be implemented a bit better (remove some of the
code I duplicated there).
diff -r
apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
fix/apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
44c44
input = new DataInputStream(new
BufferedInputStream(socket.getInputStream()));
---
input = new DataInputStream(socket.getInputStream());
diff -r
apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
fix/apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
77d76
byte[] buf = new byte[4096];
80,112c79,89
int l = 0;
ByteBuffer bb;
while ((bb = queue.peek()) != null l+bb.limit()
buf.length) {
bb = take();
System.arraycopy(bb.array(), 0, buf, l, bb.limit());
l += bb.limit();
}
if (l == 0) {
bb = take();
if (bb == CLOSE_SENTINEL)
{
disconnect();
continue;
}
if (socket != null || connect())
writeConnected(bb);
else
// clear out the queue, else gossip messages back up.
queue.clear();
} else {
if (socket != null || connect()) {
try {
output.write(buf, 0, l);
if (queue.peek() == null)
output.flush();
} catch (IOException e) {
logger.info(error writing to + endpoint);
disconnect();
}
} else {
queue.clear();
}
}
---
ByteBuffer bb = take();
if (bb == CLOSE_SENTINEL)
{
disconnect();
continue;
}
if (socket != null || connect())
writeConnected(bb);
else
// clear out the queue, else gossip messages back up.
queue.clear();
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Created: (CASSANDRA-1924) Broken keyspace strategy_option with zero replicas
Broken keyspace strategy_option with zero replicas
--
Key: CASSANDRA-1924
URL: https://issues.apache.org/jira/browse/CASSANDRA-1924
Project: Cassandra
Issue Type: Bug
Components: Core
Affects Versions: 0.7.0 rc 3, 0.7.0 rc 2, 0.7.0 rc 1, 0.7 beta 3, 0.7 beta
2, 0.7 beta 1
Reporter: Thor Carpenter
Priority: Minor
When a keyspace is defined that has strategy options specifying zero replicas
should be place in a datacenter (e.g. DCX:0), an assert is violated for any
insert and LOCAL_QUORUM reads fail. I'm not sure if the issue is that there
are no nodes in DCX or that I'm saying DCX shouldn't get any replicas, or a
combination of the two.
The broken keyspace:
create keyspace KeyspaceDC1 with
replication_factor = 1 and
placement_strategy = 'org.apache.cassandra.locator.NetworkTopologyStrategy'
and
strategy_options = [{DC1:1, DC2:0}];
The fixed keyspace:
create keyspace KeyspaceDC1 with
replication_factor = 1 and
placement_strategy = 'org.apache.cassandra.locator.NetworkTopologyStrategy'
and
strategy_options = [{DC1:1}];
To reproduce:
* Install the 0.7rc3 rpm on a single node in DC1.
* In cassandra.yaml set initial_token = 1 and specify PropertyFileSnitch.
* cassandra-topology.properties:
10.5.64.26=DC1:R1
default=DC2:R1
* Schema loaded via cassandra-cli:
create keyspace KeyspaceDC1 with
replication_factor = 1 and
placement_strategy = 'org.apache.cassandra.locator.NetworkTopologyStrategy'
and
strategy_options = [{DC1:1, DC2:0}];
use KeyspaceDC1;
create column family TestCF with
column_type = 'Standard' and
comparator = 'BytesType' and
keys_cached = 20 and
rows_cached = 2000 and
gc_grace = 0 and
read_repair_chance = 0.0;
* In cassandra-cli execute the following:
[defa...@unknown] use KeyspaceDC1;
Authenticated to keyspace: KeyspaceDC1
[defa...@keyspacedc1] set TestCF['some key']['some col'] = 'some value';
Internal error processing insert
* If you have asserts enabled, check system.log where you should find the
assertion error:
DEBUG [pool-1-thread-3] 2010-12-29 12:10:38,897 CassandraServer.java (line 362)
insert
ERROR [pool-1-thread-3] 2010-12-29 12:10:38,906 Cassandra.java (line 2960)
Internal error processing insert
java.lang.AssertionError
at
org.apache.cassandra.locator.TokenMetadata.firstTokenIndex(TokenMetadata.java:392)
at
org.apache.cassandra.locator.TokenMetadata.ringIterator(TokenMetadata.java:417)
at
org.apache.cassandra.locator.NetworkTopologyStrategy.calculateNaturalEndpoints(NetworkTopologyStrategy.java:95)
at
org.apache.cassandra.locator.AbstractReplicationStrategy.getNaturalEndpoints(AbstractReplicationStrategy.java:99)
at
org.apache.cassandra.service.StorageService.getNaturalEndpoints(StorageService.java:1411)
at
org.apache.cassandra.service.StorageService.getNaturalEndpoints(StorageService.java:1394)
at
org.apache.cassandra.service.StorageProxy.mutate(StorageProxy.java:109)
at
org.apache.cassandra.thrift.CassandraServer.doInsert(CassandraServer.java:442)
at
org.apache.cassandra.thrift.CassandraServer.insert(CassandraServer.java:379)
at
org.apache.cassandra.thrift.Cassandra$Processor$insert.process(Cassandra.java:2952)
at
org.apache.cassandra.thrift.Cassandra$Processor.process(Cassandra.java:2555)
at
org.apache.cassandra.thrift.CustomTThreadPoolServer$WorkerProcess.run(CustomTThreadPoolServer.java:167)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
* If you don't have asserts enabled, you should find that no errors are logged
but LOCAL_QUORUM reads cause TimedOutExceptions on the client.
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Commented: (CASSANDRA-1015) Internal Messaging should be backwards compatible
[ https://issues.apache.org/jira/browse/CASSANDRA-1015?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976157#action_12976157 ] Ryan King commented on CASSANDRA-1015: -- I fear that in going on own way we'll end up replicating a lot of what's already been done in these frameworks. Additionally, we make it much harder to to write code to comprehend the message in another language. I know this sounds like a YAGNI, but I've found it quite nice to be able to decode thrift RPC interchanges that are captured via tcpdump. We have to rebuild a lot if we go our own way. Internal Messaging should be backwards compatible - Key: CASSANDRA-1015 URL: https://issues.apache.org/jira/browse/CASSANDRA-1015 Project: Cassandra Issue Type: Improvement Components: Core Reporter: Ryan King Assignee: Gary Dusbabek Priority: Critical Fix For: 0.8 Currently, incompatible changes in the node-to-node communication prevent rolling restarts of clusters. In order to fix this we should: 1) use a framework that makes doing compatible changes easy 2) have a policy of only making compatible changes between versions n and n+1* * Running multiple versions should only be supported for small periods of time. Running clusters of mixed version is not needed here. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1654) EC2Snitch
[ https://issues.apache.org/jira/browse/CASSANDRA-1654?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Brandon Williams updated CASSANDRA-1654: Attachment: 1654-v2.txt v2 which should actually gossip values around. Untested. EC2Snitch - Key: CASSANDRA-1654 URL: https://issues.apache.org/jira/browse/CASSANDRA-1654 Project: Cassandra Issue Type: New Feature Components: Core Reporter: Jonathan Ellis Assignee: Brandon Williams Fix For: 0.7.1 Attachments: 1654-incomplete.txt, 1654.txt This will be a snitch extending AbstractNetworkTopologySnitch that pulls DC from instance metadata placement_availability_zone. I don't think we can get rack information so the choice is between picking something unique-per-machine to be the rack or assigning everything in the DC the same rack. Second seems easier, so let's use placement_availability_zone for rack, too. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1654) EC2Snitch
[ https://issues.apache.org/jira/browse/CASSANDRA-1654?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Brandon Williams updated CASSANDRA-1654: Attachment: (was: 1654-v2.txt) EC2Snitch - Key: CASSANDRA-1654 URL: https://issues.apache.org/jira/browse/CASSANDRA-1654 Project: Cassandra Issue Type: New Feature Components: Core Reporter: Jonathan Ellis Assignee: Brandon Williams Fix For: 0.7.1 Attachments: 1654-incomplete.txt, 1654.txt This will be a snitch extending AbstractNetworkTopologySnitch that pulls DC from instance metadata placement_availability_zone. I don't think we can get rack information so the choice is between picking something unique-per-machine to be the rack or assigning everything in the DC the same rack. Second seems easier, so let's use placement_availability_zone for rack, too. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1654) EC2Snitch
[ https://issues.apache.org/jira/browse/CASSANDRA-1654?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Brandon Williams updated CASSANDRA-1654: Attachment: 1654-v2.txt EC2Snitch - Key: CASSANDRA-1654 URL: https://issues.apache.org/jira/browse/CASSANDRA-1654 Project: Cassandra Issue Type: New Feature Components: Core Reporter: Jonathan Ellis Assignee: Brandon Williams Fix For: 0.7.1 Attachments: 1654-incomplete.txt, 1654-v2.txt, 1654.txt This will be a snitch extending AbstractNetworkTopologySnitch that pulls DC from instance metadata placement_availability_zone. I don't think we can get rack information so the choice is between picking something unique-per-machine to be the rack or assigning everything in the DC the same rack. Second seems easier, so let's use placement_availability_zone for rack, too. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1654) EC2Snitch
[ https://issues.apache.org/jira/browse/CASSANDRA-1654?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Brandon Williams updated CASSANDRA-1654: Attachment: (was: 1654-v2.txt) EC2Snitch - Key: CASSANDRA-1654 URL: https://issues.apache.org/jira/browse/CASSANDRA-1654 Project: Cassandra Issue Type: New Feature Components: Core Reporter: Jonathan Ellis Assignee: Brandon Williams Fix For: 0.7.1 Attachments: 1654-incomplete.txt, 1654.txt This will be a snitch extending AbstractNetworkTopologySnitch that pulls DC from instance metadata placement_availability_zone. I don't think we can get rack information so the choice is between picking something unique-per-machine to be the rack or assigning everything in the DC the same rack. Second seems easier, so let's use placement_availability_zone for rack, too. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Commented: (CASSANDRA-1015) Internal Messaging should be backwards compatible
[ https://issues.apache.org/jira/browse/CASSANDRA-1015?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976164#action_12976164 ] Jonathan Ellis commented on CASSANDRA-1015: --- How does Thrift/Avro handle a case like ColumnSerializer, which turns a set of bytes into either a Column, a DeletedColumn, or an ExpiredColumn depending on the contents? Internal Messaging should be backwards compatible - Key: CASSANDRA-1015 URL: https://issues.apache.org/jira/browse/CASSANDRA-1015 Project: Cassandra Issue Type: Improvement Components: Core Reporter: Ryan King Assignee: Gary Dusbabek Priority: Critical Fix For: 0.8 Currently, incompatible changes in the node-to-node communication prevent rolling restarts of clusters. In order to fix this we should: 1) use a framework that makes doing compatible changes easy 2) have a policy of only making compatible changes between versions n and n+1* * Running multiple versions should only be supported for small periods of time. Running clusters of mixed version is not needed here. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1654) EC2Snitch
[ https://issues.apache.org/jira/browse/CASSANDRA-1654?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Brandon Williams updated CASSANDRA-1654: Attachment: (was: 1654-v2.txt) EC2Snitch - Key: CASSANDRA-1654 URL: https://issues.apache.org/jira/browse/CASSANDRA-1654 Project: Cassandra Issue Type: New Feature Components: Core Reporter: Jonathan Ellis Assignee: Brandon Williams Fix For: 0.7.1 Attachments: 1654-incomplete.txt, 1654-v2.txt, 1654.txt This will be a snitch extending AbstractNetworkTopologySnitch that pulls DC from instance metadata placement_availability_zone. I don't think we can get rack information so the choice is between picking something unique-per-machine to be the rack or assigning everything in the DC the same rack. Second seems easier, so let's use placement_availability_zone for rack, too. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1859) distributed test harness
[
https://issues.apache.org/jira/browse/CASSANDRA-1859?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Stu Hood updated CASSANDRA-1859:
Attachment: 0002-Pull-whirr-0.3.0-incubating-SNAPSHOT-155-from-Twitter-.txt
0001-Add-distributed-ultra-long-running-tests-using-Whirr-j.txt
This should be ready for review. It uses Whirr to deploy a 4 node cluster in a
compute provider, pushes artifacts from the working copy to a blobstore, and
then executes JUnit tests. See {{test/distributed/README.txt}} for more details.
There are a few obvious improvements that would be nice to have, including:
* Collecting logs from the test nodes
* Reusing an existing cluster
* Launching client nodes (for performance, rather than correctness)
distributed test harness
Key: CASSANDRA-1859
URL: https://issues.apache.org/jira/browse/CASSANDRA-1859
Project: Cassandra
Issue Type: Test
Components: Tools
Reporter: Kelvin Kakugawa
Assignee: Kelvin Kakugawa
Fix For: 0.7.1
Attachments:
0001-Add-distributed-ultra-long-running-tests-using-Whirr-j.txt,
0002-Pull-whirr-0.3.0-incubating-SNAPSHOT-155-from-Twitter-.txt
Distributed Test Harness
- deploys a cluster on a cloud provider
- runs tests targeted at the cluster
- tears down the cluster
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Issue Comment Edited: (CASSANDRA-1859) distributed test harness
[
https://issues.apache.org/jira/browse/CASSANDRA-1859?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976171#action_12976171
]
Stu Hood edited comment on CASSANDRA-1859 at 12/30/10 8:54 PM:
---
This should be ready for review. It uses Whirr to deploy a 4 node cluster in a
compute provider, pushes artifacts from the working copy to a blobstore, and
then executes JUnit tests. See {{test/distributed/README.txt}} for more details.
There are a few obvious improvements that would be nice to have, including:
* Collecting logs from the test nodes
* Reusing an existing cluster
* Launching client nodes (for performance, rather than correctness)
In order to specify the tarball URL to install Cassandra from, we needed to
make a modification to Whirr, which should be in Whirr 0.3.0: see WHIRR-155.
Until 0.3.0 is released, patch 0002 adds maven.twttr.com to our Ivy config, to
pull in a version of Whirr that includes 155. Also, when 0.3.0 is released,
we'll be able to switch from hoodidge.net to Whirr's sanctioned runurl base.
was (Author: stuhood):
This should be ready for review. It uses Whirr to deploy a 4 node cluster
in a compute provider, pushes artifacts from the working copy to a blobstore,
and then executes JUnit tests. See {{test/distributed/README.txt}} for more
details.
There are a few obvious improvements that would be nice to have, including:
* Collecting logs from the test nodes
* Reusing an existing cluster
* Launching client nodes (for performance, rather than correctness)
distributed test harness
Key: CASSANDRA-1859
URL: https://issues.apache.org/jira/browse/CASSANDRA-1859
Project: Cassandra
Issue Type: Test
Components: Tools
Reporter: Kelvin Kakugawa
Assignee: Kelvin Kakugawa
Fix For: 0.7.1
Attachments:
0001-Add-distributed-ultra-long-running-tests-using-Whirr-j.txt,
0002-Pull-whirr-0.3.0-incubating-SNAPSHOT-155-from-Twitter-.txt
Distributed Test Harness
- deploys a cluster on a cloud provider
- runs tests targeted at the cluster
- tears down the cluster
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Commented: (CASSANDRA-1015) Internal Messaging should be backwards compatible
[ https://issues.apache.org/jira/browse/CASSANDRA-1015?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976176#action_12976176 ] T Jake Luciani commented on CASSANDRA-1015: --- Johnathan, this is one approach I've had success with using thrift http://tjake.posterous.com/working-with-thrift-structures So based on the type field you serialize to one of the column types. Internal Messaging should be backwards compatible - Key: CASSANDRA-1015 URL: https://issues.apache.org/jira/browse/CASSANDRA-1015 Project: Cassandra Issue Type: Improvement Components: Core Reporter: Ryan King Assignee: Gary Dusbabek Priority: Critical Fix For: 0.8 Currently, incompatible changes in the node-to-node communication prevent rolling restarts of clusters. In order to fix this we should: 1) use a framework that makes doing compatible changes easy 2) have a policy of only making compatible changes between versions n and n+1* * Running multiple versions should only be supported for small periods of time. Running clusters of mixed version is not needed here. -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1859) distributed test harness
[ https://issues.apache.org/jira/browse/CASSANDRA-1859?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Stu Hood updated CASSANDRA-1859: Attachment: 0001-Add-distributed-ultra-long-running-tests-using-Whirr-j.txt distributed test harness Key: CASSANDRA-1859 URL: https://issues.apache.org/jira/browse/CASSANDRA-1859 Project: Cassandra Issue Type: Test Components: Tools Reporter: Kelvin Kakugawa Assignee: Kelvin Kakugawa Fix For: 0.8 Attachments: 0001-Add-distributed-ultra-long-running-tests-using-Whirr-j.txt, 0002-Pull-whirr-0.3.0-incubating-SNAPSHOT-155-from-Twitter-.txt Distributed Test Harness - deploys a cluster on a cloud provider - runs tests targeted at the cluster - tears down the cluster -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1859) distributed test harness
[ https://issues.apache.org/jira/browse/CASSANDRA-1859?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Stu Hood updated CASSANDRA-1859: Attachment: (was: 0001-Add-distributed-ultra-long-running-tests-using-Whirr-j.txt) distributed test harness Key: CASSANDRA-1859 URL: https://issues.apache.org/jira/browse/CASSANDRA-1859 Project: Cassandra Issue Type: Test Components: Tools Reporter: Kelvin Kakugawa Assignee: Kelvin Kakugawa Fix For: 0.8 Attachments: 0001-Add-distributed-ultra-long-running-tests-using-Whirr-j.txt, 0002-Pull-whirr-0.3.0-incubating-SNAPSHOT-155-from-Twitter-.txt Distributed Test Harness - deploys a cluster on a cloud provider - runs tests targeted at the cluster - tears down the cluster -- This message is automatically generated by JIRA. - You can reply to this email to add a comment to the issue online.
[jira] Commented: (CASSANDRA-1896) Improve throughput by adding buffering to the inter-node TCP communication
[
https://issues.apache.org/jira/browse/CASSANDRA-1896?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976188#action_12976188
]
Tsiki commented on CASSANDRA-1896:
--
Please note also the the flush method a sockets output stream (implemented by
SocketOuputStream) does nothing. SocketOutputStream extends FileOutputStream
which extends OutputStream. The flush method is not implemeneted by
SocketOutputStream nor by FileOutputStream and the default in OutputStream does
nothing. So, if you use Nagle with a burst of short packets you may end up with
unnecessary delays and increased latency.
Improve throughput by adding buffering to the inter-node TCP communication
--
Key: CASSANDRA-1896
URL: https://issues.apache.org/jira/browse/CASSANDRA-1896
Project: Cassandra
Issue Type: Improvement
Components: Core
Affects Versions: 0.6
Reporter: Tsiki
Assignee: Brandon Williams
Fix For: 0.6.9, 0.7.1
Attachments: 1896.txt
The inbound and outbound TCP implementation under org.apache.cassandra.net
does not buffer the socket streams. A simple change in IncomingTcpConnection
and OutboundTcpConnection may give a rather big throughput increase. In my
tests, I got up t o 30% more out of my cluster. Below is the diff of these
two files with buffering included. The diff is over release 0.6.5 but can be
quite simply applied also to 0.7. I suggest perhaps to limit the buffered
input stream I added in IncomingTcpConnection to 4K. The Outbound
implementation can surely be implemented a bit better (remove some of the
code I duplicated there).
diff -r
apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
fix/apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
44c44
input = new DataInputStream(new
BufferedInputStream(socket.getInputStream()));
---
input = new DataInputStream(socket.getInputStream());
diff -r
apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
fix/apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
77d76
byte[] buf = new byte[4096];
80,112c79,89
int l = 0;
ByteBuffer bb;
while ((bb = queue.peek()) != null l+bb.limit()
buf.length) {
bb = take();
System.arraycopy(bb.array(), 0, buf, l, bb.limit());
l += bb.limit();
}
if (l == 0) {
bb = take();
if (bb == CLOSE_SENTINEL)
{
disconnect();
continue;
}
if (socket != null || connect())
writeConnected(bb);
else
// clear out the queue, else gossip messages back up.
queue.clear();
} else {
if (socket != null || connect()) {
try {
output.write(buf, 0, l);
if (queue.peek() == null)
output.flush();
} catch (IOException e) {
logger.info(error writing to + endpoint);
disconnect();
}
} else {
queue.clear();
}
}
---
ByteBuffer bb = take();
if (bb == CLOSE_SENTINEL)
{
disconnect();
continue;
}
if (socket != null || connect())
writeConnected(bb);
else
// clear out the queue, else gossip messages back up.
queue.clear();
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Updated: (CASSANDRA-1896) Improve throughput by adding buffering to the inter-node TCP communication
[
https://issues.apache.org/jira/browse/CASSANDRA-1896?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Jonathan Ellis updated CASSANDRA-1896:
--
Attachment: 1896-v2.txt
Thanks, Tsiki. That is a good explanation. I also found
http://www.stuartcheshire.org/papers/NagleDelayedAck/ a useful description of
the nagle + delayed ack interaction. You are right that we want to continue
setting NODELAY and buffering manually.
v2 attached:
- fixes ByteBuffer assumptions
- respects CLOSE_SENTINEL in buffer loop
- cleans up output writes and removes no-op flushes
Improve throughput by adding buffering to the inter-node TCP communication
--
Key: CASSANDRA-1896
URL: https://issues.apache.org/jira/browse/CASSANDRA-1896
Project: Cassandra
Issue Type: Improvement
Components: Core
Affects Versions: 0.6
Reporter: Tsiki
Assignee: Brandon Williams
Fix For: 0.6.9, 0.7.1
Attachments: 1896-v2.txt, 1896.txt
The inbound and outbound TCP implementation under org.apache.cassandra.net
does not buffer the socket streams. A simple change in IncomingTcpConnection
and OutboundTcpConnection may give a rather big throughput increase. In my
tests, I got up t o 30% more out of my cluster. Below is the diff of these
two files with buffering included. The diff is over release 0.6.5 but can be
quite simply applied also to 0.7. I suggest perhaps to limit the buffered
input stream I added in IncomingTcpConnection to 4K. The Outbound
implementation can surely be implemented a bit better (remove some of the
code I duplicated there).
diff -r
apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
fix/apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
44c44
input = new DataInputStream(new
BufferedInputStream(socket.getInputStream()));
---
input = new DataInputStream(socket.getInputStream());
diff -r
apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
fix/apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
77d76
byte[] buf = new byte[4096];
80,112c79,89
int l = 0;
ByteBuffer bb;
while ((bb = queue.peek()) != null l+bb.limit()
buf.length) {
bb = take();
System.arraycopy(bb.array(), 0, buf, l, bb.limit());
l += bb.limit();
}
if (l == 0) {
bb = take();
if (bb == CLOSE_SENTINEL)
{
disconnect();
continue;
}
if (socket != null || connect())
writeConnected(bb);
else
// clear out the queue, else gossip messages back up.
queue.clear();
} else {
if (socket != null || connect()) {
try {
output.write(buf, 0, l);
if (queue.peek() == null)
output.flush();
} catch (IOException e) {
logger.info(error writing to + endpoint);
disconnect();
}
} else {
queue.clear();
}
}
---
ByteBuffer bb = take();
if (bb == CLOSE_SENTINEL)
{
disconnect();
continue;
}
if (socket != null || connect())
writeConnected(bb);
else
// clear out the queue, else gossip messages back up.
queue.clear();
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
[jira] Commented: (CASSANDRA-1896) Improve throughput by adding buffering to the inter-node TCP communication
[
https://issues.apache.org/jira/browse/CASSANDRA-1896?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=12976194#action_12976194
]
Tsiki commented on CASSANDRA-1896:
--
I may be missing something here, but I think the order of the write operations
in the patch is reversed. If you have data both in the buffer and in the bb,
you send the bb first - but it should be sent after the buffer.
Improve throughput by adding buffering to the inter-node TCP communication
--
Key: CASSANDRA-1896
URL: https://issues.apache.org/jira/browse/CASSANDRA-1896
Project: Cassandra
Issue Type: Improvement
Components: Core
Affects Versions: 0.6
Reporter: Tsiki
Assignee: Brandon Williams
Fix For: 0.6.9, 0.7.1
Attachments: 1896-v2.txt, 1896.txt
The inbound and outbound TCP implementation under org.apache.cassandra.net
does not buffer the socket streams. A simple change in IncomingTcpConnection
and OutboundTcpConnection may give a rather big throughput increase. In my
tests, I got up t o 30% more out of my cluster. Below is the diff of these
two files with buffering included. The diff is over release 0.6.5 but can be
quite simply applied also to 0.7. I suggest perhaps to limit the buffered
input stream I added in IncomingTcpConnection to 4K. The Outbound
implementation can surely be implemented a bit better (remove some of the
code I duplicated there).
diff -r
apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
fix/apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/IncomingTcpConnection.java
44c44
input = new DataInputStream(new
BufferedInputStream(socket.getInputStream()));
---
input = new DataInputStream(socket.getInputStream());
diff -r
apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
fix/apache-cassandra-0.6.5-src/src/java/org/apache/cassandra/net/OutboundTcpConnection.java
77d76
byte[] buf = new byte[4096];
80,112c79,89
int l = 0;
ByteBuffer bb;
while ((bb = queue.peek()) != null l+bb.limit()
buf.length) {
bb = take();
System.arraycopy(bb.array(), 0, buf, l, bb.limit());
l += bb.limit();
}
if (l == 0) {
bb = take();
if (bb == CLOSE_SENTINEL)
{
disconnect();
continue;
}
if (socket != null || connect())
writeConnected(bb);
else
// clear out the queue, else gossip messages back up.
queue.clear();
} else {
if (socket != null || connect()) {
try {
output.write(buf, 0, l);
if (queue.peek() == null)
output.flush();
} catch (IOException e) {
logger.info(error writing to + endpoint);
disconnect();
}
} else {
queue.clear();
}
}
---
ByteBuffer bb = take();
if (bb == CLOSE_SENTINEL)
{
disconnect();
continue;
}
if (socket != null || connect())
writeConnected(bb);
else
// clear out the queue, else gossip messages back up.
queue.clear();
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
