Author: jbellis
Date: Tue Sep 27 15:29:14 2011
New Revision: 1176433
URL: http://svn.apache.org/viewvc?rev=1176433&view=rev
Log:
merge from 1.0.0
Modified:
cassandra/branches/cassandra-1.0/ (props changed)
cassandra/branches/cassandra-1.0/CHANGES.txt
cassandra/branches/cassandra-1.0/NEWS.txt
cassandra/branches/cassandra-1.0/bin/cassandra
cassandra/branches/cassandra-1.0/conf/cassandra.yaml
cassandra/branches/cassandra-1.0/contrib/ (props changed)
cassandra/branches/cassandra-1.0/debian/cassandra.conf
cassandra/branches/cassandra-1.0/debian/init
cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
(props changed)
cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
(props changed)
cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
(props changed)
cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
(props changed)
cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
(props changed)
cassandra/branches/cassandra-1.0/redhat/cassandra.conf
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionController.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/security/SSLFactory.java
Propchange: cassandra/branches/cassandra-1.0/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:29:14 2011
@@ -1,11 +1,11 @@
/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7:1026516-1170333,1172024
/cassandra/branches/cassandra-0.7.0:1053690-1055654
-/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1176205
/cassandra/branches/cassandra-0.8.0:1125021-1130369
/cassandra/branches/cassandra-0.8.1:1101014-1125018
/cassandra/branches/cassandra-1.0:1167106,1167185
-/cassandra/branches/cassandra-1.0.0:1167104-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0.0:1167104-1176432
/cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1:1102511-1125020
/cassandra/trunk:1167085-1167102,1169870
Modified: cassandra/branches/cassandra-1.0/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/CHANGES.txt?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-1.0/CHANGES.txt Tue Sep 27 15:29:14 2011
@@ -5,6 +5,8 @@
1.0.0-final
* Log a miningfull warning when a node receive a message for a repair session
that don't exist anymore (CASSANDRA-3256)
+ * test for NUMA policy support as well as numactl presence (CASSANDRA-3245)
+ * Fix FD leak when internode encryption is enabled (CASSANDRA-3257)
* Remove incorrect assertion in mergeIterator (CASSANDRA-3260)
@@ -27,8 +29,9 @@
* Allow using quotes in "USE <keyspace>;" CLI command (CASSANDRA-3208)
* Don't allow any cache loading exceptions to halt startup (CASSANDRA-3218)
* Fix sstableloader --ignores option (CASSANDRA-3247)
+ * File descriptor limit increased in packaging (CASSANDRA-3206)
* Fix deadlock in commit log during flush (CASSANDRA-3253)
-
+
1.0.0-beta1
* removed binarymemtable (CASSANDRA-2692)
Modified: cassandra/branches/cassandra-1.0/NEWS.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/NEWS.txt?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/NEWS.txt (original)
+++ cassandra/branches/cassandra-1.0/NEWS.txt Tue Sep 27 15:29:14 2011
@@ -204,7 +204,7 @@ Features
- optional intranode encryption; see comments around 'encryption_options'
in cassandra.yaml
- compaction multithreading and rate-limiting; see
- 'compaction_multithreading' and 'compaction_throughput_mb_per_sec' in
+ 'concurrent_compactors' and 'compaction_throughput_mb_per_sec' in
cassandra.yaml
- cassandra will limit total memtable memory usage to 1/3 of the heap
by default. This can be ajusted or disabled with the
Modified: cassandra/branches/cassandra-1.0/bin/cassandra
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/bin/cassandra?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/bin/cassandra (original)
+++ cassandra/branches/cassandra-1.0/bin/cassandra Tue Sep 27 15:29:14 2011
@@ -91,10 +91,12 @@ fi
# If numactl is available, use it. For Cassandra, the priority is to
# avoid disk I/O. Even for the purpose of CPU efficiency, we don't
-# really have CPU<->data affinity anyway.
-if which numactl >/dev/null 2>/dev/null
+# really have CPU<->data affinity anyway. Also, empirically test that numactl
+# works before trying to use it (CASSANDRA-3245).
+NUMACTL_ARGS="--interleave=all"
+if which numactl >/dev/null 2>/dev/null && numactl $NUMACTL_ARGS ls /
>/dev/null 2>/dev/null
then
- NUMACTL="numactl --interleave=all"
+ NUMACTL="numactl $NUMACTL_ARGS"
else
NUMACTL=""
fi
Modified: cassandra/branches/cassandra-1.0/conf/cassandra.yaml
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/conf/cassandra.yaml?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/conf/cassandra.yaml (original)
+++ cassandra/branches/cassandra-1.0/conf/cassandra.yaml Tue Sep 27 15:29:14
2011
@@ -137,7 +137,6 @@ concurrent_writes: 32
# Total memory to use for memtables. Cassandra will flush the largest
# memtable when this much memory is used.
# If omitted, Cassandra will set it to 1/3 of the heap.
-# If set to 0, only the old flush thresholds are used.
# memtable_total_space_in_mb: 2048
# Total space to use for commitlogs.
Propchange: cassandra/branches/cassandra-1.0/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:29:14 2011
@@ -1,11 +1,11 @@
/cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
/cassandra/branches/cassandra-0.7/contrib:1026516-1170333,1172024
/cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
-/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1175057,1175880
/cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369
/cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018
/cassandra/branches/cassandra-1.0/contrib:1167106,1167185
-/cassandra/branches/cassandra-1.0.0/contrib:1167104-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0.0/contrib:1167104-1175725,1175770-1175771,1175882,1176371
/cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/contrib:1102511-1125020
/cassandra/trunk/contrib:1167085-1167102,1169870
Modified: cassandra/branches/cassandra-1.0/debian/cassandra.conf
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/debian/cassandra.conf?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/debian/cassandra.conf (original)
+++ cassandra/branches/cassandra-1.0/debian/cassandra.conf Tue Sep 27 15:29:14
2011
@@ -1,2 +1,3 @@
# Provided by the cassandra package
cassandra - memlock unlimited
+cassandra - nofile 100000
Modified: cassandra/branches/cassandra-1.0/debian/init
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/debian/init?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/debian/init (original)
+++ cassandra/branches/cassandra-1.0/debian/init Tue Sep 27 15:29:14 2011
@@ -22,6 +22,7 @@ CONFDIR=/etc/cassandra
JSVC=/usr/bin/jsvc
WAIT_FOR_START=10
CASSANDRA_HOME=/usr/share/cassandra
+FD_LIMIT=100000
# The first existing directory is used for JAVA_HOME if needed.
JVM_SEARCH_DIRS="/usr/lib/jvm/java-6-openjdk /usr/lib/jvm/java-6-sun"
@@ -127,6 +128,7 @@ do_start()
is_running && return 1
ulimit -l unlimited
+ ulimit -n "$FD_LIMIT"
cassandra_home=`getent passwd cassandra | awk -F ':' '{ print $6; }'`
cd / # jsvc doesn't chdir() for us
Propchange:
cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:29:14 2011
@@ -1,11 +1,11 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1170333,1172024
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1175057,1175880
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018
/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167106,1167185
-/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167104-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167104-1175725,1175770-1175771,1175882,1176371
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1102511-1125020
/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1167102,1169870
Propchange:
cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:29:14 2011
@@ -1,11 +1,11 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1170333,1172024
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1175057,1175880
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018
/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167106,1167185
-/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167104-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167104-1175725,1175770-1175771,1175882,1176371
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1102511-1125020
/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1167102,1169870
Propchange:
cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:29:14 2011
@@ -1,11 +1,11 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1170333,1172024
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1175057,1175880
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018
/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167106,1167185
-/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167104-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167104-1175725,1175770-1175771,1175882,1176371
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1102511-1125020
/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1167102,1169870
Propchange:
cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:29:14 2011
@@ -1,11 +1,11 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1170333,1172024
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1175057,1175880
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018
/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167106,1167185
-/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167104-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167104-1175725,1175770-1175771,1175882,1176371
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1102511-1125020
/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1167102,1169870
Propchange:
cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:29:14 2011
@@ -1,11 +1,11 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1170333,1172024
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1175057,1175880
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018
/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167106,1167185
-/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167104-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167104-1175725,1175770-1175771,1175882,1176371
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1102511-1125020
/cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1167102,1169870
Modified: cassandra/branches/cassandra-1.0/redhat/cassandra.conf
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/redhat/cassandra.conf?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/redhat/cassandra.conf (original)
+++ cassandra/branches/cassandra-1.0/redhat/cassandra.conf Tue Sep 27 15:29:14
2011
@@ -1 +1,2 @@
cassandra - memlock unlimited
+cassandra - nofile 100000
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Tue Sep 27 15:29:14 2011
@@ -321,7 +321,8 @@ public class ColumnFamilyStore implement
*/
public static void scrubDataDirectories(String table, String columnFamily)
{
- logger.info("Removing compacted SSTable files from " + columnFamily +
" (see http://wiki.apache.org/cassandra/MemtableSSTable)");
+ logger.debug("Removing compacted SSTable files from {} (see
http://wiki.apache.org/cassandra/MemtableSSTable)", columnFamily);
+
for (Map.Entry<Descriptor,Set<Component>> sstableFiles : files(table,
columnFamily, true, true).entrySet())
{
Descriptor desc = sstableFiles.getKey();
@@ -884,6 +885,11 @@ public class ColumnFamilyStore implement
return false;
}
+ public boolean isKeyExistenceExpensive(Set<? extends SSTable>
sstablesToIgnore)
+ {
+ return compactionStrategy.isKeyExistenceExpensive(sstablesToIgnore);
+ }
+
/*
* Called after a BinaryMemtable flushes its in-memory data, or we add a
file
* via bootstrap. This information is cached in the ColumnFamilyStore.
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
Tue Sep 27 15:29:14 2011
@@ -21,9 +21,11 @@ package org.apache.cassandra.db.compacti
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.service.StorageService;
@@ -97,4 +99,10 @@ public abstract class AbstractCompaction
* @return size in bytes of the largest sstables for this strategy
*/
public abstract long getMaxSSTableSize();
+
+ /**
+ * @return true if checking for whether a key exists, ignoring @param
sstablesToIgnore,
+ * is going to be expensive
+ */
+ public abstract boolean isKeyExistenceExpensive(Set<? extends SSTable>
sstablesToIgnore);
}
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionController.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionController.java?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionController.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionController.java
Tue Sep 27 15:29:14 2011
@@ -44,6 +44,7 @@ public class CompactionController
private final boolean forceDeserialize;
public final int gcBefore;
+ public boolean keyExistenceIsExpensive;
public CompactionController(ColumnFamilyStore cfs,
Collection<SSTableReader> sstables, int gcBefore, boolean forceDeserialize)
{
@@ -52,6 +53,7 @@ public class CompactionController
this.sstables = new HashSet<SSTableReader>(sstables);
this.gcBefore = gcBefore;
this.forceDeserialize = forceDeserialize;
+ keyExistenceIsExpensive =
cfs.getCompactionStrategy().isKeyExistenceExpensive(this.sstables);
}
public String getKeyspace()
@@ -102,13 +104,19 @@ public class CompactionController
*/
public AbstractCompactedRow getCompactedRow(List<SSTableIdentityIterator>
rows)
{
- if (rows.size() == 1 && !needDeserialize() &&
!shouldPurge(rows.get(0).getKey()))
- return new EchoedRow(this, rows.get(0));
-
long rowSize = 0;
for (SSTableIdentityIterator row : rows)
rowSize += row.dataSize;
+ // in-memory echoedrow is only enabled if we think checking for the
key's existence in the other sstables,
+ // is going to be less expensive than simply de/serializing the row
again
+ if (rows.size() == 1 && !needDeserialize()
+ && (rowSize > DatabaseDescriptor.getInMemoryCompactionLimit() ||
!keyExistenceIsExpensive)
+ && !shouldPurge(rows.get(0).getKey()))
+ {
+ return new EchoedRow(this, rows.get(0));
+ }
+
if (rowSize > DatabaseDescriptor.getInMemoryCompactionLimit())
{
String keyString =
cfs.metadata.getKeyValidator().getString(rows.get(0).getKey().key);
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
Tue Sep 27 15:29:14 2011
@@ -72,11 +72,12 @@ public class CompactionTask extends Abst
Set<SSTableReader> toCompact = new HashSet<SSTableReader>(sstables);
if (!isUserDefined)
{
- if ( !allowSingletonCompaction() && toCompact.size() < 2)
+ if (!allowSingletonCompaction() && toCompact.size() < 2)
{
- logger.info("Nothing to compact in " +
cfs.getColumnFamilyName() + "." +
- "Use forceUserDefinedCompaction if you wish to
force compaction of single sstables " +
- "(e.g. for tombstone collection)");
+ String msg = "Nothing to compact in " +
cfs.getColumnFamilyName();
+ if (cfs.getCompactionStrategy() instanceof
SizeTieredCompactionStrategy)
+ msg += ". Use forceUserDefinedCompaction if you wish to
force compaction of single sstables (e.g. for tombstone collection)";
+ logger.info(msg);
return 0;
}
@@ -220,7 +221,7 @@ public class CompactionTask extends Abst
double mbps = dTime > 0 ?
(double)endsize/(1024*1024)/((double)dTime/1000) : 0;
logger.info(String.format("Compacted to %s. %,d to %,d (~%d%% of
original) bytes for %,d keys at %fMBPS. Time: %,dms.",
builder.toString(), startsize, endsize,
(int) (ratio * 100), totalkeysWritten, mbps, dTime));
- logger.info(String.format("CF Total Bytes Compacted: %,d",
CompactionTask.addToTotalBytesCompacted(endsize)));
+ logger.debug(String.format("CF Total Bytes Compacted: %,d",
CompactionTask.addToTotalBytesCompacted(endsize)));
return toCompact.size();
}
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
Tue Sep 27 15:29:14 2011
@@ -21,17 +21,17 @@ package org.apache.cassandra.db.compacti
*/
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.notifications.INotification;
import org.apache.cassandra.notifications.INotificationConsumer;
@@ -71,7 +71,7 @@ public class LeveledCompactionStrategy e
maxSSTableSizeInMB = configuredMaxSSTableSize;
cfs.getDataTracker().subscribe(this);
- logger.info(this + " subscribed to the data tracker.");
+ logger.debug("{} subscribed to the data tracker.", this);
manifest = LeveledManifest.create(cfs, this.maxSSTableSizeInMB);
logger.debug("Created {}", manifest);
@@ -159,6 +159,12 @@ public class LeveledCompactionStrategy e
return maxSSTableSizeInMB * 1024 * 1024;
}
+ public boolean isKeyExistenceExpensive(Set<? extends SSTable>
sstablesToIgnore)
+ {
+ Set<SSTableReader> L0 = ImmutableSet.copyOf(manifest.getLevel(0));
+ return Sets.difference(L0, sstablesToIgnore).size() +
manifest.getLevelCount() > 20;
+ }
+
@Override
public String toString()
{
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
Tue Sep 27 15:29:14 2011
@@ -27,8 +27,6 @@ import java.io.IOException;
import java.util.*;
import com.google.common.collect.Iterables;
-import org.apache.commons.lang.StringUtils;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,12 +52,13 @@ public class LeveledManifest
* uses a pessimistic estimate of how many keys overlap (none), so we risk
wasting memory
* or even OOMing when compacting highly overlapping sstables
*/
- private static int MAX_COMPACTING_L0 = 32;
+ static int MAX_COMPACTING_L0 = 32;
private final ColumnFamilyStore cfs;
private final List<SSTableReader>[] generations;
private final DecoratedKey[] lastCompactedKeys;
private final int maxSSTableSizeInMB;
+ private int levelCount;
private LeveledManifest(ColumnFamilyStore cfs, int maxSSTableSizeInMB)
{
@@ -135,14 +134,16 @@ public class LeveledManifest
serialize();
}
- // if the number of SSTables in the current compacted set exeeds the
target level, find an empty level
+ /**
+ * if the number of SSTables in the current compacted set *by itself*
exeeds the target level's
+ * (regardless of the level's current contents), find an empty level
instead
+ */
private int skipLevels(int newLevel, Iterable<SSTableReader> added)
{
- // skip newlevel if the resulting sstables exceed newlevel threshold
- if (maxBytesForLevel(newLevel) < SSTableReader.getTotalBytes(added)
- && SSTableReader.getTotalBytes(generations[(newLevel + 1)]) == 0)
+ while (maxBytesForLevel(newLevel) < SSTableReader.getTotalBytes(added)
+ && generations[(newLevel + 1)].isEmpty())
{
- newLevel = skipLevels(newLevel + 1, added);
+ newLevel++;
}
return newLevel;
}
@@ -171,7 +172,8 @@ public class LeveledManifest
int newLevel = minimumLevel == maximumLevel ? maximumLevel + 1 :
maximumLevel;
newLevel = skipLevels(newLevel, added);
assert newLevel > 0;
- logger.debug("Adding [{}] at L{}", StringUtils.join(added.iterator(),
", "), newLevel);
+ if (logger.isDebugEnabled())
+ logger.debug("Adding [{}] at L{}", toString(added), newLevel);
lastCompactedKeys[minimumLevel] =
SSTable.sstableOrdering.max(added).last;
for (SSTableReader ssTableReader : added)
@@ -185,7 +187,12 @@ public class LeveledManifest
StringBuilder builder = new StringBuilder();
for (SSTableReader sstable : sstables)
{
- builder.append(sstable.toString()).append("
(L").append(levelOf(sstable)).append("), ");
+ builder.append(sstable.descriptor.cfname)
+ .append('-')
+ .append(sstable.descriptor.generation)
+ .append("(L")
+ .append(levelOf(sstable))
+ .append("), ");
}
return builder.toString();
}
@@ -383,7 +390,7 @@ public class LeveledManifest
File manifestFile = new File(new File(dir, cfs.table.name),
cfs.columnFamily + ".json");
if (manifestFile.exists())
{
- logger.debug("Loading manifest from {}", manifestFile);
+ logger.debug("Found manifest at {}", manifestFile);
return manifestFile;
}
}
@@ -396,4 +403,19 @@ public class LeveledManifest
{
return "Manifest@" + hashCode();
}
+
+ public int getLevelCount()
+ {
+ for (int i = generations.length - 1; i >= 0; i--)
+ {
+ if (generations[i].size() > 0)
+ return i;
+ }
+ return 0;
+ }
+
+ public List<SSTableReader> getLevel(int i)
+ {
+ return generations[i];
+ }
}
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
Tue Sep 27 15:29:14 2011
@@ -25,6 +25,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataTracker;
+import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.utils.Pair;
@@ -181,6 +183,11 @@ public class SizeTieredCompactionStrateg
return Long.MAX_VALUE;
}
+ public boolean isKeyExistenceExpensive(Set<? extends SSTable>
sstablesToIgnore)
+ {
+ return cfs.getSSTables().size() - sstablesToIgnore.size() > 20;
+ }
+
public String toString()
{
return String.format("SizeTieredCompactionStrategy[%s/%s]",
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/security/SSLFactory.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/security/SSLFactory.java?rev=1176433&r1=1176432&r2=1176433&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/security/SSLFactory.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/security/SSLFactory.java
Tue Sep 27 15:29:14 2011
@@ -25,7 +25,6 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.SocketAddress;
import java.security.KeyStore;
import javax.net.ssl.KeyManagerFactory;
@@ -35,6 +34,7 @@ import javax.net.ssl.SSLSocket;
import javax.net.ssl.TrustManagerFactory;
import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.io.util.FileUtils;
/**
* A Factory for providing and setting up Client and Server SSL wrapped
@@ -46,7 +46,6 @@ public final class SSLFactory
private static final String ALGORITHM = "SunX509";
private static final String STORE_TYPE = "JKS";
-
public static SSLServerSocket getServerSocket(EncryptionOptions options,
InetAddress address, int port) throws IOException
{
SSLContext ctx = createSSLContext(options);
@@ -75,28 +74,39 @@ public final class SSLFactory
return socket;
}
- private static SSLContext createSSLContext(EncryptionOptions options)
throws IOException {
+ private static SSLContext createSSLContext(EncryptionOptions options)
throws IOException
+ {
+ FileInputStream tsf = new FileInputStream(options.truststore);
+ FileInputStream ksf = new FileInputStream(options.keystore);
SSLContext ctx;
- try {
+ try
+ {
ctx = SSLContext.getInstance(PROTOCOL);
- TrustManagerFactory tmf = null;
- KeyManagerFactory kmf = null;
+ TrustManagerFactory tmf;
+ KeyManagerFactory kmf;
tmf = TrustManagerFactory.getInstance(ALGORITHM);
KeyStore ts = KeyStore.getInstance(STORE_TYPE);
- ts.load(new FileInputStream(options.truststore),
options.truststore_password.toCharArray());
+ ts.load(tsf, options.truststore_password.toCharArray());
tmf.init(ts);
kmf = KeyManagerFactory.getInstance(ALGORITHM);
KeyStore ks = KeyStore.getInstance(STORE_TYPE);
- ks.load(new FileInputStream(options.keystore),
options.keystore_password.toCharArray());
+ ks.load(ksf, options.keystore_password.toCharArray());
kmf.init(ks, options.keystore_password.toCharArray());
ctx.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
- } catch (Exception e) {
+ }
+ catch (Exception e)
+ {
throw new IOException("Error creating the initializing the SSL
Context", e);
}
+ finally
+ {
+ FileUtils.closeQuietly(tsf);
+ FileUtils.closeQuietly(ksf);
+ }
return ctx;
}
}
