Author: [email protected]
Date: Fri Feb 10 14:49:22 2012
New Revision: 2079
Log:
[AMDATUCASSANDRA-157] Fixed most WARNING errors, except for the WARNING errors
during the integration test (SLF4J and MemoryMeter)
Modified:
trunk/amdatu-cassandra/cassandra-application/src/test/resources/cassandra.yaml
trunk/amdatu-cassandra/cassandra-client/src/test/resources/cassandra.yaml
trunk/amdatu-cassandra/cassandra-persistencemanager-hector/src/main/java/org/amdatu/cassandra/persistencemanager/hector/HectorCassandraPersistenceManager.java
trunk/amdatu-cassandra/cassandra-persistencemanager-hector/src/test/resources/
trunk/amdatu-cassandra/cassandra-persistencemanager/src/main/java/org/amdatu/cassandra/persistencemanager/CassandraPersistenceManager.java
trunk/amdatu-cassandra/cassandra-stores/cassandra-store-consumer/pom.xml
trunk/amdatu-cassandra/cassandra-stores/cassandra-store-useradmin/pom.xml
trunk/amdatu-cassandra/release/pom.xml
trunk/amdatu-cassandra/test-performance/test-releases/cassandra-0.2.0/pom.xml
trunk/amdatu-cassandra/test-performance/test-releases/cassandra-0.2.1/pom.xml
trunk/amdatu-cassandra/test-unit/framework-hector/src/main/resources/cassandra.yaml
trunk/amdatu-cassandra/test-unit/framework/src/main/resources/cassandra.yaml
Modified:
trunk/amdatu-cassandra/cassandra-application/src/test/resources/cassandra.yaml
==============================================================================
---
trunk/amdatu-cassandra/cassandra-application/src/test/resources/cassandra.yaml
(original)
+++
trunk/amdatu-cassandra/cassandra-application/src/test/resources/cassandra.yaml
Fri Feb 10 14:49:22 2012
@@ -15,10 +15,10 @@
# Cassandra storage config YAML
-# NOTE:
-# See http://wiki.apache.org/cassandra/StorageConfiguration for
-# full explanations of configuration directives
-# /NOTE
+#NOTE !!!!!!!! NOTE
+# See http://wiki.apache.org/cassandra/StorageConfiguration for
+# full explanations of configuration directives
+#NOTE !!!!!!!! NOTE
# The name of the cluster. This is mainly used to prevent machines in
# one logical cluster from joining another.
@@ -34,7 +34,7 @@
# the heaviest-loaded existing node. If there is no load information
# available, such as is the case with a new cluster, it will pick
# a random token, which will lead to hot spots.
-initial_token:
+initial_token: 0
# Set to true to make new [non-seed] nodes automatically migrate data
# to themselves from the pre-existing nodes in the cluster. Defaults
@@ -49,6 +49,8 @@
# this defines the maximum amount of time a dead host will have hints
# generated. After it has been dead this long, hints will be dropped.
max_hint_window_in_ms: 3600000 # one hour
+# Sleep this long after delivering each row or row fragment
+hinted_handoff_throttle_delay_in_ms: 50
# authentication backend, implementing IAuthenticator; used to identify users
authenticator: org.apache.cassandra.auth.AllowAllAuthenticator
@@ -96,26 +98,30 @@
# commitlog_sync may be either "periodic" or "batch."
# When in batch mode, Cassandra won't ack writes until the commit log
# has been fsynced to disk. It will wait up to
-# CommitLogSyncBatchWindowInMS milliseconds for other writes, before
+# commitlog_sync_batch_window_in_ms milliseconds for other writes, before
# performing the sync.
-commitlog_sync: periodic
-
-# the other option is "timed," where writes may be acked immediately
+#
+# commitlog_sync: batch
+# commitlog_sync_batch_window_in_ms: 50
+#
+# the other option is "periodic" where writes may be acked immediately
# and the CommitLog is simply synced every commitlog_sync_period_in_ms
# milliseconds.
+commitlog_sync: periodic
commitlog_sync_period_in_ms: 10000
# any class that implements the SeedProvider interface and has a constructor
that takes a Map<String, String> of
# parameters will do.
seed_provider:
- # Addresses of hosts that are deemed contact points.
- # Cassandra nodes use this list of hosts to find each other and learn
- # the topology of the ring. You must change this if you are running
- # multiple nodes!
- - class_name: org.apache.cassandra.locator.SimpleSeedProvider
- parameters:
- # seeds is actually a comma-delimited list of addresses.
- - seeds: "127.0.0.1"
+ # Addresses of hosts that are deemed contact points.
+ # Cassandra nodes use this list of hosts to find each other and learn
+ # the topology of the ring. You must change this if you are running
+ # multiple nodes!
+ - class_name: org.apache.cassandra.locator.SimpleSeedProvider
+ parameters:
+ # seeds is actually a comma-delimited list of addresses.
+ # Ex: "<ip1>,<ip2>,<ip3>"
+ - seeds: "127.0.0.1"
# emergency pressure valve: each time heap usage after a full (CMS)
# garbage collection is above this fraction of the max, Cassandra will
@@ -142,16 +148,6 @@
reduce_cache_sizes_at: 0.85
reduce_cache_capacity_to: 0.6
-# Access mode. mmapped i/o is substantially faster, but only practical on
-# a 64bit machine (which notably does not include EC2 "small" instances)
-# or relatively small datasets. "auto", the safe choice, will enable
-# mmapping on a 64bit JVM. Other values are "mmap", "mmap_index_only"
-# (which may allow you to get part of the benefits of mmap on a 32bit
-# machine by mmapping only index files) and "standard".
-# (The buffer size settings that follow only apply to standard,
-# non-mmapped i/o.)
-disk_access_mode: auto
-
# For workloads with more data than can fit in memory, Cassandra's
# bottleneck will be reads that need to fetch data from
# disk. "concurrent_reads" should be set to (16 * number_of_drives) in
@@ -169,7 +165,7 @@
# the older, per-ColumnFamily memtable flush thresholds.
# If omitted, Cassandra will set it to 1/3 of the heap.
# If set to 0, only the old flush thresholds are used.
-# memtable_total_space_in_mb: 2048
+memtable_total_space_in_mb: 0
# This sets the amount of memtable flush writer threads. These will
# be blocked by disk io, and each one will hold a memtable in memory
@@ -178,6 +174,11 @@
# By default this will be set to the amount of data directories defined.
#memtable_flush_writers: 1
+# the number of full memtables to allow pending flush, that is,
+# waiting for a writer thread. At a minimum, this should be set to
+# the maximum number of secondary indexes created on a single CF.
+memtable_flush_queue_size: 4
+
# Buffer size to use when performing contiguous column slices.
# Increase this to the size of the column slices you typically perform
sliced_buffer_size_in_kb: 64
@@ -204,12 +205,46 @@
# Leaving this blank has the same effect it does for ListenAddress,
# (i.e. it will be based on the configured hostname of the node).
rpc_address: 127.0.0.1
+
# port for Thrift to listen for clients on
rpc_port: 9161
# enable or disable keepalive on rpc connections
rpc_keepalive: true
+# Cassandra provides you with a variety of options for RPC Server
+# sync -> Creates one thread per connection but with a configurable number of
+# threads. This can be expensive in memory used for thread stack for
+# a large enough number of clients. (Hence, connection pooling is
+# very, very strongly recommended.)
+#
+# async -> Nonblocking server implementation with one thread to serve
+# rpc connections. This is not recommended for high throughput use
+# cases.
+#
+# hsha -> half sync and half async implementation with configurable number
+# of worker threads (For managing connections). IO Management is
+# done by a set of threads currently equal to the number of
+# processors in the system. The number of threads in the threadpool
+# is configured via rpc_min_threads and rpc_max_threads. (Connection
+# pooling is strongly recommended in this case too.)
+
+rpc_server_type: sync
+
+# Uncomment rpc_min|max|thread to set request pool size.
+# You would primarily set max for the sync server to safeguard against
+# misbehaved clients; if you do hit the max, Cassandra will block until one
+# disconnects before accepting more. The defaults are min of 16 and max
+# unlimited.
+#
+# For the Hsha server, you would set the max so that a fair amount of resources
+# are provided to the other working threads on the server.
+#
+# This configuration is not used for the async server.
+#
+# rpc_min_threads: 16
+# rpc_max_threads: 2048
+
# uncomment to set socket buffer sizes on rpc connections
# rpc_send_buff_size_in_bytes:
# rpc_recv_buff_size_in_bytes:
@@ -223,6 +258,12 @@
# internal thrift overhead.
thrift_max_message_length_in_mb: 16
+# Set to true to have Cassandra create a hard link to each sstable
+# flushed or streamed locally in a backups/ subdirectory of the
+# Keyspace data. Removing these links is the operator's
+# responsibility.
+incremental_backups: false
+
# Whether or not to take a snapshot before each compaction. Be
# careful using this option, since Cassandra won't clean up the
# snapshots for you. Mostly useful if you're paranoid when there
@@ -230,13 +271,9 @@
snapshot_before_compaction: false
# change this to increase the compaction thread's priority. In java, 1 is the
-# lowest priority and that is our default.
+# lowest priority and that is our default. The highest allowed is 5.
# compaction_thread_priority: 1
-# The threshold size in megabytes the binary memtable must grow to,
-# before it's submitted for flushing to disk.
-binary_memtable_throughput_in_mb: 256
-
# Add column indexes to a row after its contents reach this size.
# Increase if your column values are large, or if you have a very large
# number of columns. The competing causes are, Cassandra has to
@@ -251,13 +288,15 @@
# will be logged specifying the row key.
in_memory_compaction_limit_in_mb: 64
-# Number of compaction threads. This default to the number of processors,
-# enabling multiple compactions to execute at once. Using more than one
-# thread is highly recommended to preserve read performance in a mixed
-# read/write workload as this avoids sstables from accumulating during long
-# running compactions. The default is usually fine and if you experience
-# problems with compaction running too slowly or too fast, you should look at
+# Number of simultaneous compactions to allow, NOT including
+# validation "compactions" for anti-entropy repair. This defaults to
+# the number of cores. This can help preserve read performance in a
+# mixed read/write workload, by mitigating the tendency of small
+# sstables to accumulate during a single long running compactions. The
+# default is usually fine and if you experience problems with
+# compaction running too slowly or too fast, you should look at
# compaction_throughput_mb_per_sec first.
+#
# Uncomment to make compaction mono-threaded.
#concurrent_compactors: 1
@@ -265,9 +304,15 @@
# system. The faster you insert data, the faster you need to compact in
# order to keep the sstable count down, but in general, setting this to
# 16 to 32 times the rate you are inserting data is more than sufficient.
-# Setting this to 0 disables throttling.
+# Setting this to 0 disables throttling. Note that this account for all types
+# of compaction, including validation compaction.
compaction_throughput_mb_per_sec: 16
+# Track cached row keys during compaction, and re-cache their new
+# positions in the compacted sstable. Disable if you use really large
+# key caches.
+compaction_preheat_key_cache: true
+
# Time to wait for a reply from other nodes before failing the command
rpc_timeout_in_ms: 10000
@@ -351,9 +396,16 @@
# the request scheduling. Currently the only valid option is keyspace.
# request_scheduler_id: keyspace
-# The Index Interval determines how large the sampling of row keys
-# is for a given SSTable. The larger the sampling, the more effective
-# the index is at the cost of space.
+# index_interval controls the sampling of entries from the primrary
+# row index in terms of space versus time. The larger the interval,
+# the smaller and less effective the sampling will be. In technicial
+# terms, the interval coresponds to the number of index entries that
+# are skipped between taking each sample. All the sampled entries
+# must fit in memory. Generally, a value between 128 and 512 here
+# coupled with a large key cache size on CFs results in the best trade
+# offs. This value is not often changed, however if you have many
+# very small rows (many to an OS page), then increasing this will
+# often lower memory usage without a impact on performance.
index_interval: 128
# Enable or disable inter-node encryption
Modified:
trunk/amdatu-cassandra/cassandra-client/src/test/resources/cassandra.yaml
==============================================================================
--- trunk/amdatu-cassandra/cassandra-client/src/test/resources/cassandra.yaml
(original)
+++ trunk/amdatu-cassandra/cassandra-client/src/test/resources/cassandra.yaml
Fri Feb 10 14:49:22 2012
@@ -15,10 +15,10 @@
# Cassandra storage config YAML
-# NOTE:
-# See http://wiki.apache.org/cassandra/StorageConfiguration for
-# full explanations of configuration directives
-# /NOTE
+#NOTE !!!!!!!! NOTE
+# See http://wiki.apache.org/cassandra/StorageConfiguration for
+# full explanations of configuration directives
+#NOTE !!!!!!!! NOTE
# The name of the cluster. This is mainly used to prevent machines in
# one logical cluster from joining another.
@@ -34,7 +34,7 @@
# the heaviest-loaded existing node. If there is no load information
# available, such as is the case with a new cluster, it will pick
# a random token, which will lead to hot spots.
-initial_token:
+initial_token: 0
# Set to true to make new [non-seed] nodes automatically migrate data
# to themselves from the pre-existing nodes in the cluster. Defaults
@@ -49,6 +49,8 @@
# this defines the maximum amount of time a dead host will have hints
# generated. After it has been dead this long, hints will be dropped.
max_hint_window_in_ms: 3600000 # one hour
+# Sleep this long after delivering each row or row fragment
+hinted_handoff_throttle_delay_in_ms: 50
# authentication backend, implementing IAuthenticator; used to identify users
authenticator: org.apache.cassandra.auth.AllowAllAuthenticator
@@ -96,26 +98,30 @@
# commitlog_sync may be either "periodic" or "batch."
# When in batch mode, Cassandra won't ack writes until the commit log
# has been fsynced to disk. It will wait up to
-# CommitLogSyncBatchWindowInMS milliseconds for other writes, before
+# commitlog_sync_batch_window_in_ms milliseconds for other writes, before
# performing the sync.
-commitlog_sync: periodic
-
-# the other option is "timed," where writes may be acked immediately
+#
+# commitlog_sync: batch
+# commitlog_sync_batch_window_in_ms: 50
+#
+# the other option is "periodic" where writes may be acked immediately
# and the CommitLog is simply synced every commitlog_sync_period_in_ms
# milliseconds.
+commitlog_sync: periodic
commitlog_sync_period_in_ms: 10000
# any class that implements the SeedProvider interface and has a constructor
that takes a Map<String, String> of
# parameters will do.
seed_provider:
- # Addresses of hosts that are deemed contact points.
- # Cassandra nodes use this list of hosts to find each other and learn
- # the topology of the ring. You must change this if you are running
- # multiple nodes!
- - class_name: org.apache.cassandra.locator.SimpleSeedProvider
- parameters:
- # seeds is actually a comma-delimited list of addresses.
- - seeds: "127.0.0.1"
+ # Addresses of hosts that are deemed contact points.
+ # Cassandra nodes use this list of hosts to find each other and learn
+ # the topology of the ring. You must change this if you are running
+ # multiple nodes!
+ - class_name: org.apache.cassandra.locator.SimpleSeedProvider
+ parameters:
+ # seeds is actually a comma-delimited list of addresses.
+ # Ex: "<ip1>,<ip2>,<ip3>"
+ - seeds: "127.0.0.1"
# emergency pressure valve: each time heap usage after a full (CMS)
# garbage collection is above this fraction of the max, Cassandra will
@@ -142,16 +148,6 @@
reduce_cache_sizes_at: 0.85
reduce_cache_capacity_to: 0.6
-# Access mode. mmapped i/o is substantially faster, but only practical on
-# a 64bit machine (which notably does not include EC2 "small" instances)
-# or relatively small datasets. "auto", the safe choice, will enable
-# mmapping on a 64bit JVM. Other values are "mmap", "mmap_index_only"
-# (which may allow you to get part of the benefits of mmap on a 32bit
-# machine by mmapping only index files) and "standard".
-# (The buffer size settings that follow only apply to standard,
-# non-mmapped i/o.)
-disk_access_mode: auto
-
# For workloads with more data than can fit in memory, Cassandra's
# bottleneck will be reads that need to fetch data from
# disk. "concurrent_reads" should be set to (16 * number_of_drives) in
@@ -169,7 +165,7 @@
# the older, per-ColumnFamily memtable flush thresholds.
# If omitted, Cassandra will set it to 1/3 of the heap.
# If set to 0, only the old flush thresholds are used.
-# memtable_total_space_in_mb: 2048
+memtable_total_space_in_mb: 0
# This sets the amount of memtable flush writer threads. These will
# be blocked by disk io, and each one will hold a memtable in memory
@@ -178,6 +174,11 @@
# By default this will be set to the amount of data directories defined.
#memtable_flush_writers: 1
+# the number of full memtables to allow pending flush, that is,
+# waiting for a writer thread. At a minimum, this should be set to
+# the maximum number of secondary indexes created on a single CF.
+memtable_flush_queue_size: 4
+
# Buffer size to use when performing contiguous column slices.
# Increase this to the size of the column slices you typically perform
sliced_buffer_size_in_kb: 64
@@ -204,12 +205,46 @@
# Leaving this blank has the same effect it does for ListenAddress,
# (i.e. it will be based on the configured hostname of the node).
rpc_address: 127.0.0.1
+
# port for Thrift to listen for clients on
rpc_port: 9161
# enable or disable keepalive on rpc connections
rpc_keepalive: true
+# Cassandra provides you with a variety of options for RPC Server
+# sync -> Creates one thread per connection but with a configurable number of
+# threads. This can be expensive in memory used for thread stack for
+# a large enough number of clients. (Hence, connection pooling is
+# very, very strongly recommended.)
+#
+# async -> Nonblocking server implementation with one thread to serve
+# rpc connections. This is not recommended for high throughput use
+# cases.
+#
+# hsha -> half sync and half async implementation with configurable number
+# of worker threads (For managing connections). IO Management is
+# done by a set of threads currently equal to the number of
+# processors in the system. The number of threads in the threadpool
+# is configured via rpc_min_threads and rpc_max_threads. (Connection
+# pooling is strongly recommended in this case too.)
+
+rpc_server_type: sync
+
+# Uncomment rpc_min|max|thread to set request pool size.
+# You would primarily set max for the sync server to safeguard against
+# misbehaved clients; if you do hit the max, Cassandra will block until one
+# disconnects before accepting more. The defaults are min of 16 and max
+# unlimited.
+#
+# For the Hsha server, you would set the max so that a fair amount of resources
+# are provided to the other working threads on the server.
+#
+# This configuration is not used for the async server.
+#
+# rpc_min_threads: 16
+# rpc_max_threads: 2048
+
# uncomment to set socket buffer sizes on rpc connections
# rpc_send_buff_size_in_bytes:
# rpc_recv_buff_size_in_bytes:
@@ -223,6 +258,12 @@
# internal thrift overhead.
thrift_max_message_length_in_mb: 16
+# Set to true to have Cassandra create a hard link to each sstable
+# flushed or streamed locally in a backups/ subdirectory of the
+# Keyspace data. Removing these links is the operator's
+# responsibility.
+incremental_backups: false
+
# Whether or not to take a snapshot before each compaction. Be
# careful using this option, since Cassandra won't clean up the
# snapshots for you. Mostly useful if you're paranoid when there
@@ -230,13 +271,9 @@
snapshot_before_compaction: false
# change this to increase the compaction thread's priority. In java, 1 is the
-# lowest priority and that is our default.
+# lowest priority and that is our default. The highest allowed is 5.
# compaction_thread_priority: 1
-# The threshold size in megabytes the binary memtable must grow to,
-# before it's submitted for flushing to disk.
-binary_memtable_throughput_in_mb: 256
-
# Add column indexes to a row after its contents reach this size.
# Increase if your column values are large, or if you have a very large
# number of columns. The competing causes are, Cassandra has to
@@ -251,13 +288,15 @@
# will be logged specifying the row key.
in_memory_compaction_limit_in_mb: 64
-# Number of compaction threads. This default to the number of processors,
-# enabling multiple compactions to execute at once. Using more than one
-# thread is highly recommended to preserve read performance in a mixed
-# read/write workload as this avoids sstables from accumulating during long
-# running compactions. The default is usually fine and if you experience
-# problems with compaction running too slowly or too fast, you should look at
+# Number of simultaneous compactions to allow, NOT including
+# validation "compactions" for anti-entropy repair. This defaults to
+# the number of cores. This can help preserve read performance in a
+# mixed read/write workload, by mitigating the tendency of small
+# sstables to accumulate during a single long running compactions. The
+# default is usually fine and if you experience problems with
+# compaction running too slowly or too fast, you should look at
# compaction_throughput_mb_per_sec first.
+#
# Uncomment to make compaction mono-threaded.
#concurrent_compactors: 1
@@ -265,9 +304,15 @@
# system. The faster you insert data, the faster you need to compact in
# order to keep the sstable count down, but in general, setting this to
# 16 to 32 times the rate you are inserting data is more than sufficient.
-# Setting this to 0 disables throttling.
+# Setting this to 0 disables throttling. Note that this account for all types
+# of compaction, including validation compaction.
compaction_throughput_mb_per_sec: 16
+# Track cached row keys during compaction, and re-cache their new
+# positions in the compacted sstable. Disable if you use really large
+# key caches.
+compaction_preheat_key_cache: true
+
# Time to wait for a reply from other nodes before failing the command
rpc_timeout_in_ms: 10000
@@ -351,9 +396,16 @@
# the request scheduling. Currently the only valid option is keyspace.
# request_scheduler_id: keyspace
-# The Index Interval determines how large the sampling of row keys
-# is for a given SSTable. The larger the sampling, the more effective
-# the index is at the cost of space.
+# index_interval controls the sampling of entries from the primrary
+# row index in terms of space versus time. The larger the interval,
+# the smaller and less effective the sampling will be. In technicial
+# terms, the interval coresponds to the number of index entries that
+# are skipped between taking each sample. All the sampled entries
+# must fit in memory. Generally, a value between 128 and 512 here
+# coupled with a large key cache size on CFs results in the best trade
+# offs. This value is not often changed, however if you have many
+# very small rows (many to an OS page), then increasing this will
+# often lower memory usage without a impact on performance.
index_interval: 128
# Enable or disable inter-node encryption
Modified:
trunk/amdatu-cassandra/cassandra-persistencemanager-hector/src/main/java/org/amdatu/cassandra/persistencemanager/hector/HectorCassandraPersistenceManager.java
==============================================================================
---
trunk/amdatu-cassandra/cassandra-persistencemanager-hector/src/main/java/org/amdatu/cassandra/persistencemanager/hector/HectorCassandraPersistenceManager.java
(original)
+++
trunk/amdatu-cassandra/cassandra-persistencemanager-hector/src/main/java/org/amdatu/cassandra/persistencemanager/hector/HectorCassandraPersistenceManager.java
Fri Feb 10 14:49:22 2012
@@ -45,8 +45,6 @@
* The expected return type of the value contained by the specified
column
* @param rowKey
* The key of the row to retrieve
- * @param superColumn
- * The name of the superColumns to retrieve
* @param clazz
* The generic type T of all values being retrieved (i.e.
String.class).
* @return Super row with the specified row key in the specified
ColumnFamily containing all SuperColumns
Modified:
trunk/amdatu-cassandra/cassandra-persistencemanager/src/main/java/org/amdatu/cassandra/persistencemanager/CassandraPersistenceManager.java
==============================================================================
---
trunk/amdatu-cassandra/cassandra-persistencemanager/src/main/java/org/amdatu/cassandra/persistencemanager/CassandraPersistenceManager.java
(original)
+++
trunk/amdatu-cassandra/cassandra-persistencemanager/src/main/java/org/amdatu/cassandra/persistencemanager/CassandraPersistenceManager.java
Fri Feb 10 14:49:22 2012
@@ -87,7 +87,7 @@
* Name of the ColumnFamily to check
* @param rowKey
* The row key to check
- * @param superColumName
+ * @param superColumnName
* The name of the SuperColumn to search for. Must be null if the
specified ColumnFamily is not of
* type 'Super' and not null if it is
* @param columnName
@@ -215,7 +215,7 @@
* @param superColumn
* the name of the super column to retrieve the value from (must be
null in case the ColumnFamily is
* of type "Standard").
- * @param columnName
+ * @param column
* The name of the column to retrieve the value from.
* @param expectedClazz
* the expected return type. Must be one of; String.class,
byte[].class, Double.class,
Modified:
trunk/amdatu-cassandra/cassandra-stores/cassandra-store-consumer/pom.xml
==============================================================================
--- trunk/amdatu-cassandra/cassandra-stores/cassandra-store-consumer/pom.xml
(original)
+++ trunk/amdatu-cassandra/cassandra-stores/cassandra-store-consumer/pom.xml
Fri Feb 10 14:49:22 2012
@@ -84,7 +84,7 @@
<Bundle-Activator>org.amdatu.cassandra.store.consumer.osgi.Activator</Bundle-Activator>
<Bundle-SymbolicName>org.amdatu.cassandra.store.consumer</Bundle-SymbolicName>
<Embed-Dependency>*;scope=compile</Embed-Dependency>
- <Embed-Transitive>true</Embed-Transitive>
+ <Embed-Transitive>false</Embed-Transitive>
<Import-Package>
!org.joda.time,
org.amdatu.cassandra.listener;version="[1.0,2.0)",
Modified:
trunk/amdatu-cassandra/cassandra-stores/cassandra-store-useradmin/pom.xml
==============================================================================
--- trunk/amdatu-cassandra/cassandra-stores/cassandra-store-useradmin/pom.xml
(original)
+++ trunk/amdatu-cassandra/cassandra-stores/cassandra-store-useradmin/pom.xml
Fri Feb 10 14:49:22 2012
@@ -86,7 +86,6 @@
*
</Import-Package>
<Export-Package>
- org.amdatu.cassandra.store.useradmin;version="1.0.0"
</Export-Package>
<Embed-Dependency>*;scope=compile</Embed-Dependency>
<Embed-Transitive>true</Embed-Transitive>
Modified: trunk/amdatu-cassandra/release/pom.xml
==============================================================================
--- trunk/amdatu-cassandra/release/pom.xml (original)
+++ trunk/amdatu-cassandra/release/pom.xml Fri Feb 10 14:49:22 2012
@@ -23,7 +23,7 @@
<version>0.2.3-SNAPSHOT</version>
</parent>
<artifactId>amdatu-cassandra</artifactId>
- <packaging>pom</packaging>
+ <packaging>jar</packaging>
<name>Amdatu Cassandra - release</name>
<description>Release module for the Amdatu Cassandra project.</description>
Modified:
trunk/amdatu-cassandra/test-performance/test-releases/cassandra-0.2.0/pom.xml
==============================================================================
---
trunk/amdatu-cassandra/test-performance/test-releases/cassandra-0.2.0/pom.xml
(original)
+++
trunk/amdatu-cassandra/test-performance/test-releases/cassandra-0.2.0/pom.xml
Fri Feb 10 14:49:22 2012
@@ -23,7 +23,7 @@
<version>0.2.3-SNAPSHOT</version>
</parent>
<artifactId>org.amdatu.cassandra.test.performance.release-0.2.0</artifactId>
- <packaging>pom</packaging>
+ <packaging>jar</packaging>
<name>Amdatu Cassandra - Performance Test release - 0.2.0</name>
<properties>
Modified:
trunk/amdatu-cassandra/test-performance/test-releases/cassandra-0.2.1/pom.xml
==============================================================================
---
trunk/amdatu-cassandra/test-performance/test-releases/cassandra-0.2.1/pom.xml
(original)
+++
trunk/amdatu-cassandra/test-performance/test-releases/cassandra-0.2.1/pom.xml
Fri Feb 10 14:49:22 2012
@@ -23,7 +23,7 @@
<version>0.2.3-SNAPSHOT</version>
</parent>
<artifactId>org.amdatu.cassandra.test.performance.release-0.2.1</artifactId>
- <packaging>pom</packaging>
+ <packaging>jar</packaging>
<name>Amdatu Cassandra - Performance Test release - 0.2.1</name>
<properties>
Modified:
trunk/amdatu-cassandra/test-unit/framework-hector/src/main/resources/cassandra.yaml
==============================================================================
---
trunk/amdatu-cassandra/test-unit/framework-hector/src/main/resources/cassandra.yaml
(original)
+++
trunk/amdatu-cassandra/test-unit/framework-hector/src/main/resources/cassandra.yaml
Fri Feb 10 14:49:22 2012
@@ -15,10 +15,10 @@
# Cassandra storage config YAML
-# NOTE:
-# See http://wiki.apache.org/cassandra/StorageConfiguration for
-# full explanations of configuration directives
-# /NOTE
+#NOTE !!!!!!!! NOTE
+# See http://wiki.apache.org/cassandra/StorageConfiguration for
+# full explanations of configuration directives
+#NOTE !!!!!!!! NOTE
# The name of the cluster. This is mainly used to prevent machines in
# one logical cluster from joining another.
@@ -34,7 +34,7 @@
# the heaviest-loaded existing node. If there is no load information
# available, such as is the case with a new cluster, it will pick
# a random token, which will lead to hot spots.
-initial_token:
+initial_token: 0
# Set to true to make new [non-seed] nodes automatically migrate data
# to themselves from the pre-existing nodes in the cluster. Defaults
@@ -49,6 +49,8 @@
# this defines the maximum amount of time a dead host will have hints
# generated. After it has been dead this long, hints will be dropped.
max_hint_window_in_ms: 3600000 # one hour
+# Sleep this long after delivering each row or row fragment
+hinted_handoff_throttle_delay_in_ms: 50
# authentication backend, implementing IAuthenticator; used to identify users
authenticator: org.apache.cassandra.auth.AllowAllAuthenticator
@@ -96,26 +98,30 @@
# commitlog_sync may be either "periodic" or "batch."
# When in batch mode, Cassandra won't ack writes until the commit log
# has been fsynced to disk. It will wait up to
-# CommitLogSyncBatchWindowInMS milliseconds for other writes, before
+# commitlog_sync_batch_window_in_ms milliseconds for other writes, before
# performing the sync.
-commitlog_sync: periodic
-
-# the other option is "timed," where writes may be acked immediately
+#
+# commitlog_sync: batch
+# commitlog_sync_batch_window_in_ms: 50
+#
+# the other option is "periodic" where writes may be acked immediately
# and the CommitLog is simply synced every commitlog_sync_period_in_ms
# milliseconds.
+commitlog_sync: periodic
commitlog_sync_period_in_ms: 10000
# any class that implements the SeedProvider interface and has a constructor
that takes a Map<String, String> of
# parameters will do.
seed_provider:
- # Addresses of hosts that are deemed contact points.
- # Cassandra nodes use this list of hosts to find each other and learn
- # the topology of the ring. You must change this if you are running
- # multiple nodes!
- - class_name: org.apache.cassandra.locator.SimpleSeedProvider
- parameters:
- # seeds is actually a comma-delimited list of addresses.
- - seeds: "127.0.0.1"
+ # Addresses of hosts that are deemed contact points.
+ # Cassandra nodes use this list of hosts to find each other and learn
+ # the topology of the ring. You must change this if you are running
+ # multiple nodes!
+ - class_name: org.apache.cassandra.locator.SimpleSeedProvider
+ parameters:
+ # seeds is actually a comma-delimited list of addresses.
+ # Ex: "<ip1>,<ip2>,<ip3>"
+ - seeds: "127.0.0.1"
# emergency pressure valve: each time heap usage after a full (CMS)
# garbage collection is above this fraction of the max, Cassandra will
@@ -142,16 +148,6 @@
reduce_cache_sizes_at: 0.85
reduce_cache_capacity_to: 0.6
-# Access mode. mmapped i/o is substantially faster, but only practical on
-# a 64bit machine (which notably does not include EC2 "small" instances)
-# or relatively small datasets. "auto", the safe choice, will enable
-# mmapping on a 64bit JVM. Other values are "mmap", "mmap_index_only"
-# (which may allow you to get part of the benefits of mmap on a 32bit
-# machine by mmapping only index files) and "standard".
-# (The buffer size settings that follow only apply to standard,
-# non-mmapped i/o.)
-disk_access_mode: auto
-
# For workloads with more data than can fit in memory, Cassandra's
# bottleneck will be reads that need to fetch data from
# disk. "concurrent_reads" should be set to (16 * number_of_drives) in
@@ -169,7 +165,7 @@
# the older, per-ColumnFamily memtable flush thresholds.
# If omitted, Cassandra will set it to 1/3 of the heap.
# If set to 0, only the old flush thresholds are used.
-# memtable_total_space_in_mb: 2048
+memtable_total_space_in_mb: 0
# This sets the amount of memtable flush writer threads. These will
# be blocked by disk io, and each one will hold a memtable in memory
@@ -178,6 +174,11 @@
# By default this will be set to the amount of data directories defined.
#memtable_flush_writers: 1
+# the number of full memtables to allow pending flush, that is,
+# waiting for a writer thread. At a minimum, this should be set to
+# the maximum number of secondary indexes created on a single CF.
+memtable_flush_queue_size: 4
+
# Buffer size to use when performing contiguous column slices.
# Increase this to the size of the column slices you typically perform
sliced_buffer_size_in_kb: 64
@@ -204,12 +205,46 @@
# Leaving this blank has the same effect it does for ListenAddress,
# (i.e. it will be based on the configured hostname of the node).
rpc_address: 127.0.0.1
+
# port for Thrift to listen for clients on
rpc_port: 9161
# enable or disable keepalive on rpc connections
rpc_keepalive: true
+# Cassandra provides you with a variety of options for RPC Server
+# sync -> Creates one thread per connection but with a configurable number of
+# threads. This can be expensive in memory used for thread stack for
+# a large enough number of clients. (Hence, connection pooling is
+# very, very strongly recommended.)
+#
+# async -> Nonblocking server implementation with one thread to serve
+# rpc connections. This is not recommended for high throughput use
+# cases.
+#
+# hsha -> half sync and half async implementation with configurable number
+# of worker threads (For managing connections). IO Management is
+# done by a set of threads currently equal to the number of
+# processors in the system. The number of threads in the threadpool
+# is configured via rpc_min_threads and rpc_max_threads. (Connection
+# pooling is strongly recommended in this case too.)
+
+rpc_server_type: sync
+
+# Uncomment rpc_min|max|thread to set request pool size.
+# You would primarily set max for the sync server to safeguard against
+# misbehaved clients; if you do hit the max, Cassandra will block until one
+# disconnects before accepting more. The defaults are min of 16 and max
+# unlimited.
+#
+# For the Hsha server, you would set the max so that a fair amount of resources
+# are provided to the other working threads on the server.
+#
+# This configuration is not used for the async server.
+#
+# rpc_min_threads: 16
+# rpc_max_threads: 2048
+
# uncomment to set socket buffer sizes on rpc connections
# rpc_send_buff_size_in_bytes:
# rpc_recv_buff_size_in_bytes:
@@ -223,6 +258,12 @@
# internal thrift overhead.
thrift_max_message_length_in_mb: 16
+# Set to true to have Cassandra create a hard link to each sstable
+# flushed or streamed locally in a backups/ subdirectory of the
+# Keyspace data. Removing these links is the operator's
+# responsibility.
+incremental_backups: false
+
# Whether or not to take a snapshot before each compaction. Be
# careful using this option, since Cassandra won't clean up the
# snapshots for you. Mostly useful if you're paranoid when there
@@ -230,13 +271,9 @@
snapshot_before_compaction: false
# change this to increase the compaction thread's priority. In java, 1 is the
-# lowest priority and that is our default.
+# lowest priority and that is our default. The highest allowed is 5.
# compaction_thread_priority: 1
-# The threshold size in megabytes the binary memtable must grow to,
-# before it's submitted for flushing to disk.
-binary_memtable_throughput_in_mb: 256
-
# Add column indexes to a row after its contents reach this size.
# Increase if your column values are large, or if you have a very large
# number of columns. The competing causes are, Cassandra has to
@@ -251,13 +288,15 @@
# will be logged specifying the row key.
in_memory_compaction_limit_in_mb: 64
-# Number of compaction threads. This default to the number of processors,
-# enabling multiple compactions to execute at once. Using more than one
-# thread is highly recommended to preserve read performance in a mixed
-# read/write workload as this avoids sstables from accumulating during long
-# running compactions. The default is usually fine and if you experience
-# problems with compaction running too slowly or too fast, you should look at
+# Number of simultaneous compactions to allow, NOT including
+# validation "compactions" for anti-entropy repair. This defaults to
+# the number of cores. This can help preserve read performance in a
+# mixed read/write workload, by mitigating the tendency of small
+# sstables to accumulate during a single long running compactions. The
+# default is usually fine and if you experience problems with
+# compaction running too slowly or too fast, you should look at
# compaction_throughput_mb_per_sec first.
+#
# Uncomment to make compaction mono-threaded.
#concurrent_compactors: 1
@@ -265,9 +304,15 @@
# system. The faster you insert data, the faster you need to compact in
# order to keep the sstable count down, but in general, setting this to
# 16 to 32 times the rate you are inserting data is more than sufficient.
-# Setting this to 0 disables throttling.
+# Setting this to 0 disables throttling. Note that this account for all types
+# of compaction, including validation compaction.
compaction_throughput_mb_per_sec: 16
+# Track cached row keys during compaction, and re-cache their new
+# positions in the compacted sstable. Disable if you use really large
+# key caches.
+compaction_preheat_key_cache: true
+
# Time to wait for a reply from other nodes before failing the command
rpc_timeout_in_ms: 10000
@@ -351,9 +396,16 @@
# the request scheduling. Currently the only valid option is keyspace.
# request_scheduler_id: keyspace
-# The Index Interval determines how large the sampling of row keys
-# is for a given SSTable. The larger the sampling, the more effective
-# the index is at the cost of space.
+# index_interval controls the sampling of entries from the primrary
+# row index in terms of space versus time. The larger the interval,
+# the smaller and less effective the sampling will be. In technicial
+# terms, the interval coresponds to the number of index entries that
+# are skipped between taking each sample. All the sampled entries
+# must fit in memory. Generally, a value between 128 and 512 here
+# coupled with a large key cache size on CFs results in the best trade
+# offs. This value is not often changed, however if you have many
+# very small rows (many to an OS page), then increasing this will
+# often lower memory usage without a impact on performance.
index_interval: 128
# Enable or disable inter-node encryption
Modified:
trunk/amdatu-cassandra/test-unit/framework/src/main/resources/cassandra.yaml
==============================================================================
---
trunk/amdatu-cassandra/test-unit/framework/src/main/resources/cassandra.yaml
(original)
+++
trunk/amdatu-cassandra/test-unit/framework/src/main/resources/cassandra.yaml
Fri Feb 10 14:49:22 2012
@@ -15,10 +15,10 @@
# Cassandra storage config YAML
-# NOTE:
-# See http://wiki.apache.org/cassandra/StorageConfiguration for
-# full explanations of configuration directives
-# /NOTE
+#NOTE !!!!!!!! NOTE
+# See http://wiki.apache.org/cassandra/StorageConfiguration for
+# full explanations of configuration directives
+#NOTE !!!!!!!! NOTE
# The name of the cluster. This is mainly used to prevent machines in
# one logical cluster from joining another.
@@ -34,7 +34,7 @@
# the heaviest-loaded existing node. If there is no load information
# available, such as is the case with a new cluster, it will pick
# a random token, which will lead to hot spots.
-initial_token:
+initial_token: 0
# Set to true to make new [non-seed] nodes automatically migrate data
# to themselves from the pre-existing nodes in the cluster. Defaults
@@ -49,6 +49,8 @@
# this defines the maximum amount of time a dead host will have hints
# generated. After it has been dead this long, hints will be dropped.
max_hint_window_in_ms: 3600000 # one hour
+# Sleep this long after delivering each row or row fragment
+hinted_handoff_throttle_delay_in_ms: 50
# authentication backend, implementing IAuthenticator; used to identify users
authenticator: org.apache.cassandra.auth.AllowAllAuthenticator
@@ -96,26 +98,30 @@
# commitlog_sync may be either "periodic" or "batch."
# When in batch mode, Cassandra won't ack writes until the commit log
# has been fsynced to disk. It will wait up to
-# CommitLogSyncBatchWindowInMS milliseconds for other writes, before
+# commitlog_sync_batch_window_in_ms milliseconds for other writes, before
# performing the sync.
-commitlog_sync: periodic
-
-# the other option is "timed," where writes may be acked immediately
+#
+# commitlog_sync: batch
+# commitlog_sync_batch_window_in_ms: 50
+#
+# the other option is "periodic" where writes may be acked immediately
# and the CommitLog is simply synced every commitlog_sync_period_in_ms
# milliseconds.
+commitlog_sync: periodic
commitlog_sync_period_in_ms: 10000
# any class that implements the SeedProvider interface and has a constructor
that takes a Map<String, String> of
# parameters will do.
seed_provider:
- # Addresses of hosts that are deemed contact points.
- # Cassandra nodes use this list of hosts to find each other and learn
- # the topology of the ring. You must change this if you are running
- # multiple nodes!
- - class_name: org.apache.cassandra.locator.SimpleSeedProvider
- parameters:
- # seeds is actually a comma-delimited list of addresses.
- - seeds: "127.0.0.1"
+ # Addresses of hosts that are deemed contact points.
+ # Cassandra nodes use this list of hosts to find each other and learn
+ # the topology of the ring. You must change this if you are running
+ # multiple nodes!
+ - class_name: org.apache.cassandra.locator.SimpleSeedProvider
+ parameters:
+ # seeds is actually a comma-delimited list of addresses.
+ # Ex: "<ip1>,<ip2>,<ip3>"
+ - seeds: "127.0.0.1"
# emergency pressure valve: each time heap usage after a full (CMS)
# garbage collection is above this fraction of the max, Cassandra will
@@ -142,16 +148,6 @@
reduce_cache_sizes_at: 0.85
reduce_cache_capacity_to: 0.6
-# Access mode. mmapped i/o is substantially faster, but only practical on
-# a 64bit machine (which notably does not include EC2 "small" instances)
-# or relatively small datasets. "auto", the safe choice, will enable
-# mmapping on a 64bit JVM. Other values are "mmap", "mmap_index_only"
-# (which may allow you to get part of the benefits of mmap on a 32bit
-# machine by mmapping only index files) and "standard".
-# (The buffer size settings that follow only apply to standard,
-# non-mmapped i/o.)
-disk_access_mode: auto
-
# For workloads with more data than can fit in memory, Cassandra's
# bottleneck will be reads that need to fetch data from
# disk. "concurrent_reads" should be set to (16 * number_of_drives) in
@@ -169,7 +165,7 @@
# the older, per-ColumnFamily memtable flush thresholds.
# If omitted, Cassandra will set it to 1/3 of the heap.
# If set to 0, only the old flush thresholds are used.
-# memtable_total_space_in_mb: 2048
+memtable_total_space_in_mb: 0
# This sets the amount of memtable flush writer threads. These will
# be blocked by disk io, and each one will hold a memtable in memory
@@ -178,6 +174,11 @@
# By default this will be set to the amount of data directories defined.
#memtable_flush_writers: 1
+# the number of full memtables to allow pending flush, that is,
+# waiting for a writer thread. At a minimum, this should be set to
+# the maximum number of secondary indexes created on a single CF.
+memtable_flush_queue_size: 4
+
# Buffer size to use when performing contiguous column slices.
# Increase this to the size of the column slices you typically perform
sliced_buffer_size_in_kb: 64
@@ -204,12 +205,46 @@
# Leaving this blank has the same effect it does for ListenAddress,
# (i.e. it will be based on the configured hostname of the node).
rpc_address: 127.0.0.1
+
# port for Thrift to listen for clients on
rpc_port: 9161
# enable or disable keepalive on rpc connections
rpc_keepalive: true
+# Cassandra provides you with a variety of options for RPC Server
+# sync -> Creates one thread per connection but with a configurable number of
+# threads. This can be expensive in memory used for thread stack for
+# a large enough number of clients. (Hence, connection pooling is
+# very, very strongly recommended.)
+#
+# async -> Nonblocking server implementation with one thread to serve
+# rpc connections. This is not recommended for high throughput use
+# cases.
+#
+# hsha -> half sync and half async implementation with configurable number
+# of worker threads (For managing connections). IO Management is
+# done by a set of threads currently equal to the number of
+# processors in the system. The number of threads in the threadpool
+# is configured via rpc_min_threads and rpc_max_threads. (Connection
+# pooling is strongly recommended in this case too.)
+
+rpc_server_type: sync
+
+# Uncomment rpc_min|max|thread to set request pool size.
+# You would primarily set max for the sync server to safeguard against
+# misbehaved clients; if you do hit the max, Cassandra will block until one
+# disconnects before accepting more. The defaults are min of 16 and max
+# unlimited.
+#
+# For the Hsha server, you would set the max so that a fair amount of resources
+# are provided to the other working threads on the server.
+#
+# This configuration is not used for the async server.
+#
+# rpc_min_threads: 16
+# rpc_max_threads: 2048
+
# uncomment to set socket buffer sizes on rpc connections
# rpc_send_buff_size_in_bytes:
# rpc_recv_buff_size_in_bytes:
@@ -223,6 +258,12 @@
# internal thrift overhead.
thrift_max_message_length_in_mb: 16
+# Set to true to have Cassandra create a hard link to each sstable
+# flushed or streamed locally in a backups/ subdirectory of the
+# Keyspace data. Removing these links is the operator's
+# responsibility.
+incremental_backups: false
+
# Whether or not to take a snapshot before each compaction. Be
# careful using this option, since Cassandra won't clean up the
# snapshots for you. Mostly useful if you're paranoid when there
@@ -230,13 +271,9 @@
snapshot_before_compaction: false
# change this to increase the compaction thread's priority. In java, 1 is the
-# lowest priority and that is our default.
+# lowest priority and that is our default. The highest allowed is 5.
# compaction_thread_priority: 1
-# The threshold size in megabytes the binary memtable must grow to,
-# before it's submitted for flushing to disk.
-binary_memtable_throughput_in_mb: 256
-
# Add column indexes to a row after its contents reach this size.
# Increase if your column values are large, or if you have a very large
# number of columns. The competing causes are, Cassandra has to
@@ -251,13 +288,15 @@
# will be logged specifying the row key.
in_memory_compaction_limit_in_mb: 64
-# Number of compaction threads. This default to the number of processors,
-# enabling multiple compactions to execute at once. Using more than one
-# thread is highly recommended to preserve read performance in a mixed
-# read/write workload as this avoids sstables from accumulating during long
-# running compactions. The default is usually fine and if you experience
-# problems with compaction running too slowly or too fast, you should look at
+# Number of simultaneous compactions to allow, NOT including
+# validation "compactions" for anti-entropy repair. This defaults to
+# the number of cores. This can help preserve read performance in a
+# mixed read/write workload, by mitigating the tendency of small
+# sstables to accumulate during a single long running compactions. The
+# default is usually fine and if you experience problems with
+# compaction running too slowly or too fast, you should look at
# compaction_throughput_mb_per_sec first.
+#
# Uncomment to make compaction mono-threaded.
#concurrent_compactors: 1
@@ -265,9 +304,15 @@
# system. The faster you insert data, the faster you need to compact in
# order to keep the sstable count down, but in general, setting this to
# 16 to 32 times the rate you are inserting data is more than sufficient.
-# Setting this to 0 disables throttling.
+# Setting this to 0 disables throttling. Note that this account for all types
+# of compaction, including validation compaction.
compaction_throughput_mb_per_sec: 16
+# Track cached row keys during compaction, and re-cache their new
+# positions in the compacted sstable. Disable if you use really large
+# key caches.
+compaction_preheat_key_cache: true
+
# Time to wait for a reply from other nodes before failing the command
rpc_timeout_in_ms: 10000
@@ -351,9 +396,16 @@
# the request scheduling. Currently the only valid option is keyspace.
# request_scheduler_id: keyspace
-# The Index Interval determines how large the sampling of row keys
-# is for a given SSTable. The larger the sampling, the more effective
-# the index is at the cost of space.
+# index_interval controls the sampling of entries from the primrary
+# row index in terms of space versus time. The larger the interval,
+# the smaller and less effective the sampling will be. In technicial
+# terms, the interval coresponds to the number of index entries that
+# are skipped between taking each sample. All the sampled entries
+# must fit in memory. Generally, a value between 128 and 512 here
+# coupled with a large key cache size on CFs results in the best trade
+# offs. This value is not often changed, however if you have many
+# very small rows (many to an OS page), then increasing this will
+# often lower memory usage without a impact on performance.
index_interval: 128
# Enable or disable inter-node encryption
_______________________________________________
Amdatu-commits mailing list
[email protected]
http://lists.amdatu.org/mailman/listinfo/amdatu-commits
