Author: ivol37 at gmail.com
Date: Thu Jan 6 15:56:09 2011
New Revision: 571
Log:
[AMDATU-250] Added fallbacks for internal Cassandra errors. When adding a
keyspace or columnfamily fails, it is retried till a maxiumum number of 3
times. Most of the times the first retry will succeed.
Modified:
trunk/amdatu-cassandra/cassandra-application/src/main/java/org/amdatu/cassandra/application/service/CassandraDaemonServiceImpl.java
Modified:
trunk/amdatu-cassandra/cassandra-application/src/main/java/org/amdatu/cassandra/application/service/CassandraDaemonServiceImpl.java
==============================================================================
---
trunk/amdatu-cassandra/cassandra-application/src/main/java/org/amdatu/cassandra/application/service/CassandraDaemonServiceImpl.java
(original)
+++
trunk/amdatu-cassandra/cassandra-application/src/main/java/org/amdatu/cassandra/application/service/CassandraDaemonServiceImpl.java
Thu Jan 6 15:56:09 2011
@@ -85,7 +85,7 @@
m_daemon.activate();
m_cassandraServer = new CassandraServer();
- m_logService.log(LogService.LOG_INFO, "Cassandra Daemon started");
+ m_logService.log(LogService.LOG_INFO, "Cassandra Daemon started.
State="+ m_cassandraServer.state());
}
public void stop() {
@@ -126,8 +126,37 @@
public void addKeyspace(String name) throws InvalidRequestException,
TException {
List<CfDef> empty = new ArrayList<CfDef>();
KsDef ksDef = new KsDef(name, DEFAULT_PLACEMENT_STRATEGY,
DEFAULT_REPLICATION_FACTOR, empty);
- m_cassandraServer.system_add_keyspace(ksDef);
+ try {
+ m_cassandraServer.system_add_keyspace(ksDef);
+ } catch (InvalidRequestException e) {
+ // FIXME: This is a fallback for recovering from issue AMDATU-250.
Hopefully it will be fixed in
+ // subsequent Cassandra versions. In case added a columnfamily
fails on a InvalidRequestException
+ // we retry adding the column family for a maximum of 3 times.
Most of the times, the second try
+ // already succeeds.
+ int retry = 0;
+ boolean failure = true;
+ while (retry < 3 && failure) {
+ retry++;
+ m_logService.log(LogService.LOG_WARNING, "Failed to add
keyspace '" + name + "', retry " + retry);
+ try {
+ // Wait 1 second before retrying
+ Thread.sleep(1000);
+ }
+ catch (InterruptedException e2) {
+ }
+ try {
+ failure = false;
+ m_cassandraServer.system_add_keyspace(ksDef);
+ } catch (InvalidRequestException e2) {
+ if (retry >= 3) {
+ throw e2;
+ }
+ failure = true;
+ }
+ }
+ m_logService.log(LogService.LOG_WARNING, "Successfully recovered
from this Cassandra error, keyspace '" + name + "' added");
+ }
// Publish an event that a new keyspace has been added
Map<String, String> properties = new HashMap<String, String>();
properties.put(EVENT_ADMIN_KEYSPACE_ADDED, name);
@@ -173,9 +202,40 @@
cfDef.column_type = columnType;
cfDef.comparator_type = comparatorType;
cfDef.subcomparator_type = subComparatorType;
+ try {
+ m_cassandraServer.set_keyspace(keyspace);
+ m_cassandraServer.system_add_column_family(cfDef);
+ } catch (InvalidRequestException e) {
+ // FIXME: This is a fallback for recovering from issue AMDATU-250.
Hopefully it will be fixed in
+ // subsequent Cassandra versions. In case added a columnfamily
fails on a InvalidRequestException
+ // we retry adding the column family for a maximum of 3 times.
Most of the times, the second try
+ // already succeeds.
+ int retry = 0;
+ boolean failure = true;
+ while (retry < 3 && failure) {
+ retry++;
+ m_logService.log(LogService.LOG_WARNING, "Failed to add
ColumnFamily '" + cfName + "' to keyspace '" + keyspace
+ + "', retry " + retry);
+ try {
+ // Wait 1 second before retrying
+ Thread.sleep(1000);
+ }
+ catch (InterruptedException e2) {
+ }
- m_cassandraServer.set_keyspace(keyspace);
- m_cassandraServer.system_add_column_family(cfDef);
+ try {
+ failure = false;
+ m_cassandraServer.system_add_column_family(cfDef);
+ } catch (InvalidRequestException e2) {
+ if (retry >= 3) {
+ throw e2;
+ }
+ failure = true;
+ }
+ }
+ m_logService.log(LogService.LOG_WARNING, "Successfully recovered
from this Cassandra error, ColumnFamily '" + cfName
+ + "' added to keyspace '" + keyspace);
+ }
}
public boolean isColumnFamilyChanged(String keyspace, String cfName,
String columnType, String comparatorType,
