Repository: gora Updated Branches: refs/heads/master 64443cc75 -> 41aae36e9
GORA-347 Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/41aae36e Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/41aae36e Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/41aae36e Branch: refs/heads/master Commit: 41aae36e900fc730ac60af2bf8283012310728b9 Parents: 64443cc Author: Renato Marroquin <[email protected]> Authored: Wed Jun 25 19:40:24 2014 +0200 Committer: Renato Marroquin <[email protected]> Committed: Wed Jun 25 19:40:24 2014 +0200 ---------------------------------------------------------------------- .../gora/cassandra/store/CassandraClient.java | 40 +++++++++++---- .../gora/cassandra/store/CassandraMapping.java | 53 ++++++++++++++++++-- .../gora/cassandra/store/HectorUtils.java | 27 ++++++++-- .../src/test/conf/gora-cassandra-mapping.xml | 41 +++++++++++++-- 4 files changed, 141 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/41aae36e/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/CassandraClient.java ---------------------------------------------------------------------- diff --git a/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/CassandraClient.java b/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/CassandraClient.java index 7d5ebd2..50ec240 100644 --- a/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/CassandraClient.java +++ b/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/CassandraClient.java @@ -116,8 +116,13 @@ public class CassandraClient<K, T extends PersistentBase> { cfDef.setComparatorType(ComparatorType.BYTESTYPE); } - keyspaceDefinition = HFactory.createKeyspaceDefinition(this.cassandraMapping.getKeyspaceName(), - "org.apache.cassandra.locator.SimpleStrategy", 1, columnFamilyDefinitions); + keyspaceDefinition = HFactory.createKeyspaceDefinition( + this.cassandraMapping.getKeyspaceName(), + this.cassandraMapping.getKeyspaceReplicationStrategy(), + this.cassandraMapping.getKeyspaceReplicationFactor(), + columnFamilyDefinitions + ); + this.cluster.addKeyspace(keyspaceDefinition, true); // LOG.info("Keyspace '" + this.cassandraMapping.getKeyspaceName() + "' in cluster '" + this.cassandraMapping.getClusterName() + "' was created on host '" + this.cassandraMapping.getHostName() + "'"); @@ -174,20 +179,30 @@ public class CassandraClient<K, T extends PersistentBase> { */ public void addColumn(K key, String fieldName, Object value) { if (value == null) { + LOG.debug( "field:"+fieldName+", its value is null."); return; } ByteBuffer byteBuffer = toByteBuffer(value); String columnFamily = this.cassandraMapping.getFamily(fieldName); String columnName = this.cassandraMapping.getColumn(fieldName); - String ttlAttr = this.cassandraMapping.getColumnsAttribs().get(fieldName); - if (ttlAttr == null) - ttlAttr = CassandraMapping.DEFAULT_COLUMNS_TTL; - + if (columnName == null) { - LOG.warn("Column name is null for field=" + fieldName + " with value=" + value.toString()); - return; + LOG.warn("Column name is null for field=" + fieldName ); + return; + } + + if( LOG.isDebugEnabled() ) LOG.debug( "fieldName:"+fieldName +" columnName:" + columnName ); + + String ttlAttr = this.cassandraMapping.getColumnsAttribs().get(columnName); + + if ( null == ttlAttr ){ + ttlAttr = CassandraMapping.DEFAULT_COLUMNS_TTL; + if( LOG.isDebugEnabled() ) LOG.debug( "ttl was not set for field:" + fieldName + " .Using " + ttlAttr ); + } else { + if( LOG.isDebugEnabled() ) LOG.debug( "ttl for field:" + fieldName + " is " + ttlAttr ); } + synchronized(mutator) { HectorUtils.insertColumn(mutator, key, columnFamily, columnName, byteBuffer, ttlAttr); } @@ -230,9 +245,14 @@ public class CassandraClient<K, T extends PersistentBase> { String columnFamily = this.cassandraMapping.getFamily(fieldName); String superColumnName = this.cassandraMapping.getColumn(fieldName); - String ttlAttr = this.cassandraMapping.getColumnsAttribs().get(fieldName); - if (ttlAttr == null) + String ttlAttr = this.cassandraMapping.getColumnsAttribs().get(superColumnName); + if ( null == ttlAttr ) { ttlAttr = CassandraMapping.DEFAULT_COLUMNS_TTL; + if( LOG.isDebugEnabled() ) LOG.debug( "ttl was not set for field:" + fieldName + " .Using " + ttlAttr ); + } else { + if( LOG.isDebugEnabled() ) LOG.debug( "ttl for field:" + fieldName + " is " + ttlAttr ); + } + synchronized(mutator) { HectorUtils.insertSubColumn(mutator, key, columnFamily, superColumnName, columnName, byteBuffer, ttlAttr); } http://git-wip-us.apache.org/repos/asf/gora/blob/41aae36e/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/CassandraMapping.java ---------------------------------------------------------------------- diff --git a/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/CassandraMapping.java b/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/CassandraMapping.java index f8d5315..e3b1243 100644 --- a/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/CassandraMapping.java +++ b/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/CassandraMapping.java @@ -46,12 +46,19 @@ public class CassandraMapping { private static final String GCGRACE_SECONDS_ATTRIBUTE = "gc_grace_seconds"; private static final String COLUMNS_TTL_ATTRIBUTE = "ttl"; - public static final String DEFAULT_COLUMNS_TTL = "60"; - public static final int DEFAULT_GCGRACE_SECONDS = 30; + private static final String REPLICATION_FACTOR_ATTRIBUTE = "replication_factor"; + private static final String REPLICATION_STRATEGY_ATTRIBUTE = "placement_strategy"; + + public static final String DEFAULT_REPLICATION_FACTOR = "1"; + public static final String DEFAULT_REPLICATION_STRATEGY = "org.apache.cassandra.locator.SimpleStrategy"; + public static final String DEFAULT_COLUMNS_TTL = "0"; + public static final String DEFAULT_GCGRACE_SECONDS = "0"; private String hostName; private String clusterName; private String keyspaceName; + private String keyspaceStrategy; + private int keyspaceRF; /** @@ -106,6 +113,22 @@ public class CassandraMapping { public String getKeyspaceName() { return this.keyspaceName; } + + /** + * gets the replication strategy + * @return string class name to be used for strategy + */ + public String getKeyspaceReplicationStrategy() { + return this.keyspaceStrategy; + } + + /** + * gets the replication factor + * @return int replication factor + */ + public int getKeyspaceReplicationFactor() { + return this.keyspaceRF; + } /** * Primary class for loading Cassandra configuration from the 'MAPPING_FILE'. @@ -149,6 +172,26 @@ public class CassandraMapping { } } + // setting replication strategy + this.keyspaceStrategy = keyspace.getAttributeValue( REPLICATION_STRATEGY_ATTRIBUTE ); + if( null == this.keyspaceStrategy ) { + this.keyspaceStrategy = DEFAULT_REPLICATION_STRATEGY; + } + if( LOG.isDebugEnabled() ) { + LOG.debug( "setting Keyspace replication strategy to " + this.keyspaceStrategy ); + } + + // setting replication factor + String tmp = keyspace.getAttributeValue( REPLICATION_FACTOR_ATTRIBUTE ); + if( null == tmp ) { + tmp = DEFAULT_REPLICATION_FACTOR; + } + this.keyspaceRF = Integer.parseInt( tmp ); + if( LOG.isDebugEnabled() ) { + LOG.debug( "setting Keyspace replication factor to " + this.keyspaceRF ); + } + + // load column family definitions List<Element> elements = keyspace.getChildren(); for (Element element: elements) { @@ -166,7 +209,9 @@ public class CassandraMapping { String gcgrace_scs = element.getAttributeValue(GCGRACE_SECONDS_ATTRIBUTE); if (gcgrace_scs == null) { LOG.warn("Error locating gc_grace_seconds attribute for '" + familyName + "' column family"); - LOG.warn("Using default set to: " + DEFAULT_GCGRACE_SECONDS); + LOG.warn("Using gc_grace_seconds default value which is: " + DEFAULT_GCGRACE_SECONDS + + " and is viable ONLY FOR A SINGLE NODE CLUSTER"); + LOG.warn("please update the gora-cassandra-mapping.xml file to avoid seeing this warning"); } else { if (LOG.isDebugEnabled()) { LOG.debug("Located gc_grace_seconds: '" + gcgrace_scs + "'" ); @@ -191,7 +236,7 @@ public class CassandraMapping { cfDef.setComparatorType(ComparatorType.BYTESTYPE); cfDef.setDefaultValidationClass(ComparatorType.BYTESTYPE.getClassName()); - cfDef.setGcGraceSeconds(gcgrace_scs!=null?Integer.parseInt(gcgrace_scs):DEFAULT_GCGRACE_SECONDS); + cfDef.setGcGraceSeconds(Integer.parseInt( gcgrace_scs!=null?gcgrace_scs:DEFAULT_GCGRACE_SECONDS)); this.columnFamilyDefinitions.put(familyName, cfDef); } http://git-wip-us.apache.org/repos/asf/gora/blob/41aae36e/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/HectorUtils.java ---------------------------------------------------------------------- diff --git a/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/HectorUtils.java b/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/HectorUtils.java index da13f04..0c20cf7 100644 --- a/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/HectorUtils.java +++ b/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/HectorUtils.java @@ -48,15 +48,36 @@ public class HectorUtils<K,T extends Persistent> { public static<K> HColumn<ByteBuffer,ByteBuffer> createColumn(ByteBuffer name, ByteBuffer value, String ttlAttr) { - return HFactory.createColumn(name, value, ByteBufferSerializer.get(), ByteBufferSerializer.get()).setTtl(Integer.parseInt(ttlAttr)); + int ttl = Integer.parseInt(ttlAttr); + HColumn<ByteBuffer,ByteBuffer> col = HFactory.createColumn(name, value, ByteBufferSerializer.get(), ByteBufferSerializer.get()); + + if( 0 < ttl ) { + col.setTtl( ttl ); + } + + return col; } public static<K> HColumn<String,ByteBuffer> createColumn(String name, ByteBuffer value, String ttlAttr) { - return HFactory.createColumn(name, value, StringSerializer.get(), ByteBufferSerializer.get()).setTtl(Integer.parseInt(ttlAttr)); + int ttl = Integer.parseInt(ttlAttr); + HColumn<String,ByteBuffer> col = HFactory.createColumn(name, value, StringSerializer.get(), ByteBufferSerializer.get()); + + if( 0 < ttl ) { + col.setTtl( ttl ); + } + + return col; } public static<K> HColumn<Integer,ByteBuffer> createColumn(Integer name, ByteBuffer value, String ttlAttr) { - return HFactory.createColumn(name, value, IntegerSerializer.get(), ByteBufferSerializer.get()).setTtl(Integer.parseInt(ttlAttr)); + int ttl = Integer.parseInt(ttlAttr); + HColumn<Integer,ByteBuffer> col = HFactory.createColumn(name, value, IntegerSerializer.get(), ByteBufferSerializer.get()); + + if( 0 < ttl ) { + col.setTtl( ttl ); + } + + return col; } http://git-wip-us.apache.org/repos/asf/gora/blob/41aae36e/gora-cassandra/src/test/conf/gora-cassandra-mapping.xml ---------------------------------------------------------------------- diff --git a/gora-cassandra/src/test/conf/gora-cassandra-mapping.xml b/gora-cassandra/src/test/conf/gora-cassandra-mapping.xml index 70e411a..7aec1ca 100644 --- a/gora-cassandra/src/test/conf/gora-cassandra-mapping.xml +++ b/gora-cassandra/src/test/conf/gora-cassandra-mapping.xml @@ -17,18 +17,53 @@ limitations under the License. --> +<!-- + The value of 'host' attribute of keyspace tag should match exactly what is in + gora.properties file. Essentially this means that if you are using port number, you should + use it every where regardless of whether it is the default port or not. + At runtime Gora will otherwise try to connect to localhost + https://issues.apache.org/jira/browse/GORA-269 + + The values of 'replication_factor' and 'placement_strategy' attribute of keyspace tag + only apply if gora create the kyespace. they have no effect if this is being used against + an existing keyspace. the default value for 'replication_factor' is '1' + + The value of 'placement_strategy' should be a fully qualifed class name that is known to + the cassansra cluster, not the application or gora. As of this writing, the classes that ship + with cassandra are: + 'org.apache.cassandra.locator.SimpleStrategy' + 'org.apache.cassandra.locator.NetworkTopologyStrategy' + gora cassandra would use SimpleStrategy by default if no value for this attribute is specified + + The default value of 'gc_grace_seconds' is '0' which is ONLY VIABLE FOR SINGLE NODE + CLUSTER. you should update this value according to your cluster configuration. + https://wiki.apache.org/cassandra/StorageConfiguration + + The value of 'ttl' (time to live) attribute of field tag should most likely always + be zero unless you want Cassandra to create Tombstones and delete portions of your + data once this period expires. Any positive value is read and bound to the number + of seconds after which the value for that field will disappear. The default value of ttl + is '0' + + More information on gora-cassandra configuration and mapping's can be found + at http://gora.apache.org/current/gora-cassandra.html +--> + <gora-otd> - <keyspace name="Employee" host="localhost" cluster="Gora Cassandra Test Cluster"> + <keyspace name="Employee" host="localhost" placement_strategy="org.apache.cassandra.locator.SimpleStrategy" + replication_factor="1" cluster="Gora Cassandra Test Cluster"> <family name="p" gc_grace_seconds="5"/> <family name="sc" type="super" /> </keyspace> - <keyspace name="WebPage" host="localhost" cluster="Gora Cassandra Test Cluster"> + <keyspace name="WebPage" host="localhost" placement_strategy="org.apache.cassandra.locator.SimpleStrategy" + replication_factor="1" cluster="Gora Cassandra Test Cluster"> <family name="p" gc_grace_seconds="5"/> <family name="sc" type="super"/> </keyspace> - <keyspace name="TokenDatum" host="localhost" cluster="Gora Cassandra Test Cluster"> + <keyspace name="TokenDatum" host="localhost" placement_strategy="org.apache.cassandra.locator.SimpleStrategy" + replication_factor="1" cluster="Gora Cassandra Test Cluster"> <family name="p" gc_grace_seconds="5"/> <family name="sc" type="super"/> </keyspace>
