Add caching options to the table creation, supporting pre 2.1 and 2.1 table properties in Cassandra. Don't default the Cassandra version in the tests.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bb225e8a Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bb225e8a Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bb225e8a Branch: refs/heads/datastax-cass-driver Commit: bb225e8a37cfbe1419d98e61247af45a53f39733 Parents: 6ef4c32 Author: Michael Russo <[email protected]> Authored: Sun Feb 14 17:24:52 2016 -0800 Committer: Michael Russo <[email protected]> Committed: Sun Feb 14 17:24:52 2016 -0800 ---------------------------------------------------------------------- .../main/resources/usergrid-default.properties | 2 +- .../src/test/resources/dynamic-test.properties | 1 - .../src/test/resources/usergrid-CHOP.properties | 1 - .../src/test/resources/usergrid-UNIT.properties | 1 - .../persistence/core/datastax/CQLUtils.java | 97 +++++++++++++++-- .../core/datastax/TableDefinition.java | 17 +-- .../core/datastax/impl/DatastaxClusterImpl.java | 1 - .../migration/schema/MigrationManagerImpl.java | 2 +- .../persistence/core/datastax/CQLUtilsTest.java | 105 ++++++++++++++++++- .../src/test/resources/usergrid-UNIT.properties | 1 - .../src/test/resources/usergrid-AWS.properties | 1 - .../src/test/resources/usergrid-CHOP.properties | 1 - .../src/test/resources/usergrid-UNIT.properties | 1 - .../src/test/resources/dynamic-test.properties | 1 - .../src/test/resources/usergrid-CHOP.properties | 1 - .../src/test/resources/usergrid-UNIT.properties | 1 - 16 files changed, 196 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/config/src/main/resources/usergrid-default.properties ---------------------------------------------------------------------- diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties index 94d02ff..89f5dd9 100644 --- a/stack/config/src/main/resources/usergrid-default.properties +++ b/stack/config/src/main/resources/usergrid-default.properties @@ -59,7 +59,7 @@ usergrid.persistence=CP # Set a property to tell Usergrid which version of cassandra is being used. # -#cassandra.version=1.2 +#cassandra.version=2.1 # Set the Cassandra cluster name that this instance of Usergrid should use. http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties index b847f50..ca203f9 100644 --- a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties +++ b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties @@ -2,7 +2,6 @@ # safe dynamic property defaults for our testing via IDE or Maven cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties index d276934..92d0041 100644 --- a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties +++ b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties @@ -2,7 +2,6 @@ cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 # a comma delimited private IP address list to your chop cassandra cluster # define this in your settings.xml and have it as an always active profile http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties index bec1c88..4a2d031 100644 --- a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties +++ b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties @@ -1,7 +1,6 @@ # Keep nothing but overriding test defaults in here cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java index 38b6b7b..f04e1a9 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/CQLUtils.java @@ -18,10 +18,10 @@ */ package org.apache.usergrid.persistence.core.datastax; -import com.datastax.driver.core.DataType; -import com.datastax.driver.core.ProtocolVersion; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import org.apache.usergrid.persistence.core.astyanax.CassandraFig; import org.apache.usergrid.persistence.core.util.StringUtils; import java.nio.ByteBuffer; @@ -29,6 +29,8 @@ import java.util.*; public class CQLUtils { + private final CassandraFig cassandraFig; + private final static ObjectMapper mapper = new ObjectMapper(); public enum ACTION { CREATE, UPDATE @@ -41,6 +43,7 @@ public class CQLUtils { static String EQUAL = "="; static String COMPRESSION = "compression"; static String COMPACTION = "compaction"; + static String CACHING = "caching"; static String GC_GRACE_SECONDS = "gc_grace_seconds"; static String PRIMARY_KEY = "PRIMARY KEY"; static String COMPACT_STORAGE = "COMPACT STORAGE"; @@ -49,8 +52,12 @@ public class CQLUtils { static String PAREN_LEFT = "("; static String PAREN_RIGHT = ")"; + @Inject + public CQLUtils ( final CassandraFig cassandraFig ){ - private final static ObjectMapper mapper = new ObjectMapper(); + this.cassandraFig = cassandraFig; + + } public static String getFormattedReplication(String strategy, String strategyOptions) throws JsonProcessingException { @@ -72,7 +79,8 @@ public class CQLUtils { } - public static String getTableCQL(TableDefinition tableDefinition, ACTION tableAction) throws Exception { + public static String getTableCQL( CassandraFig cassandraFig, TableDefinition tableDefinition, + ACTION tableAction) throws Exception { StringJoiner cql = new StringJoiner(" "); @@ -120,9 +128,9 @@ public class CQLUtils { .add(AND) .add(COMPRESSION).add(EQUAL).add( getMapAsCQLString( tableDefinition.getCompression() ) ) .add(AND) - .add(GC_GRACE_SECONDS).add(EQUAL).add( tableDefinition.getGcGraceSeconds() ); - - + .add(GC_GRACE_SECONDS).add(EQUAL).add( tableDefinition.getGcGraceSeconds() ) + .add(AND) + .add(CACHING).add(EQUAL).add( getCachingOptions( cassandraFig, tableDefinition.getCacheOption() ) ); return cql.toString(); @@ -144,6 +152,81 @@ public class CQLUtils { } + public static String getCachingOptions(CassandraFig cassandraFig, TableDefinition.CacheOption cacheOption) throws JsonProcessingException { + + // Cassandra 2.0 and below has a different CQL syntax for caching + if( Double.parseDouble( cassandraFig.getVersion() ) <= 2.0 ){ + + return quote( getLegacyCacheValue( cacheOption ) ); + + } else { + + return getCacheValue( cacheOption ); + } + + } + + + public static String getCacheValue( TableDefinition.CacheOption cacheOption ) throws JsonProcessingException { + + + Map<String, Object> cacheValue = new HashMap<>(2); + switch (cacheOption) { + + case ALL: + cacheValue.put("keys", "ALL"); + cacheValue.put("rows_per_partition", "ALL"); + break; + + case KEYS: + cacheValue.put("keys", "ALL"); + cacheValue.put("rows_per_partition", "NONE"); + break; + + case ROWS: + cacheValue.put("keys", "NONE"); + cacheValue.put("rows_per_partition", "ALL"); + break; + + case NONE: + cacheValue.put("keys", "NONE"); + cacheValue.put("rows_per_partition", "NONE"); + break; + + } + + return getMapAsCQLString( cacheValue ); + + } + + public static String getLegacyCacheValue( TableDefinition.CacheOption cacheOption ){ + + String cacheValue = "none"; // default to no caching + switch (cacheOption) { + + case ALL: + cacheValue = "all"; + break; + + case KEYS: + cacheValue = "keys_only"; + break; + + case ROWS: + cacheValue = "rows_only"; + break; + + case NONE: + cacheValue = "none"; + break; + + } + + return cacheValue; + + } + + /** * Below functions borrowed from Astyanax until the schema is re-written to be more CQL friendly */ http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java index 2f83513..d749d40 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/TableDefinition.java @@ -32,22 +32,7 @@ public class TableDefinition { public enum CacheOption { - ALL( "ALL" ), - KEYS( "KEYS_ONLY" ), - ROWS( "ROWS_ONLY" ), - NONE( "NONE" ); - - private String value; - - - CacheOption( String value ) { - this.value = value; - } - - - public String getValue() { - return value; - } + ALL, KEYS, ROWS, NONE } http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java index 702c1e3..875c5c2 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/datastax/impl/DatastaxClusterImpl.java @@ -26,7 +26,6 @@ import com.google.inject.Singleton; import org.apache.usergrid.persistence.core.astyanax.CassandraFig; import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; -import org.safehaus.guicyfig.Overrides; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java index 9ab403d..3630fc8 100644 --- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java +++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/schema/MigrationManagerImpl.java @@ -138,7 +138,7 @@ public class MigrationManagerImpl implements MigrationManager { private void createTable(TableDefinition tableDefinition ) throws Exception { - String CQL = CQLUtils.getTableCQL( tableDefinition, CQLUtils.ACTION.CREATE ); + String CQL = CQLUtils.getTableCQL( cassandraFig, tableDefinition, CQLUtils.ACTION.CREATE ); if (logger.isDebugEnabled()){ logger.debug( CQL ); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java index c47312d..5bad54e 100644 --- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java +++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/datastax/CQLUtilsTest.java @@ -20,7 +20,14 @@ package org.apache.usergrid.persistence.core.datastax; import com.datastax.driver.core.DataType; +import com.google.inject.Inject; +import org.apache.usergrid.persistence.core.astyanax.CassandraFig; +import org.apache.usergrid.persistence.core.guice.TestCommonModule; +import org.apache.usergrid.persistence.core.test.ITRunner; +import org.apache.usergrid.persistence.core.test.UseModules; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,13 +35,22 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +@RunWith( ITRunner.class ) +@UseModules( TestCommonModule.class ) public class CQLUtilsTest { private static final Logger logger = LoggerFactory.getLogger( CQLUtilsTest.class ); + @Inject + CassandraFig cassandraFig; + + @Test public void testTableCQL() throws Exception { @@ -64,8 +80,8 @@ public class CQLUtilsTest { clusteringOrder ); - String createCQL = CQLUtils.getTableCQL(table1, CQLUtils.ACTION.CREATE); - String updateCQL = CQLUtils.getTableCQL(table1, CQLUtils.ACTION.UPDATE); + String createCQL = CQLUtils.getTableCQL(cassandraFig, table1, CQLUtils.ACTION.CREATE); + String updateCQL = CQLUtils.getTableCQL(cassandraFig, table1, CQLUtils.ACTION.UPDATE); assertTrue( createCQL.contains(CQLUtils.CREATE_TABLE ) && @@ -85,4 +101,89 @@ public class CQLUtilsTest { } + @Test + public void testLegacyCachingOptions() throws Exception{ + + final CassandraFig cassandraFig = mock(CassandraFig.class); + when(cassandraFig.getVersion()).thenReturn("2.0"); + + Map<String, DataType.Name> columns = new HashMap<>(); + columns.put("key", DataType.Name.BLOB); + columns.put("column1", DataType.Name.TEXT); + columns.put("value", DataType.Name.BLOB); + + List<String> partitionKeys = new ArrayList<>(); + partitionKeys.add("key"); + + List<String> columnKeys = new ArrayList<>(); + columnKeys.add("column1"); + + Map<String, String> clusteringOrder = new HashMap<>(); + clusteringOrder.put("column1", "DESC"); + + + + TableDefinition table1 = new TableDefinition( + CQLUtils.quote("table1"), + partitionKeys, + columnKeys, + columns, + TableDefinition.CacheOption.KEYS, + clusteringOrder + ); + + String createCQL = CQLUtils.getTableCQL(cassandraFig, table1, CQLUtils.ACTION.CREATE); + logger.info(createCQL); + assertTrue( + createCQL.contains( "\"keys_only\"" ) && + !createCQL.contains( "'keys':'ALL'" ) + + ); + + + + } + + @Test + public void testCachingOptions() throws Exception { + + final CassandraFig cassandraFig = mock(CassandraFig.class); + when(cassandraFig.getVersion()).thenReturn("2.1"); + + Map<String, DataType.Name> columns = new HashMap<>(); + columns.put("key", DataType.Name.BLOB); + columns.put("column1", DataType.Name.TEXT); + columns.put("value", DataType.Name.BLOB); + + List<String> partitionKeys = new ArrayList<>(); + partitionKeys.add("key"); + + List<String> columnKeys = new ArrayList<>(); + columnKeys.add("column1"); + + Map<String, String> clusteringOrder = new HashMap<>(); + clusteringOrder.put("column1", "DESC"); + + + + TableDefinition table1 = new TableDefinition( + CQLUtils.quote("table1"), + partitionKeys, + columnKeys, + columns, + TableDefinition.CacheOption.KEYS, + clusteringOrder + ); + + String createCQL = CQLUtils.getTableCQL(cassandraFig, table1, CQLUtils.ACTION.CREATE); + logger.info(createCQL); + assertTrue( + createCQL.contains( "'keys':'ALL'" ) && + !createCQL.contains( "\"keys_only\"" ) + + ); + + + } + } http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties ---------------------------------------------------------------------- diff --git a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties index 2004d9e..5f59711 100644 --- a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties +++ b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties @@ -1,7 +1,6 @@ # Keep nothing but overriding test defaults in here cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties ---------------------------------------------------------------------- diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties b/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties index e630bb8..e737815 100644 --- a/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties +++ b/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties @@ -1,7 +1,6 @@ # Keep nothing but overriding test defaults in here cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 cassandra.hosts= #cassandra.hosts=localhost cassandra.cluster_name=Usergrid http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties ---------------------------------------------------------------------- diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties index d276934..92d0041 100644 --- a/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties +++ b/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties @@ -2,7 +2,6 @@ cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 # a comma delimited private IP address list to your chop cassandra cluster # define this in your settings.xml and have it as an always active profile http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties ---------------------------------------------------------------------- diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties index e1c4c81..6f8a7c5 100644 --- a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties +++ b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties @@ -1,7 +1,6 @@ # Keep nothing but overriding test defaults in here cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties index 4ac97d8..7b869af 100644 --- a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties +++ b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties @@ -2,7 +2,6 @@ # safe dynamic property defaults for our testing via IDE or Maven cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 cassandra.hosts=localhost cassandra.cluster_name=Usergrid collections.keyspace=Usergrid_Collections http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties index ce40600..ff4f3bb 100644 --- a/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties +++ b/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties @@ -1,7 +1,6 @@ # These are for CHOP environment settings cassandra.connections=50 cassandra.port=9160 -cassandra.version=1.2 # a comma delimited private IP address list to your chop cassandra cluster # define this in your settings.xml and have it as an always active profile http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb225e8a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties index 5d80cac..3c853ac 100644 --- a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties +++ b/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties @@ -3,7 +3,6 @@ cassandra.embedded=true cassandra.hosts=127.0.0.1 cassandra.port=9160 -cassandra.version=1.2 cassandra.cluster_name=Usergrid cassandra.connections=50
