Add optional codecs to support all the datatypes for native serialization
Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/43439820 Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/43439820 Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/43439820 Branch: refs/heads/master Commit: 434398203b50c8bc4a71b116ebfa99ef6d0de8a2 Parents: cc452f8 Author: madhawa <madhaw...@gmail.com> Authored: Wed Jul 12 20:46:02 2017 +0530 Committer: madhawa <madhaw...@gmail.com> Committed: Wed Jul 12 20:46:02 2017 +0530 ---------------------------------------------------------------------- .../nativeSerialization/ComplexTypes.java | 102 ++++++++++++++ .../generated/nativeSerialization/User.java | 3 - .../gora/cassandra/store/CassandraClient.java | 134 ++++++++++++++++++- .../store/CassandraMappingBuilder.java | 4 +- .../gora/cassandra/store/CassandraStore.java | 2 +- .../gora-cassandra-mapping.xml | 19 ++- .../conf/nativeSerialization/gora.properties | 6 +- ...stCassandraStoreWithNativeSerialization.java | 43 ++++++ 8 files changed, 297 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/43439820/gora-cassandra-cql/src/examples/java/org/apache/gora/cassandra/example/generated/nativeSerialization/ComplexTypes.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/examples/java/org/apache/gora/cassandra/example/generated/nativeSerialization/ComplexTypes.java b/gora-cassandra-cql/src/examples/java/org/apache/gora/cassandra/example/generated/nativeSerialization/ComplexTypes.java new file mode 100644 index 0000000..ac8de28 --- /dev/null +++ b/gora-cassandra-cql/src/examples/java/org/apache/gora/cassandra/example/generated/nativeSerialization/ComplexTypes.java @@ -0,0 +1,102 @@ +package org.apache.gora.cassandra.example.generated.nativeSerialization; + +import com.datastax.driver.mapping.annotations.Column; +import com.datastax.driver.mapping.annotations.PartitionKey; +import com.datastax.driver.mapping.annotations.Table; +import org.apache.gora.cassandra.persistent.CassandraNativePersistent; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +/** + * Sample class for test native cassandra persistent. + */ +@Table(keyspace = "nativeTestKeySpace", name = "documents", + readConsistency = "QUORUM", + writeConsistency = "QUORUM", + caseSensitiveKeyspace = false, + caseSensitiveTable = true) +public class ComplexTypes extends CassandraNativePersistent { + + @Column + private List<String> listDataType; + @Column + private Map<String, String> mapDataType; + @Column + private String[] stringArrayDataType; + @Column + private int[] intArrayDataType; + @Column + private Set<String> setDataType; + @PartitionKey + @Column + private String id; + @Column + private List<UUID> listUUIDDataType; + + public List<UUID> getListUUIDDataType() { + return listUUIDDataType; + } + + public void setListUUIDDataType(List<UUID> listUUIDDataType) { + this.listUUIDDataType = listUUIDDataType; + } + + + public ComplexTypes(String id) { + this.id = id; + } + + public List<String> getListDataType() { + return listDataType; + } + + public void setListDataType(List<String> listDataType) { + this.listDataType = listDataType; + } + + public Map<String, String> getMapDataType() { + return mapDataType; + } + + public void setMapDataType(Map<String, String> mapDataType) { + this.mapDataType = mapDataType; + } + + public ComplexTypes() { + } + + public String[] getStringArrayDataType() { + return stringArrayDataType; + } + + public void setStringArrayDataType(String[] stringArrayDataType) { + this.stringArrayDataType = stringArrayDataType; + } + + public int[] getIntArrayDataType() { + return intArrayDataType; + } + + public void setIntArrayDataType(int[] intArrayDataType) { + this.intArrayDataType = intArrayDataType; + } + + public Set<String> getSetDataType() { + return setDataType; + } + + public void setSetDataType(Set<String> setDataType) { + this.setDataType = setDataType; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } +} http://git-wip-us.apache.org/repos/asf/gora/blob/43439820/gora-cassandra-cql/src/examples/java/org/apache/gora/cassandra/example/generated/nativeSerialization/User.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/examples/java/org/apache/gora/cassandra/example/generated/nativeSerialization/User.java b/gora-cassandra-cql/src/examples/java/org/apache/gora/cassandra/example/generated/nativeSerialization/User.java index 105dfb7..2b60429 100644 --- a/gora-cassandra-cql/src/examples/java/org/apache/gora/cassandra/example/generated/nativeSerialization/User.java +++ b/gora-cassandra-cql/src/examples/java/org/apache/gora/cassandra/example/generated/nativeSerialization/User.java @@ -43,9 +43,6 @@ public class User extends CassandraNativePersistent { @Column(name = "dob") private Date dateOfBirth; - @Transient - private boolean dirty; - public User() { } http://git-wip-us.apache.org/repos/asf/gora/blob/43439820/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraClient.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraClient.java b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraClient.java index 7f89bbb..196f6a3 100644 --- a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraClient.java +++ b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraClient.java @@ -2,6 +2,7 @@ package org.apache.gora.cassandra.store; import com.datastax.driver.core.Cluster; import com.datastax.driver.core.ConsistencyLevel; +import com.datastax.driver.core.DataType; import com.datastax.driver.core.HostDistance; import com.datastax.driver.core.PoolingOptions; import com.datastax.driver.core.ProtocolOptions; @@ -20,6 +21,15 @@ import com.datastax.driver.core.policies.LatencyAwarePolicy; import com.datastax.driver.core.policies.LoggingRetryPolicy; import com.datastax.driver.core.policies.RoundRobinPolicy; import com.datastax.driver.core.policies.TokenAwarePolicy; +import com.datastax.driver.extras.codecs.arrays.DoubleArrayCodec; +import com.datastax.driver.extras.codecs.arrays.FloatArrayCodec; +import com.datastax.driver.extras.codecs.arrays.IntArrayCodec; +import com.datastax.driver.extras.codecs.arrays.LongArrayCodec; +import com.datastax.driver.extras.codecs.arrays.ObjectArrayCodec; +import com.datastax.driver.extras.codecs.date.SimpleDateCodec; +import com.datastax.driver.extras.codecs.date.SimpleTimestampCodec; +import com.datastax.driver.extras.codecs.jdk8.OptionalCodec; +import org.apache.gora.cassandra.bean.Field; import org.jdom.Document; import org.jdom.Element; import org.jdom.JDOMException; @@ -30,6 +40,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Properties; /** @@ -53,18 +64,139 @@ public class CassandraClient { private Session session; + private CassandraMapping mapping; - public void initialize(Properties properties) throws Exception { + + void initialize(Properties properties, CassandraMapping mapping) throws Exception { Cluster.Builder builder = Cluster.builder(); List<String> codecs = readCustomCodec(properties); builder = populateSettings(builder, properties); + this.mapping = mapping; this.cluster = builder.build(); if (codecs != null) { registerCustomCodecs(codecs); } + registerOptionalCodecs(); this.session = this.cluster.connect(); } + private void registerOptionalCodecs() { + // Optional Codecs for natives + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.ascii())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.bigint())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.blob())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.cboolean())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.cdouble())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.cfloat())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.cint())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.counter())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.date())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.decimal())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.inet())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.smallInt())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.time())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.timestamp())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.timeUUID())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.tinyInt())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.varint())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.varchar())); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.uuid())); + // Optional Array Codecs + this.cluster.getConfiguration().getCodecRegistry().register(new IntArrayCodec()); + this.cluster.getConfiguration().getCodecRegistry().register(new DoubleArrayCodec()); + this.cluster.getConfiguration().getCodecRegistry().register(new FloatArrayCodec()); + this.cluster.getConfiguration().getCodecRegistry().register(new LongArrayCodec()); + this.cluster.getConfiguration().getCodecRegistry().register(new ObjectArrayCodec<>( + DataType.list(DataType.varchar()), + String[].class, + TypeCodec.varchar())); + // Optional Time Codecs + this.cluster.getConfiguration().getCodecRegistry().register(new SimpleDateCodec()); + this.cluster.getConfiguration().getCodecRegistry().register(new SimpleTimestampCodec()); + + for (Field field : this.mapping.getFieldList()) { + String columnType = field.getType().toLowerCase(Locale.ENGLISH); + //http://docs.datastax.com/en/cql/3.3/cql/cql_reference/cql_data_types_c.html + if (columnType.contains("list")) { + columnType = columnType.substring(columnType.indexOf("<") + 1, columnType.indexOf(">")); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.list(getTypeCodec(columnType)))); + } else if (columnType.contains("set")) { + columnType = columnType.substring(columnType.indexOf("<") + 1, columnType.indexOf(">")); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.set(getTypeCodec(columnType)))); + } else if (columnType.contains("map")) { + String[] columnTypes = columnType.substring(columnType.indexOf("<") + 1, columnType.indexOf(">")).split(","); + this.cluster.getConfiguration().getCodecRegistry().register(new OptionalCodec<>(TypeCodec.map(TypeCodec.set(getTypeCodec(columnTypes[0])), TypeCodec.set(getTypeCodec(columnTypes[1]))))); + } + } + } + + private TypeCodec getTypeCodec(String columnType) { + TypeCodec typeCodec; + switch (columnType) { + case "ascii": + typeCodec = TypeCodec.ascii(); + break; + case "bigint": + typeCodec = TypeCodec.bigint(); + break; + case "blob": + typeCodec = TypeCodec.blob(); + break; + case "boolean": + typeCodec = TypeCodec.cboolean(); + break; + case "counter": + typeCodec = TypeCodec.counter(); + break; + case "date": + typeCodec = TypeCodec.date(); + break; + case "decimal": + typeCodec = TypeCodec.decimal(); + break; + case "double": + typeCodec = TypeCodec.cdouble(); + break; + case "float": + typeCodec = TypeCodec.cfloat(); + break; + case "inet": + typeCodec = TypeCodec.inet(); + break; + case "int": + typeCodec = TypeCodec.cint(); + break; + case "smallint": + typeCodec = TypeCodec.smallInt(); + break; + case "time": + typeCodec = TypeCodec.time(); + break; + case "timestamp": + typeCodec = TypeCodec.timestamp(); + break; + case "timeuuid": + typeCodec = TypeCodec.timeUUID(); + break; + case "tinyint": + typeCodec = TypeCodec.tinyInt(); + break; + case "uuid": + typeCodec = TypeCodec.uuid(); + break; + case "varint": + typeCodec = TypeCodec.varint(); + break; + case "varchar": + case "text": + typeCodec = TypeCodec.varchar(); + break; + default: + LOG.error("Unsupported Cassandra datatype: {} ", columnType); + throw new RuntimeException("Unsupported Cassandra datatype: " + columnType); + } + return typeCodec; + } private Cluster.Builder populateSettings(Cluster.Builder builder, Properties properties) { String serversParam = properties.getProperty(CassandraStoreParameters.CASSANDRA_SERVERS); http://git-wip-us.apache.org/repos/asf/gora/blob/43439820/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraMappingBuilder.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraMappingBuilder.java b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraMappingBuilder.java index 3302bcb..0231ac3 100644 --- a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraMappingBuilder.java +++ b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraMappingBuilder.java @@ -188,7 +188,7 @@ public class CassandraMappingBuilder<K, T extends Persistent> { keyField.setColumnName(attributeValue); break; case "type": - keyField.setType(attributeValue); + keyField.setType(attributeValue.replace("(","<").replace(")",">")); break; case "order": keyField.setOrder(ClusterKeyField.Order.valueOf(attributeValue.toUpperCase(Locale.ENGLISH))); @@ -222,7 +222,7 @@ public class CassandraMappingBuilder<K, T extends Persistent> { fieldKey.setColumnName(attributeValue); break; case "type": - fieldKey.setType(attributeValue); + fieldKey.setType(attributeValue.replace("(","<").replace(")",">")); break; default: fieldKey.addProperty(attributeName, attributeValue); http://git-wip-us.apache.org/repos/asf/gora/blob/43439820/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraStore.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraStore.java b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraStore.java index c2ea388..5e209d9 100644 --- a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraStore.java +++ b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraStore.java @@ -89,7 +89,7 @@ public class CassandraStore<K, T extends Persistent> implements DataStore<K, T> CassandraMappingBuilder mappingBuilder = new CassandraMappingBuilder(this); mapping = mappingBuilder.readMapping(mappingFile); CassandraClient cassandraClient = new CassandraClient(); - cassandraClient.initialize(properties); + cassandraClient.initialize(properties, mapping); cassandraSerializer = CassandraSerializer.getSerializer(cassandraClient, properties.getProperty(CassandraStoreParameters.CASSANDRA_SERIALIZATION_TYPE), keyClass, persistentClass, mapping); } catch (Exception e) { throw new RuntimeException("Error while initializing Cassandra store: " + e.getMessage(), e); http://git-wip-us.apache.org/repos/asf/gora/blob/43439820/gora-cassandra-cql/src/test/conf/nativeSerialization/gora-cassandra-mapping.xml ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/test/conf/nativeSerialization/gora-cassandra-mapping.xml b/gora-cassandra-cql/src/test/conf/nativeSerialization/gora-cassandra-mapping.xml index b8c2df8..2a5434d 100644 --- a/gora-cassandra-cql/src/test/conf/nativeSerialization/gora-cassandra-mapping.xml +++ b/gora-cassandra-cql/src/test/conf/nativeSerialization/gora-cassandra-mapping.xml @@ -56,10 +56,21 @@ </keyspace> <class name="org.apache.gora.cassandra.example.generated.nativeSerialization.User" keyClass="java.util.UUID" keyspace="nativeTestKeySpace" - table="Users" > - <field name="userId" column="user_id" type="uuid" ttl="10" primarykey="true"/> - <field name="name" column="name" type="text" ttl="10"/> - <field name="dateOfBirth" column="dob" type="timestamp" ttl="10"/> + table="users" > + <field name="userId" column="user_id" type="uuid" primarykey="true"/> + <field name="name" column="name" type="text" /> + <field name="dateOfBirth" column="dob" type="timestamp" /> + </class> + + <class name="org.apache.gora.cassandra.example.generated.nativeSerialization.ComplexTypes" keyClass="java.lang.String" keyspace="nativeTestKeySpace" + table="documents" > + <field name="id" column="id" type="text" primarykey="true"/> + <field name="listDataType" column="listDataType" type="list(text)"/> + <field name="listUUIDDataType" column="listUUIDDataType" type="list(uuid)"/> + <field name="mapDataType" column="mapDataType" type="map(text,text)" /> + <field name="stringArrayDataType" column="stringArrayDataType" type="list(text)" /> + <field name="intArrayDataType" column="intArrayDataType" type="list(int)" /> + <field name="setDataType" column="setDataType" type="set(text)" /> </class> </gora-otd> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/gora/blob/43439820/gora-cassandra-cql/src/test/conf/nativeSerialization/gora.properties ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/test/conf/nativeSerialization/gora.properties b/gora-cassandra-cql/src/test/conf/nativeSerialization/gora.properties index 0996f12..febd442 100644 --- a/gora-cassandra-cql/src/test/conf/nativeSerialization/gora.properties +++ b/gora-cassandra-cql/src/test/conf/nativeSerialization/gora.properties @@ -18,11 +18,7 @@ gora.datastore.default=org.apache.gora.cassandra.CassandraStore gora.cassandrastore.cluster=Gora Test Cluster gora.cassandrastore.host=localhost:9160 -# property is annotated in CassandraClient#checkKeyspace() -# options are ANY, ONE, TWO, THREE, LOCAL_QUORUM, EACH_QUORUM, QUORUM and ALL. -gora.cassandrastore.cf.consistency.level=ONE -gora.cassandrastore.read.consistency.level=QUORUM -gora.cassandrastore.write.consistency.level=ONE + http://git-wip-us.apache.org/repos/asf/gora/blob/43439820/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/store/TestCassandraStoreWithNativeSerialization.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/store/TestCassandraStoreWithNativeSerialization.java b/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/store/TestCassandraStoreWithNativeSerialization.java index 1fc76c6..88d6267 100644 --- a/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/store/TestCassandraStoreWithNativeSerialization.java +++ b/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/store/TestCassandraStoreWithNativeSerialization.java @@ -18,11 +18,13 @@ package org.apache.gora.cassandra.store; import org.apache.gora.cassandra.GoraCassandraTestDriver; +import org.apache.gora.cassandra.example.generated.nativeSerialization.ComplexTypes; import org.apache.gora.cassandra.example.generated.nativeSerialization.User; import org.apache.gora.cassandra.query.CassandraQuery; import org.apache.gora.query.Query; import org.apache.gora.query.Result; import org.apache.gora.store.DataStore; +import org.apache.gora.util.GoraException; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -30,8 +32,11 @@ import org.junit.BeforeClass; import org.junit.Test; import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Properties; import java.util.UUID; @@ -214,6 +219,9 @@ public class TestCassandraStoreWithNativeSerialization { Assert.assertEquals(1, i); } + /** + * In this test case, delete by query method behavior of the data store is testing. + */ @Test public void testDeleteByQuery() throws Exception { userDataStore.truncateSchema(); @@ -239,6 +247,9 @@ public class TestCassandraStoreWithNativeSerialization { Assert.assertEquals(partialDeletedUser.getDateOfBirth(),user2.getDateOfBirth()); } + /** + * In this test case, update by quert method behavior of the data store is testing. + */ @Test public void testUpdateByQuery() { userDataStore.truncateSchema(); @@ -259,4 +270,36 @@ public class TestCassandraStoreWithNativeSerialization { User user = userDataStore.get(id1); Assert.assertEquals(user.getName(),"madhawa"); } + + @Test + public void testComplexTypes() throws GoraException { + DataStore<String, ComplexTypes> documentDataStore = testDriver.createDataStore(String.class, ComplexTypes.class); + ComplexTypes document = new ComplexTypes("document1"); + document.setIntArrayDataType(new int[]{1,2,3}); + document.setStringArrayDataType(new String[] {"madhawa", "kasun", "gunasekara", "pannipitiya", "srilanka"}); + document.setListDataType(new ArrayList<>(Arrays.asList("gora","nutch","tika","opennlp", "olingo"))); + document.setSetDataType(new HashSet<>(Arrays.asList("important", "keeper"))); + HashMap<String,String> map = new HashMap<>(); + map.put("LK","Colombo"); + document.setMapDataType(map); + documentDataStore.put("document1", document); + ComplexTypes retrievedDocuemnt = documentDataStore.get("document1"); + // verify list data + for(int i=0; i<document.getListDataType().size(); i++) { + Assert.assertEquals(document.getListDataType().get(i), retrievedDocuemnt.getListDataType().get(i)); + } + // verify set data + for(int i=0; i<document.getSetDataType().size(); i++) { + Assert.assertTrue(Arrays.equals(document.getSetDataType().toArray(), retrievedDocuemnt.getSetDataType().toArray())); + } + // verify array data + for(int i=0; i<document.getIntArrayDataType().length; i++) { + Assert.assertTrue(Arrays.equals(document.getIntArrayDataType(), retrievedDocuemnt.getIntArrayDataType())); + } + for(int i=0; i<document.getStringArrayDataType().length; i++) { + Assert.assertTrue(Arrays.equals(document.getStringArrayDataType(), retrievedDocuemnt.getStringArrayDataType())); + } + // verify map data + Assert.assertEquals(map.get("LK"),retrievedDocuemnt.getMapDataType().get("LK")); + } }