adding avro serialization insted pure java and unit tests
Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/a05928a7 Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/a05928a7 Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/a05928a7 Branch: refs/heads/master Commit: a05928a7e347e841ed0bb4359908b5feca390fb2 Parents: fd80478 Author: Kevin Ratnasekera <djkevi...@yahoo.com> Authored: Mon Aug 1 02:27:40 2016 +0530 Committer: Kevin Ratnasekera <djkevi...@yahoo.com> Committed: Mon Aug 1 02:27:40 2016 +0530 ---------------------------------------------------------------------- .../org/apache/gora/compiler/GoraCompiler.java | 5 + .../apache/gora/compiler/templates/record.vm | 29 ++- .../gora/examples/generated/Employee.java | 29 ++- .../examples/generated/ImmutableFields.java | 31 ++- .../gora/examples/generated/Metadata.java | 31 ++- .../gora/examples/generated/TokenDatum.java | 29 ++- .../org/apache/gora/examples/generated/V2.java | 31 ++- .../apache/gora/examples/generated/WebPage.java | 31 ++- .../impl/DirtyCollectionWrapper.java | 20 +- .../apache/gora/persistency/impl/DirtyFlag.java | 2 +- .../persistency/impl/DirtyIteratorWrapper.java | 20 +- .../persistency/impl/DirtyListIterator.java | 18 ++ .../gora/persistency/impl/DirtyListWrapper.java | 18 ++ .../gora/persistency/impl/DirtyMapWrapper.java | 20 +- .../gora/persistency/impl/DirtySetWrapper.java | 18 ++ .../gora/persistency/impl/PersistentBase.java | 15 +- .../mapreduce/TestPersistentSerialization.java | 42 ++-- .../gora/mock/persistency/MockPersistent.java | 27 +++ .../apache/gora/jcache/store/JCacheStore.java | 39 +++- .../jcache/store/JCacheGoraDataStoreTest.java | 194 +++++++++++++++++++ gora-jcache/src/test/resources/gora.properties | 3 +- gora-jcache/src/test/resources/hazelcast.xml | 2 +- .../tutorial/log/generated/MetricDatum.java | 29 ++- .../gora/tutorial/log/generated/Pageview.java | 34 +++- 24 files changed, 656 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-compiler/src/main/java/org/apache/gora/compiler/GoraCompiler.java ---------------------------------------------------------------------- diff --git a/gora-compiler/src/main/java/org/apache/gora/compiler/GoraCompiler.java b/gora-compiler/src/main/java/org/apache/gora/compiler/GoraCompiler.java index 529f1ef..6716e3c 100644 --- a/gora-compiler/src/main/java/org/apache/gora/compiler/GoraCompiler.java +++ b/gora-compiler/src/main/java/org/apache/gora/compiler/GoraCompiler.java @@ -29,6 +29,7 @@ import java.util.Set; import org.apache.avro.Schema; import org.apache.avro.Schema.Field; import org.apache.avro.Schema.Type; +import org.apache.avro.SchemaNormalization; import org.apache.avro.compiler.specific.SpecificCompiler; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.node.JsonNodeFactory; @@ -273,4 +274,8 @@ public class GoraCompiler extends SpecificCompiler { return newSchema; } + public static long fingerprint64(Schema schema) { + return SchemaNormalization.parsingFingerprint64(schema); + } + } http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-compiler/src/main/velocity/org/apache/gora/compiler/templates/record.vm ---------------------------------------------------------------------- diff --git a/gora-compiler/src/main/velocity/org/apache/gora/compiler/templates/record.vm b/gora-compiler/src/main/velocity/org/apache/gora/compiler/templates/record.vm index daf5095..aa1844d 100644 --- a/gora-compiler/src/main/velocity/org/apache/gora/compiler/templates/record.vm +++ b/gora-compiler/src/main/velocity/org/apache/gora/compiler/templates/record.vm @@ -24,7 +24,7 @@ package $schema.getNamespace(); #end public class ${this.mangle($schema.getName())}#if ($schema.isError()) extends org.apache.avro.specific.SpecificExceptionBase#else extends org.apache.gora.persistency.impl.PersistentBase#end implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.javaEscape($schema.toString())}"); - + private static final long serialVersionUID = ${this.fingerprint64($schema)}L; /** Enum containing all data bean's fields. */ public static enum Field { #set ($i = 0) @@ -353,6 +353,33 @@ public class ${this.mangle($schema.getName())}#if ($schema.isError()) extends or #end } + + private static final org.apache.avro.io.DatumWriter + DATUM_WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); + private static final org.apache.avro.io.DatumReader + DATUM_READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); + + @Override + public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + out.write(super.getDirtyBytes().array()); + DATUM_WRITER$.write + (this, org.apache.avro.io.EncoderFactory.get() + .directBinaryEncoder((java.io.OutputStream) out, + null)); + } + + @Override + public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + byte[] __g__dirty = new byte[getFieldsCount()]; + in.read(__g__dirty); + super.setDirtyBytes(java.nio.ByteBuffer.wrap(__g__dirty)); + DATUM_READER$.read + (this, org.apache.avro.io.DecoderFactory.get() + .directBinaryDecoder((java.io.InputStream) in, + null)); + } } http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java ---------------------------------------------------------------------- diff --git a/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java b/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java index e6e1c4c..253f7b2 100644 --- a/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java +++ b/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java @@ -17,7 +17,7 @@ package org.apache.gora.examples.generated; public class Employee extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Employee\",\"namespace\":\"org.apache.gora.examples.generated\",\"fields\":[{\"name\":\"name\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"dateOfBirth\",\"type\":\"long\",\"default\":0},{\"name\":\"ssn\",\"type\":\"string\",\"default\":\"\"},{\"name\":\"salary\",\"type\":\"int\",\"default\":0},{\"name\":\"boss\",\"type\":[\"null\",\"Employee\",\"string\"],\"default\":null},{\"name\":\"webpage\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"WebPage\",\"fields\":[{\"name\":\"url\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"content\",\"type\":[\"null\",\"bytes\"],\"default\":null},{\"name\":\"parsedContent\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":{}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":[\"null\",\"string\"]},\"default\":{}},{\"name\":\"headers\",\"type\":[\"null\",{\"typ e\":\"map\",\"values\":[\"null\",\"string\"]}],\"default\":null},{\"name\":\"metadata\",\"type\":{\"type\":\"record\",\"name\":\"Metadata\",\"fields\":[{\"name\":\"version\",\"type\":\"int\",\"default\":0},{\"name\":\"data\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":{}}]},\"default\":null}]}],\"default\":null}],\"default\":null}"); - + private static final long serialVersionUID = -6468893522296148608L; /** Enum containing all data bean's fields. */ public static enum Field { NAME(0, "name"), @@ -667,6 +667,33 @@ public class Employee extends org.apache.gora.persistency.impl.PersistentBase im } + + private static final org.apache.avro.io.DatumWriter + DATUM_WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); + private static final org.apache.avro.io.DatumReader + DATUM_READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); + + @Override + public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + out.write(super.getDirtyBytes().array()); + DATUM_WRITER$.write + (this, org.apache.avro.io.EncoderFactory.get() + .directBinaryEncoder((java.io.OutputStream) out, + null)); + } + + @Override + public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + byte[] __g__dirty = new byte[getFieldsCount()]; + in.read(__g__dirty); + super.setDirtyBytes(java.nio.ByteBuffer.wrap(__g__dirty)); + DATUM_READER$.read + (this, org.apache.avro.io.DecoderFactory.get() + .directBinaryDecoder((java.io.InputStream) in, + null)); + } } http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/examples/java/org/apache/gora/examples/generated/ImmutableFields.java ---------------------------------------------------------------------- diff --git a/gora-core/src/examples/java/org/apache/gora/examples/generated/ImmutableFields.java b/gora-core/src/examples/java/org/apache/gora/examples/generated/ImmutableFields.java index b0e4609..edb91d0 100644 --- a/gora-core/src/examples/java/org/apache/gora/examples/generated/ImmutableFields.java +++ b/gora-core/src/examples/java/org/apache/gora/examples/generated/ImmutableFields.java @@ -19,7 +19,7 @@ package org.apache.gora.examples.generated; /** Record with only immutable or dirtyable fields, used for testing */ public class ImmutableFields extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"ImmutableFields\",\"namespace\":\"org.apache.gora.examples.generated\",\"doc\":\"Record with only immutable or dirtyable fields, used for testing\",\"fields\":[{\"name\":\"v1\",\"type\":\"int\",\"default\":0},{\"name\":\"v2\",\"type\":[{\"type\":\"record\",\"name\":\"V2\",\"fields\":[{\"name\":\"v3\",\"type\":\"int\",\"default\":0}]},\"null\"],\"default\":null}]}"); - + private static final long serialVersionUID = -6468893522296148698L; /** Enum containing all data bean's fields. */ public static enum Field { V1(0, "v1"), @@ -338,6 +338,33 @@ public class ImmutableFields extends org.apache.gora.persistency.impl.Persistent } - + + private static final org.apache.avro.io.DatumWriter + DATUM_WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); + private static final org.apache.avro.io.DatumReader + DATUM_READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); + + @Override + public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + out.write(super.getDirtyBytes().array()); + DATUM_WRITER$.write + (this, org.apache.avro.io.EncoderFactory.get() + .directBinaryEncoder((java.io.OutputStream) out, + null)); + } + + @Override + public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + byte[] __g__dirty = new byte[getFieldsCount()]; + in.read(__g__dirty); + super.setDirtyBytes(java.nio.ByteBuffer.wrap(__g__dirty)); + DATUM_READER$.read + (this, org.apache.avro.io.DecoderFactory.get() + .directBinaryDecoder((java.io.InputStream) in, + null)); + } + } http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java ---------------------------------------------------------------------- diff --git a/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java b/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java index a09f16a..66d1f2c 100644 --- a/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java +++ b/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java @@ -7,7 +7,7 @@ package org.apache.gora.examples.generated; @SuppressWarnings("all") public class Metadata extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Metadata\",\"namespace\":\"org.apache.gora.examples.generated\",\"fields\":[{\"name\":\"version\",\"type\":\"int\",\"default\":0},{\"name\":\"data\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":null}]}"); - + private static final long serialVersionUID = -6468893522296178608L; /** Enum containing all data bean's fields. */ public static enum Field { VERSION(0, "version"), @@ -326,6 +326,33 @@ public class Metadata extends org.apache.gora.persistency.impl.PersistentBase im } - + + private static final org.apache.avro.io.DatumWriter + DATUM_WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); + private static final org.apache.avro.io.DatumReader + DATUM_READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); + + @Override + public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + out.write(super.getDirtyBytes().array()); + DATUM_WRITER$.write + (this, org.apache.avro.io.EncoderFactory.get() + .directBinaryEncoder((java.io.OutputStream) out, + null)); + } + + @Override + public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + byte[] __g__dirty = new byte[getFieldsCount()]; + in.read(__g__dirty); + super.setDirtyBytes(java.nio.ByteBuffer.wrap(__g__dirty)); + DATUM_READER$.read + (this, org.apache.avro.io.DecoderFactory.get() + .directBinaryDecoder((java.io.InputStream) in, + null)); + } + } http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java ---------------------------------------------------------------------- diff --git a/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java b/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java index 53de084..391c912 100644 --- a/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java +++ b/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java @@ -18,7 +18,7 @@ package org.apache.gora.examples.generated; @SuppressWarnings("all") public class TokenDatum extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"TokenDatum\",\"namespace\":\"org.apache.gora.examples.generated\",\"fields\":[{\"name\":\"count\",\"type\":\"int\",\"default\":0}]}"); - + private static final long serialVersionUID = -6468894522296148608L; /** Enum containing all data bean's fields. */ public static enum Field { COUNT(0, "count"), @@ -254,6 +254,33 @@ public class TokenDatum extends org.apache.gora.persistency.impl.PersistentBase } + + private static final org.apache.avro.io.DatumWriter + DATUM_WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); + private static final org.apache.avro.io.DatumReader + DATUM_READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); + + @Override + public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + out.write(super.getDirtyBytes().array()); + DATUM_WRITER$.write + (this, org.apache.avro.io.EncoderFactory.get() + .directBinaryEncoder((java.io.OutputStream) out, + null)); + } + + @Override + public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + byte[] __g__dirty = new byte[getFieldsCount()]; + in.read(__g__dirty); + super.setDirtyBytes(java.nio.ByteBuffer.wrap(__g__dirty)); + DATUM_READER$.read + (this, org.apache.avro.io.DecoderFactory.get() + .directBinaryDecoder((java.io.InputStream) in, + null)); + } } http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/examples/java/org/apache/gora/examples/generated/V2.java ---------------------------------------------------------------------- diff --git a/gora-core/src/examples/java/org/apache/gora/examples/generated/V2.java b/gora-core/src/examples/java/org/apache/gora/examples/generated/V2.java index 7cb9c5f..2092e1b 100644 --- a/gora-core/src/examples/java/org/apache/gora/examples/generated/V2.java +++ b/gora-core/src/examples/java/org/apache/gora/examples/generated/V2.java @@ -18,7 +18,7 @@ package org.apache.gora.examples.generated; @SuppressWarnings("all") public class V2 extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"V2\",\"namespace\":\"org.apache.gora.examples.generated\",\"fields\":[{\"name\":\"v3\",\"type\":\"int\",\"default\":0}]}"); - + private static final long serialVersionUID = -6468893522496148608L; /** Enum containing all data bean's fields. */ public static enum Field { V3(0, "v3"), @@ -254,6 +254,33 @@ public class V2 extends org.apache.gora.persistency.impl.PersistentBase implemen } - + + private static final org.apache.avro.io.DatumWriter + DATUM_WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); + private static final org.apache.avro.io.DatumReader + DATUM_READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); + + @Override + public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + out.write(super.getDirtyBytes().array()); + DATUM_WRITER$.write + (this, org.apache.avro.io.EncoderFactory.get() + .directBinaryEncoder((java.io.OutputStream) out, + null)); + } + + @Override + public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + byte[] __g__dirty = new byte[getFieldsCount()]; + in.read(__g__dirty); + super.setDirtyBytes(java.nio.ByteBuffer.wrap(__g__dirty)); + DATUM_READER$.read + (this, org.apache.avro.io.DecoderFactory.get() + .directBinaryDecoder((java.io.InputStream) in, + null)); + } + } http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java ---------------------------------------------------------------------- diff --git a/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java b/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java index dd74df6..0a7b7ea 100644 --- a/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java +++ b/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java @@ -7,7 +7,7 @@ package org.apache.gora.examples.generated; @SuppressWarnings("all") public class WebPage extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"WebPage\",\"namespace\":\"org.apache.gora.examples.generated\",\"fields\":[{\"name\":\"url\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"content\",\"type\":[\"null\",\"bytes\"],\"default\":null},{\"name\":\"parsedContent\",\"type\":{\"type\":\"array\",\"items\":\"string\"},\"default\":null},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":{}},{\"name\":\"headers\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",\"string\"]}],\"default\":null},{\"name\":\"metadata\",\"type\":{\"type\":\"record\",\"name\":\"Metadata\",\"fields\":[{\"name\":\"version\",\"type\":\"int\",\"default\":0},{\"name\":\"data\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":null}]},\"default\":null},{\"name\":\"byteData\",\"type\":{\"type\":\"map\",\"values\":\"bytes\"},\"default\":{}},{\"name\":\"stringDat a\",\"type\":{\"type\":\"map\",\"values\":\"string\"},\"default\":{}}],\"default\":null}"); - + private static final long serialVersionUID = -6468893522236148608L; /** Enum containing all data bean's fields. */ public static enum Field { URL(0, "url"), @@ -825,6 +825,33 @@ public class WebPage extends org.apache.gora.persistency.impl.PersistentBase imp } - + + private static final org.apache.avro.io.DatumWriter + DATUM_WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); + private static final org.apache.avro.io.DatumReader + DATUM_READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); + + @Override + public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + out.write(super.getDirtyBytes().array()); + DATUM_WRITER$.write + (this, org.apache.avro.io.EncoderFactory.get() + .directBinaryEncoder((java.io.OutputStream) out, + null)); + } + + @Override + public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + byte[] __g__dirty = new byte[getFieldsCount()]; + in.read(__g__dirty); + super.setDirtyBytes(java.nio.ByteBuffer.wrap(__g__dirty)); + DATUM_READER$.read + (this, org.apache.avro.io.DecoderFactory.get() + .directBinaryDecoder((java.io.InputStream) in, + null)); + } + } http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyCollectionWrapper.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyCollectionWrapper.java b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyCollectionWrapper.java index b75dbea..e082d03 100644 --- a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyCollectionWrapper.java +++ b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyCollectionWrapper.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.gora.persistency.impl; import java.util.Collection; @@ -15,7 +33,7 @@ import org.apache.gora.persistency.Dirtyable; * The type of the list that this wrapper wraps. */ public class DirtyCollectionWrapper<T> implements Dirtyable, - Collection<T>, java.io.Serializable { + Collection<T> { /** The delegate list that the wrapper wraps */ private final Collection<T> delegate; http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyFlag.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyFlag.java b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyFlag.java index 852c98f..33fd21e 100644 --- a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyFlag.java +++ b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyFlag.java @@ -19,7 +19,7 @@ package org.apache.gora.persistency.impl; import org.apache.gora.persistency.Dirtyable; -final class DirtyFlag implements Dirtyable, java.io.Serializable { +final class DirtyFlag implements Dirtyable { private boolean dirty; http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyIteratorWrapper.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyIteratorWrapper.java b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyIteratorWrapper.java index dbb2e21..f7963c1 100644 --- a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyIteratorWrapper.java +++ b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyIteratorWrapper.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.gora.persistency.impl; import java.util.Iterator; @@ -5,7 +23,7 @@ import java.util.Iterator; /** * Sets the dirty flag if the iterator's remove method is called. */ -final class DirtyIteratorWrapper<T> implements Iterator<T>, java.io.Serializable { +final class DirtyIteratorWrapper<T> implements Iterator<T> { private final DirtyFlag dirtyFlag; private Iterator<T> delegateIterator; http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyListIterator.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyListIterator.java b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyListIterator.java index c378cf3..2d4f58e 100644 --- a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyListIterator.java +++ b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyListIterator.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.gora.persistency.impl; import java.util.ListIterator; http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyListWrapper.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyListWrapper.java b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyListWrapper.java index 3b09310..3113ac6 100644 --- a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyListWrapper.java +++ b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyListWrapper.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.gora.persistency.impl; import java.util.Collection; http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyMapWrapper.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyMapWrapper.java b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyMapWrapper.java index f1f0440..1311685 100644 --- a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyMapWrapper.java +++ b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtyMapWrapper.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.gora.persistency.impl; import java.util.Collection; @@ -9,7 +27,7 @@ import org.apache.gora.persistency.Dirtyable; import com.google.common.base.Function; import com.google.common.collect.Collections2; -public class DirtyMapWrapper<K, V> implements Map<K, V>, Dirtyable, java.io.Serializable { +public class DirtyMapWrapper<K, V> implements Map<K, V>, Dirtyable { public static class DirtyEntryWrapper<K, V> implements Entry<K, V>, Dirtyable { private final Entry<K, V> entryDelegate; http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtySetWrapper.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtySetWrapper.java b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtySetWrapper.java index 67db07a..b691044 100644 --- a/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtySetWrapper.java +++ b/gora-core/src/main/java/org/apache/gora/persistency/impl/DirtySetWrapper.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.gora.persistency.impl; import java.util.Collection; http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java b/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java index 691684e..4f8cb83 100644 --- a/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java +++ b/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java @@ -17,7 +17,6 @@ */ package org.apache.gora.persistency.impl; -import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Collection; import java.util.List; @@ -33,13 +32,13 @@ import org.apache.gora.persistency.Persistent; * Base classs implementing common functionality for Persistent classes. */ public abstract class PersistentBase extends SpecificRecordBase implements - Persistent, java.io.Serializable { + Persistent, java.io.Externalizable { /** Bytes used to represent weather or not a field is dirty. */ - private byte[] __g__dirty; + private ByteBuffer __g__dirty; public PersistentBase() { - __g__dirty = new byte[getFieldsCount()]; + __g__dirty = ByteBuffer.wrap(new byte[getFieldsCount()]); } public abstract int getFieldsCount(); @@ -182,8 +181,12 @@ public abstract class PersistentBase extends SpecificRecordBase implements setDirty(getSchema().getField(field).pos()); } - private ByteBuffer getDirtyBytes() { - return java.nio.ByteBuffer.wrap(__g__dirty); + public ByteBuffer getDirtyBytes() { + return __g__dirty; + } + + public void setDirtyBytes(ByteBuffer __g__dirty) { + this.__g__dirty = __g__dirty; } @Override http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java ---------------------------------------------------------------------- diff --git a/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java b/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java index 141e9cf..b2b66ab 100644 --- a/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java +++ b/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java @@ -31,7 +31,6 @@ import org.apache.gora.store.DataStoreFactory; import org.apache.gora.store.DataStoreTestUtil; import org.apache.gora.util.TestIOUtils; import org.apache.hadoop.conf.Configuration; -import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -48,13 +47,12 @@ public class TestPersistentSerialization { * what we get 'before' and 'after' (de)serialization processes. * @throws Exception */ - @Ignore @SuppressWarnings("unchecked") @Test public void testSerdeEmployee() throws Exception { MemStore<String, Employee> store = DataStoreFactory.getDataStore( - MemStore.class, String.class, Employee.class, new Configuration()); + MemStore.class, String.class, Employee.class, new Configuration()); Employee employee = DataStoreTestUtil.createEmployee(); @@ -64,14 +62,13 @@ public class TestPersistentSerialization { /** * Creates an Employee object but only sets one field as dirty. * We then do (de)serialization and check 'before' and 'after' - * states. + * states. * @throws Exception */ - @Ignore @Test public void testSerdeEmployeeOneField() throws Exception { Employee employee = Employee.newBuilder().build(); - employee.setSsn(new StringBuffer("11111")); + employee.setSsn(new Utf8("11111")); TestIOUtils.testSerializeDeserialize(employee); } @@ -79,36 +76,34 @@ public class TestPersistentSerialization { /** * Creates an Employee object setting only two fields as dirty. * We then do (de)serialization and check 'before' and 'after' - * states. + * states. * @throws Exception */ - @Ignore @Test public void testSerdeEmployeeTwoFields() throws Exception { Employee employee = Employee.newBuilder().build(); - employee.setSsn(new StringBuffer("11111")); + employee.setSsn(new Utf8("11111")); employee.setSalary(100); TestIOUtils.testSerializeDeserialize(employee); } /** - * Creates an WebPage object in-memory setting several fields to dirty. + * Creates an WebPage object in-memory setting several fields to dirty. * Run a query over the persistent data. - * Asserts that the results can be serialized and + * Asserts that the results can be serialized and * deserialzed without loosing data. We do this by asserting * what we get 'before' and 'after' (de)serialization processes. - * Also simple assertion for equal number of URL's in WebPage + * Also simple assertion for equal number of URL's in WebPage * and results. * @throws Exception */ - @Ignore @SuppressWarnings("unchecked") @Test public void testSerdeWebPage() throws Exception { MemStore<String, WebPage> store = DataStoreFactory.getDataStore( - MemStore.class, String.class, WebPage.class, new Configuration()); + MemStore.class, String.class, WebPage.class, new Configuration()); WebPageDataCreator.createWebPageData(store); Result<String, WebPage> result = store.newQuery().execute(); @@ -123,8 +118,8 @@ public class TestPersistentSerialization { } /** - * Creates multiple WebPage objects setting several fields to dirty. - * Asserts that the data can be serialized and + * Creates multiple WebPage objects setting several fields to dirty. + * Asserts that the data can be serialized and * deserialzed without loosing data. We do this by asserting * what we get 'before' and 'after' (de)serialization processes. * @throws Exception @@ -135,17 +130,16 @@ public class TestPersistentSerialization { WebPage page2 = WebPage.newBuilder().build(); WebPage page3 = WebPage.newBuilder().build(); - page1.setUrl(new StringBuffer("foo")); - page2.setUrl(new StringBuffer("baz")); - page3.setUrl(new StringBuffer("bar")); + page1.setUrl(new Utf8("foo")); + page2.setUrl(new Utf8("baz")); + page3.setUrl(new Utf8("bar")); page1.setParsedContent(new ArrayList<CharSequence>()); - page1.getParsedContent().add(new StringBuffer("coo")); - page2.setParsedContent(new ArrayList<CharSequence>()); - page2.getParsedContent().add(new StringBuffer("coo2")); - page3.setParsedContent(new ArrayList<CharSequence>()); - page3.getParsedContent().add(new StringBuffer("coo3")); + page1.getParsedContent().add(new Utf8("coo")); + page2.setOutlinks(new HashMap<CharSequence, CharSequence>()); + page2.getOutlinks().put(new Utf8("a"), new Utf8("b")); TestIOUtils.testSerializeDeserialize(page1, page2, page3); } } + http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java ---------------------------------------------------------------------- diff --git a/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java b/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java index 1a83952..a559b1d 100644 --- a/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java +++ b/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java @@ -93,4 +93,31 @@ public class MockPersistent extends PersistentBase { return new MockPersistent(); } + private final org.apache.avro.io.DatumWriter + DATUM_WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(getSchema()); + private final org.apache.avro.io.DatumReader + DATUM_READER$ = new org.apache.avro.specific.SpecificDatumReader(getSchema()); + + @Override + public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + out.write(super.getDirtyBytes().array()); + DATUM_WRITER$.write + (this, org.apache.avro.io.EncoderFactory.get() + .directBinaryEncoder((java.io.OutputStream) out, + null)); + } + + @Override + public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + byte[] __g__dirty = new byte[getFieldsCount()]; + in.read(__g__dirty); + super.setDirtyBytes(java.nio.ByteBuffer.wrap(__g__dirty)); + DATUM_READER$.read + (this, org.apache.avro.io.DecoderFactory.get() + .directBinaryDecoder((java.io.InputStream) in, + null)); + } + } http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-jcache/src/main/java/org/apache/gora/jcache/store/JCacheStore.java ---------------------------------------------------------------------- diff --git a/gora-jcache/src/main/java/org/apache/gora/jcache/store/JCacheStore.java b/gora-jcache/src/main/java/org/apache/gora/jcache/store/JCacheStore.java index 4838cfe..fb70265 100644 --- a/gora-jcache/src/main/java/org/apache/gora/jcache/store/JCacheStore.java +++ b/gora-jcache/src/main/java/org/apache/gora/jcache/store/JCacheStore.java @@ -91,6 +91,7 @@ public class JCacheStore<K,T extends PersistentBase> extends DataStoreBase<K,T> private static final String HAZELCAST_CACHE_BINARY_IN_MEMORY_FORMAT_IDENTIFIER = "BINARY"; private static final String HAZELCAST_CACHE_OBJECT_IN_MEMORY_FORMAT_IDENTIFIER = "OBJECT"; private static final String HAZELCAST_CACHE_NATIVE_IN_MEMORY_FORMAT_IDENTIFIER = "NATIVE"; + private static final String JCACHE_AUTO_CREATE_CACHE_PROPERTY_KEY ="jcache.auto.create.cache"; private String goraCacheNamespace = GORA_DEFAULT_JCACHE_NAMESPACE; private static final Logger LOG = LoggerFactory.getLogger(JCacheStore.class); private DataStore<K, T> persistentDataStore; @@ -196,8 +197,16 @@ public class JCacheStore<K,T extends PersistentBase> extends DataStoreBase<K,T> null, true, true ) ); - cache = manager.createCache(persistentClass.getSimpleName(), - cacheConfig).unwrap(ICache.class); + if (properties.getProperty(JCACHE_AUTO_CREATE_CACHE_PROPERTY_KEY) != null) { + Boolean createCache = Boolean.valueOf(properties.getProperty(JCACHE_AUTO_CREATE_CACHE_PROPERTY_KEY)); + if (createCache) { + cache = manager.createCache(persistentClass.getSimpleName(), + cacheConfig).unwrap(ICache.class); + } + } else { + cache = manager.createCache(persistentClass.getSimpleName(), + cacheConfig).unwrap(ICache.class); + } LOG.info("JCache Gora datastore initialized successfully."); } @@ -208,7 +217,8 @@ public class JCacheStore<K,T extends PersistentBase> extends DataStoreBase<K,T> @Override public void createSchema() { - if (manager.getCache(super.getPersistentClass().getSimpleName()) == null) { + if (manager.getCache(super.getPersistentClass().getSimpleName(), keyClass, persistentClass) == null) { + cacheEntryList.clear(); cache = manager.createCache(persistentClass.getSimpleName(), cacheConfig).unwrap(ICache.class); } @@ -219,6 +229,7 @@ public class JCacheStore<K,T extends PersistentBase> extends DataStoreBase<K,T> @Override public void deleteSchema() { + cacheEntryList.clear(); manager.destroyCache(super.getPersistentClass().getSimpleName()); persistentDataStore.deleteSchema(); LOG.info("Deleted schema on persistent store and destroyed cache for persistent bean " @@ -227,8 +238,7 @@ public class JCacheStore<K,T extends PersistentBase> extends DataStoreBase<K,T> @Override public boolean schemaExists() { - return (manager.getCache(super.getPersistentClass().getSimpleName()) != null) - && persistentDataStore.schemaExists(); + return (manager.getCache(super.getPersistentClass().getSimpleName(), keyClass, persistentClass) != null); } @Override @@ -251,10 +261,18 @@ public class JCacheStore<K,T extends PersistentBase> extends DataStoreBase<K,T> } T clonedPersistent = AvroUtils.deepClonePersistent(persitent); clonedPersistent.clear(); - for (String field : fields) { - Schema.Field otherField = persitent.getSchema().getField(field); - int index = otherField.pos(); - clonedPersistent.put(index, persitent.get(index)); + if (fields != null && fields.length > 0) { + for (String field : fields) { + Schema.Field otherField = persitent.getSchema().getField(field); + int index = otherField.pos(); + clonedPersistent.put(index, persitent.get(index)); + } + } else { + for (String field : otherFieldStrings) { + Schema.Field otherField = persitent.getSchema().getField(field); + int index = otherField.pos(); + clonedPersistent.put(index, persitent.get(index)); + } } return clonedPersistent; } @@ -376,7 +394,8 @@ public class JCacheStore<K,T extends PersistentBase> extends DataStoreBase<K,T> @Override public void close() { flush(); - if (!cache.isDestroyed()) { + cacheEntryList.clear(); + if (!cache.isDestroyed() && !manager.isClosed()) { cache.destroy(); } if (!manager.isClosed()) { http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-jcache/src/test/java/org/apache/gora/jcache/store/JCacheGoraDataStoreTest.java ---------------------------------------------------------------------- diff --git a/gora-jcache/src/test/java/org/apache/gora/jcache/store/JCacheGoraDataStoreTest.java b/gora-jcache/src/test/java/org/apache/gora/jcache/store/JCacheGoraDataStoreTest.java new file mode 100644 index 0000000..51eb086 --- /dev/null +++ b/gora-jcache/src/test/java/org/apache/gora/jcache/store/JCacheGoraDataStoreTest.java @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gora.jcache.store; + +import org.apache.gora.examples.WebPageDataCreator; +import org.apache.gora.examples.generated.Employee; +import org.apache.gora.examples.generated.WebPage; +import org.apache.gora.persistency.BeanFactory; +import org.apache.gora.persistency.impl.BeanFactoryImpl; +import org.apache.gora.query.Query; +import org.apache.gora.store.DataStore; +import org.apache.gora.store.DataStoreFactory; +import org.apache.gora.store.DataStoreTestBase; +import org.apache.gora.store.DataStoreTestUtil; +import org.apache.hadoop.conf.Configuration; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +import static org.apache.gora.examples.WebPageDataCreator.SORTED_URLS; +import static org.apache.gora.examples.WebPageDataCreator.URLS; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +public class JCacheGoraDataStoreTest extends DataStoreTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(JCacheGoraDataStoreTest.class); + private static final int NUM_KEYS = 4; + private Configuration conf = new Configuration(); + + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + //mandatory to clean up hazelcast instances + //this is not handled at super class level + super.employeeStore.close(); + super.webPageStore.close(); + } + + @SuppressWarnings("unchecked") + protected DataStore<String, Employee> createEmployeeDataStore() throws IOException { + return DataStoreFactory.getDataStore(String.class, Employee.class, conf, true); + } + + @SuppressWarnings("unchecked") + protected DataStore<String, WebPage> createWebPageDataStore() throws IOException { + return DataStoreFactory.getDataStore(String.class, WebPage.class, conf, true); + } + + @Test + public void testGetMissingValue() throws IOException { + DataStore<String, WebPage> store = super.webPageStore; + WebPage nullWebPage = store.get("missing", new String[0]); + assertNull(nullWebPage); + } + + @Test + public void testDeleteSchema() throws Exception { + DataStore<String, WebPage> store = super.webPageStore; + assertTrue(store.schemaExists()); + store.deleteSchema(); + assertTrue(!store.schemaExists()); + } + + @Test + public void testSchemaExists() throws Exception { + DataStore<String, WebPage> store = super.webPageStore; + assertTrue(store.schemaExists()); + } + + @Test + public void testStorePutGet() throws Exception { + String key = "org.apache.gora:http:/"; + DataStore<String, WebPage> store = super.webPageStore; + assumeTrue(store.get(key, new String[0]) == null); + store.put(key, WebPage.newBuilder().build()); + assertNotNull(store.get(key, new String[0])); + } + + @Test + public void testGetWithFields() throws Exception { + + DataStore<String, WebPage> store = super.webPageStore; + BeanFactory<String, WebPage> beanFactory = new BeanFactoryImpl<>(String.class, WebPage.class); + store.setBeanFactory(beanFactory); + WebPageDataCreator.createWebPageData(store); + String[] interestFields = new String[2]; + interestFields[0] = "url"; + interestFields[1] = "content"; + WebPage page = store.get(URLS[1], interestFields); + assertNotNull(page); + assertNotNull(page.getUrl()); + assertEquals(page.getUrl().toString(), URLS[1]); + assertNotNull(page.getContent()); + assertEquals("Map of Outlinks should have a size of '0' as it is omitted at retrieval", + 0, page.getOutlinks().size()); + assertEquals("Map of Parsed Content should have a size of '0' as it is omitted at retrieval", + 0, page.getParsedContent().size()); + } + + @Test + public void testDeleteByQueryFields() throws Exception { + + DataStore<String, WebPage> store = super.webPageStore; + BeanFactory<String, WebPage> beanFactory = new BeanFactoryImpl<>(String.class, WebPage.class); + store.setBeanFactory(beanFactory); + Query<String, WebPage> query; + WebPageDataCreator.createWebPageData(store); + query = store.newQuery(); + query.setFields("outlinks", "parsedContent", "content"); + Query<String, WebPage> newQuery = store.newQuery(); + newQuery.setStartKey(SORTED_URLS[0]); + newQuery.setEndKey(SORTED_URLS[9]); + newQuery.setFields("outlinks", "parsedContent", "content"); + DataStoreTestUtil.assertNumResults(newQuery, URLS.length); + store.deleteByQuery(query); + store.deleteByQuery(query); + store.deleteByQuery(query); + store.flush(); + DataStoreTestUtil.assertNumResults(store.newQuery(), URLS.length); + for (String SORTED_URL : SORTED_URLS) { + WebPage page = store.get(SORTED_URL); + assertNotNull(page); + + assertNotNull(page.getUrl()); + assertEquals(page.getUrl().toString(), SORTED_URL); + assertEquals("Map of Outlinks should have a size of '0' as the deleteByQuery " + + "not only removes the data but also the data structure.", 0, page.getOutlinks().size()); + assertEquals(0, page.getParsedContent().size()); + if (page.getContent() != null) { + LOG.info("url:" + page.getUrl().toString()); + LOG.info("limit:" + page.getContent().limit()); + } else { + assertNull(page.getContent()); + } + } + WebPageDataCreator.createWebPageData(store); + query = store.newQuery(); + query.setFields("url"); + String startKey = SORTED_URLS[NUM_KEYS]; + String endKey = SORTED_URLS[SORTED_URLS.length - NUM_KEYS]; + query.setStartKey(startKey); + query.setEndKey(endKey); + DataStoreTestUtil.assertNumResults(store.newQuery(), URLS.length); + store.deleteByQuery(query); + store.deleteByQuery(query); + store.deleteByQuery(query); + store.flush(); + DataStoreTestUtil.assertNumResults(store.newQuery(), URLS.length); + for (int i = 0; i < URLS.length; i++) { + WebPage page = store.get(URLS[i]); + assertNotNull(page); + if (URLS[i].compareTo(startKey) < 0 || URLS[i].compareTo(endKey) > 0) { + DataStoreTestUtil.assertWebPage(page, i); + } else { + assertNull(page.getUrl()); + assertNotNull(page.getOutlinks()); + assertNotNull(page.getParsedContent()); + assertNotNull(page.getContent()); + assertTrue(page.getOutlinks().size() > 0); + assertTrue(page.getParsedContent().size() > 0); + } + } + + } +} http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-jcache/src/test/resources/gora.properties ---------------------------------------------------------------------- diff --git a/gora-jcache/src/test/resources/gora.properties b/gora-jcache/src/test/resources/gora.properties index ccbd282..929115a 100644 --- a/gora-jcache/src/test/resources/gora.properties +++ b/gora-jcache/src/test/resources/gora.properties @@ -14,4 +14,5 @@ # limitations under the License. gora.cache.datastore.default=org.apache.gora.jcache.store.JCacheStore -gora.datastore.jcache.provider=com.hazelcast.cache.HazelcastCachingProvider +gora.datastore.jcache.provider=com.hazelcast.cache.impl.HazelcastServerCachingProvider +gora.datastore.default=org.apache.gora.memory.store.MemStore http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-jcache/src/test/resources/hazelcast.xml ---------------------------------------------------------------------- diff --git a/gora-jcache/src/test/resources/hazelcast.xml b/gora-jcache/src/test/resources/hazelcast.xml index 5f24979..dcf138e 100755 --- a/gora-jcache/src/test/resources/hazelcast.xml +++ b/gora-jcache/src/test/resources/hazelcast.xml @@ -27,7 +27,7 @@ <network> <join> <multicast enabled="false"/> - <tcp-ip enabled="true"> + <tcp-ip enabled="false"> <member>127.0.0.1</member> </tcp-ip> </join> http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/MetricDatum.java ---------------------------------------------------------------------- diff --git a/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/MetricDatum.java b/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/MetricDatum.java index 0d7be00..df9662d 100644 --- a/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/MetricDatum.java +++ b/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/MetricDatum.java @@ -6,7 +6,7 @@ package org.apache.gora.tutorial.log.generated; public class MetricDatum extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"MetricDatum\",\"namespace\":\"org.apache.gora.tutorial.log.generated\",\"fields\":[{\"name\":\"metricDimension\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"timestamp\",\"type\":\"long\",\"default\":0},{\"name\":\"metric\",\"type\":\"long\",\"default\":0}],\"default\":null}"); - + private static final long serialVersionUID = -6468793522296148608L; /** Enum containing all data bean's fields. */ public static enum Field { METRIC_DIMENSION(0, "metricDimension"), @@ -401,6 +401,33 @@ public class MetricDatum extends org.apache.gora.persistency.impl.PersistentBase } + + private static final org.apache.avro.io.DatumWriter + DATUM_WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); + private static final org.apache.avro.io.DatumReader + DATUM_READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); + + @Override + public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + out.write(super.getDirtyBytes().array()); + DATUM_WRITER$.write + (this, org.apache.avro.io.EncoderFactory.get() + .directBinaryEncoder((java.io.OutputStream) out, + null)); + } + + @Override + public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + byte[] __g__dirty = new byte[getFieldsCount()]; + in.read(__g__dirty); + super.setDirtyBytes(java.nio.ByteBuffer.wrap(__g__dirty)); + DATUM_READER$.read + (this, org.apache.avro.io.DecoderFactory.get() + .directBinaryDecoder((java.io.InputStream) in, + null)); + } } http://git-wip-us.apache.org/repos/asf/gora/blob/a05928a7/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/Pageview.java ---------------------------------------------------------------------- diff --git a/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/Pageview.java b/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/Pageview.java index 4ace856..861be2b 100644 --- a/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/Pageview.java +++ b/gora-tutorial/src/main/java/org/apache/gora/tutorial/log/generated/Pageview.java @@ -3,11 +3,12 @@ * * DO NOT EDIT DIRECTLY */ -package org.apache.gora.tutorial.log.generated; +package org.apache.gora.tutorial.log.generated; + @SuppressWarnings("all") public class Pageview extends org.apache.gora.persistency.impl.PersistentBase implements org.apache.avro.specific.SpecificRecord, org.apache.gora.persistency.Persistent { public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Pageview\",\"namespace\":\"org.apache.gora.tutorial.log.generated\",\"fields\":[{\"name\":\"url\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"timestamp\",\"type\":\"long\",\"default\":0},{\"name\":\"ip\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"httpMethod\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"httpStatusCode\",\"type\":\"int\",\"default\":0},{\"name\":\"responseSize\",\"type\":\"int\",\"default\":0},{\"name\":\"referrer\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"userAgent\",\"type\":[\"null\",\"string\"],\"default\":null}],\"default\":null}"); - + private static final long serialVersionUID = -6468893522296148608L; /** Enum containing all data bean's fields. */ public static enum Field { URL(0, "url"), @@ -806,6 +807,33 @@ public class Pageview extends org.apache.gora.persistency.impl.PersistentBase im } - + + private static final org.apache.avro.io.DatumWriter + DATUM_WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); + private static final org.apache.avro.io.DatumReader + DATUM_READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); + + @Override + public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + out.write(super.getDirtyBytes().array()); + DATUM_WRITER$.write + (this, org.apache.avro.io.EncoderFactory.get() + .directBinaryEncoder((java.io.OutputStream) out, + null)); + } + + @Override + public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + byte[] __g__dirty = new byte[getFieldsCount()]; + in.read(__g__dirty); + super.setDirtyBytes(java.nio.ByteBuffer.wrap(__g__dirty)); + DATUM_READER$.read + (this, org.apache.avro.io.DecoderFactory.get() + .directBinaryDecoder((java.io.InputStream) in, + null)); + } + }
