Repository: crunch Updated Branches: refs/heads/master 7f85ee581 -> 2bd612eb9
CRUNCH-540: Make AvroReflectDeepCopier serializable (for real though.) Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/2bd612eb Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/2bd612eb Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/2bd612eb Branch: refs/heads/master Commit: 2bd612eb9c115f857f5776f58fbd336028a438b3 Parents: 7f85ee5 Author: Josh Wills <[email protected]> Authored: Tue Jul 7 14:16:07 2015 -0700 Committer: Josh Wills <[email protected]> Committed: Tue Jul 7 14:16:07 2015 -0700 ---------------------------------------------------------------------- .../crunch/types/avro/AvroDeepCopier.java | 8 +++--- .../org/apache/crunch/types/avro/AvroType.java | 2 +- .../crunch/types/avro/AvroDeepCopierTest.java | 30 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/2bd612eb/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java index dcc4a19..5000f8d 100644 --- a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java +++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroDeepCopier.java @@ -99,10 +99,10 @@ abstract class AvroDeepCopier<T> implements DeepCopier<T>, Serializable { */ public static class AvroReflectDeepCopier<T> extends AvroDeepCopier<T> { - private DatumReader<T> datumReader; - private DatumWriter<T> datumWriter; - private BinaryEncoder binaryEncoder; - private BinaryDecoder binaryDecoder; + private transient DatumReader<T> datumReader; + private transient DatumWriter<T> datumWriter; + private transient BinaryEncoder binaryEncoder; + private transient BinaryDecoder binaryDecoder; private final Class<T> valueClass; public AvroReflectDeepCopier(Class<T> valueClass, Schema schema) { http://git-wip-us.apache.org/repos/asf/crunch/blob/2bd612eb/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroType.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroType.java b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroType.java index 528a600..97278f0 100644 --- a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroType.java +++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroType.java @@ -70,7 +70,7 @@ public class AvroType<T> implements PType<T> { private final List<PType> subTypes; private AvroRecordType recordType; private DeepCopier<T> deepCopier; - private boolean initialized = false; + private transient boolean initialized = false; public AvroType(Class<T> typeClass, Schema schema, DeepCopier<T> deepCopier, PType... ptypes) { this(typeClass, schema, IdentityFn.getInstance(), IdentityFn.getInstance(), deepCopier, null, ptypes); http://git-wip-us.apache.org/repos/asf/crunch/blob/2bd612eb/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroDeepCopierTest.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroDeepCopierTest.java b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroDeepCopierTest.java index da8dd28..4f5cb75 100644 --- a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroDeepCopierTest.java +++ b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroDeepCopierTest.java @@ -22,11 +22,16 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.nio.ByteBuffer; import com.google.common.collect.Lists; import org.apache.avro.generic.GenericData.Record; import org.apache.crunch.test.Person; +import org.apache.crunch.types.PType; import org.apache.crunch.types.avro.AvroDeepCopier.AvroSpecificDeepCopier; import org.apache.hadoop.conf.Configuration; import org.junit.Test; @@ -88,6 +93,31 @@ public class AvroDeepCopierTest { } @Test + public void testSerializableReflectPType() throws Exception { + ReflectedPerson person = new ReflectedPerson(); + person.setName("John Doe"); + person.setAge(42); + person.setSiblingnames(Lists.<String>newArrayList()); + + PType<ReflectedPerson> rptype = Avros.reflects(ReflectedPerson.class); + rptype.initialize(new Configuration()); + ReflectedPerson copy1 = rptype.getDetachedValue(person); + assertEquals(copy1, person); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos); + oos.writeObject(rptype); + oos.close(); + + ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray())); + rptype = (PType<ReflectedPerson>) ois.readObject(); + + rptype.initialize(new Configuration()); + ReflectedPerson copy2 = rptype.getDetachedValue(person); + assertEquals(person, copy2); + } + + @Test public void testDeepCopyReflect_Null() { AvroDeepCopier<ReflectedPerson> avroDeepCopier = new AvroDeepCopier.AvroReflectDeepCopier<ReflectedPerson>( ReflectedPerson.class, Avros.reflects(ReflectedPerson.class).getSchema());
