Updated Branches: refs/heads/master 03dd4b52b -> 0666b0f89
[CRUNCH-43] - Remove tuple re-use for avro serialization Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/0666b0f8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/0666b0f8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/0666b0f8 Branch: refs/heads/master Commit: 0666b0f89e9eb5f78bbcf561e823b1d343aa434f Parents: 03dd4b5 Author: Gabriel Reid <[email protected]> Authored: Sat Aug 11 22:42:57 2012 +0200 Committer: Gabriel Reid <[email protected]> Committed: Sun Aug 12 21:41:30 2012 +0200 ---------------------------------------------------------------------- .../java/org/apache/crunch/types/avro/Avros.java | 21 ++++++++------ .../org/apache/crunch/types/avro/AvrosTest.java | 19 +++++++++++++ 2 files changed, 31 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/0666b0f8/crunch/src/main/java/org/apache/crunch/types/avro/Avros.java ---------------------------------------------------------------------- diff --git a/crunch/src/main/java/org/apache/crunch/types/avro/Avros.java b/crunch/src/main/java/org/apache/crunch/types/avro/Avros.java index 1fd7baa..00a297c 100644 --- a/crunch/src/main/java/org/apache/crunch/types/avro/Avros.java +++ b/crunch/src/main/java/org/apache/crunch/types/avro/Avros.java @@ -458,9 +458,8 @@ public class Avros { private final List<AvroType> avroTypes; private final String jsonSchema; private final boolean isReflect; - - private transient GenericRecord record; - + private transient Schema schema; + public TupleToGenericRecord(Schema schema, PType<?>... ptypes) { this.fns = Lists.newArrayList(); this.avroTypes = Lists.newArrayList(); @@ -493,19 +492,23 @@ public class Avros { @Override public void initialize() { - Schema schema = new Schema.Parser().parse(jsonSchema); - if (isReflect) { - this.record = new ReflectGenericRecord(schema); - } else { - this.record = new GenericData.Record(schema); - } + this.schema = new Schema.Parser().parse(jsonSchema); for (MapFn fn : fns) { fn.setContext(getContext()); } } + + private GenericRecord createRecord(){ + if (isReflect) { + return new ReflectGenericRecord(schema); + } else { + return new GenericData.Record(schema); + } + } @Override public GenericRecord map(Tuple input) { + GenericRecord record = createRecord(); for (int i = 0; i < input.size(); i++) { Object v = input.get(i); if (v == null) { http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/0666b0f8/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java ---------------------------------------------------------------------- diff --git a/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java b/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java index 082ddec..aec19f2 100644 --- a/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java +++ b/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java @@ -17,6 +17,7 @@ */ package org.apache.crunch.types.avro; +import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -37,6 +38,7 @@ import org.apache.crunch.Tuple3; import org.apache.crunch.Tuple4; import org.apache.crunch.TupleN; import org.apache.crunch.test.Person; +import org.apache.crunch.test.StringWrapper; import org.apache.crunch.types.PTableType; import org.apache.crunch.types.PType; import org.apache.hadoop.io.IntWritable; @@ -291,5 +293,22 @@ public class AvrosTest { assertEquals(pair, doubleMappedPair); } + + @Test + public void testPairOutputMapFn_VerifyNoObjectReuse(){ + StringWrapper stringWrapper = new StringWrapper("Test"); + + Pair<Integer,StringWrapper> pair = Pair.of(1, stringWrapper); + + AvroType<Pair<Integer, StringWrapper>> pairType = Avros.pairs(Avros.ints(), Avros.reflects(StringWrapper.class)); + + pairType.getOutputMapFn().initialize(); + + Object outputMappedValueA = pairType.getOutputMapFn().map(pair); + Object outputMappedValueB = pairType.getOutputMapFn().map(pair); + + assertEquals(outputMappedValueA, outputMappedValueB); + assertNotSame(outputMappedValueA, outputMappedValueB); + } }
