Updated Branches: refs/heads/master a3b56322d -> cb92723af
CRUNCH-289 Allow materializing Avro SpecificRecords Due to issues outlined in AVRO-1240, the correct classloader must be supplied to the SpecificData class to allow reading SpecificRecords. This patch passes a custom classloader to the SpecificData constructor to allow locally materializing SpecificRecords when running with "hadoop jar". Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/cb92723a Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/cb92723a Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/cb92723a Branch: refs/heads/master Commit: cb92723af09cee63735cbf6b5dac5e86dd480e9c Parents: a3b5632 Author: Gabriel Reid <[email protected]> Authored: Wed Oct 30 21:42:14 2013 +0100 Committer: Gabriel Reid <[email protected]> Committed: Fri Nov 1 05:10:27 2013 +0100 ---------------------------------------------------------------------- .../apache/crunch/io/avro/AvroFileReaderFactory.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/cb92723a/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java b/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java index becde73..efde4ec 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java @@ -26,6 +26,7 @@ import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.io.DatumReader; import org.apache.avro.mapred.FsInput; import org.apache.avro.reflect.ReflectDatumReader; +import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificDatumReader; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -61,7 +62,7 @@ public class AvroFileReaderFactory<T> implements FileReaderFactory<T> { this.recordReader = new GenericDatumReader<T>(schema); this.mapFn = IdentityFn.<T>getInstance(); } - + static <T> DatumReader<T> createDatumReader(AvroType<T> avroType) { if (avroType.hasReflect()) { if (avroType.hasSpecific()) { @@ -69,7 +70,16 @@ public class AvroFileReaderFactory<T> implements FileReaderFactory<T> { } return new ReflectDatumReader<T>(avroType.getSchema()); } else if (avroType.hasSpecific()) { - return new SpecificDatumReader<T>(avroType.getSchema()); + // Use the classloader of the avro type as the classloader + // for loading SpecificData classes. This is a best-effort + // approach to avoid running into AVRO-1240. We can't just + // use the SpecificDatumReader(Class) constructor because the + // type class here isn't necessarily a SpecificData class, it + // might just contain one as a subtype. + return new SpecificDatumReader<T>( + avroType.getSchema(), + avroType.getSchema(), + new SpecificData(avroType.getClass().getClassLoader())); } else { return new GenericDatumReader<T>(avroType.getSchema()); }
