CRUNCH-539 Fix reading WritableComparables bimap Correct the reading of the registered WritableComparables (as serialized in Writables) to get around the fact that the context class loader is not used by ObjectInputStream when resolving classes when this code is called.
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/a727606e Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/a727606e Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/a727606e Branch: refs/heads/master Commit: a727606eaeaea2d78a161685d9d29217864e6846 Parents: 8d9c540 Author: Gabriel Reid <[email protected]> Authored: Wed Jul 8 07:27:32 2015 +0200 Committer: Gabriel Reid <[email protected]> Committed: Thu Jul 9 20:12:51 2015 +0200 ---------------------------------------------------------------------- .../apache/crunch/types/writable/Writables.java | 34 +++++++++++--------- .../crunch/types/writable/WritablesTest.java | 8 +++++ 2 files changed, 26 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/a727606e/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java b/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java index 23775ed..89d08ee 100644 --- a/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java +++ b/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java @@ -18,21 +18,20 @@ package org.apache.crunch.types.writable; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableBiMap; -import org.apache.commons.codec.binary.Base64; import org.apache.crunch.CrunchRuntimeException; import org.apache.crunch.MapFn; import org.apache.crunch.Pair; @@ -130,22 +129,25 @@ public class Writables { private static final String WRITABLE_COMPARABLE_CODES = "crunch.writable.comparable.codes"; - private static void serializeWritableComparableCodes(Configuration conf) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(baos); - oos.writeObject(WRITABLE_CODES); - oos.close(); - conf.set(WRITABLE_COMPARABLE_CODES, Base64.encodeBase64String(baos.toByteArray())); + static void serializeWritableComparableCodes(Configuration conf) throws IOException { + Map<Integer, String> codeToClassNameMap = Maps.transformValues(WRITABLE_CODES, + new Function<Class<? extends Writable>, String>() { + @Override + public String apply(Class<? extends Writable> input) { + return input.getName(); + } + }); + conf.set(WRITABLE_COMPARABLE_CODES, Joiner.on(';').withKeyValueSeparator(":").join(codeToClassNameMap)); } static void reloadWritableComparableCodes(Configuration conf) throws Exception { if (conf.get(WRITABLE_COMPARABLE_CODES) != null) { - ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(conf.get(WRITABLE_COMPARABLE_CODES))); - ObjectInputStream ois = new ObjectInputStream(bais); - BiMap<Integer, Class<? extends Writable>> codes = (BiMap<Integer, Class<? extends Writable>>) ois.readObject(); - ois.close(); - for (Map.Entry<Integer, Class<? extends Writable>> e : codes.entrySet()) { - WRITABLE_CODES.put(e.getKey(), e.getValue()); + Map<String, String> codeToClassName = Splitter.on(';') + .withKeyValueSeparator(":").split(conf.get(WRITABLE_COMPARABLE_CODES)); + for (Map.Entry<String, String> codeToClassNameEntry : codeToClassName.entrySet()) { + WRITABLE_CODES.put( + Integer.parseInt(codeToClassNameEntry.getKey()), + (Class<? extends Writable>) Class.forName(codeToClassNameEntry.getValue())); } } } http://git-wip-us.apache.org/repos/asf/crunch/blob/a727606e/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java b/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java index 9af3dea..f5c40d7 100644 --- a/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java +++ b/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java @@ -34,6 +34,7 @@ import org.apache.crunch.Tuple4; import org.apache.crunch.TupleN; import org.apache.crunch.types.PTableType; import org.apache.crunch.types.PType; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.DoubleWritable; @@ -252,6 +253,13 @@ public class WritablesTest { public void testRegisterComparable() throws Exception { Writables.registerComparable(TestWritable.class); assertNotNull(Writables.WRITABLE_CODES.inverse().get(TestWritable.class)); + + // Also try serializing/deserializing from a configuration, just to make sure this works + Configuration conf = new Configuration(); + Writables.serializeWritableComparableCodes(conf); + Writables.WRITABLE_CODES.clear(); + Writables.reloadWritableComparableCodes(conf); + assertNotNull(Writables.WRITABLE_CODES.inverse().get(TestWritable.class)); } @SuppressWarnings({ "unchecked", "rawtypes" })
