Repository: giraph Updated Branches: refs/heads/trunk 505c98d9a -> 0c1e2ebce
GIRAPH-1195 closes #76 Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/0c1e2ebc Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/0c1e2ebc Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/0c1e2ebc Branch: refs/heads/trunk Commit: 0c1e2ebce6e4f6e1a47d4dec5c0b8475433ef8df Parents: 505c98d Author: Dionysios Logothetis <dlogothe...@gmail.com> Authored: Thu May 31 11:07:33 2018 -0700 Committer: Dionysios Logothetis <dionys...@fb.com> Committed: Thu May 31 11:07:33 2018 -0700 ---------------------------------------------------------------------- .../apache/giraph/writable/kryo/HadoopKryo.java | 14 ++--- .../ImmutableBiMapSerializerUtils.java | 60 ++++++++++++++++++++ .../serializers/ImmutableMapSerializer.java | 40 ------------- .../writable/kryo/KryoWritableWrapperTest.java | 38 +++++++++++++ pom.xml | 12 +++- 5 files changed, 113 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/0c1e2ebc/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java index 2713316..5d51a3f 100644 --- a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java +++ b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java @@ -27,6 +27,7 @@ import java.util.Map.Entry; import java.util.Random; import com.esotericsoftware.kryo.util.DefaultClassResolver; +import de.javakaffee.kryoserializers.guava.ImmutableMapSerializer; import org.apache.giraph.conf.GiraphConfigurationSettable; import com.esotericsoftware.kryo.ClassResolver; import com.esotericsoftware.kryo.ReferenceResolver; @@ -39,7 +40,7 @@ import org.apache.giraph.writable.kryo.serializers.ArraysAsListSerializer; import org.apache.giraph.writable.kryo.serializers.CollectionsNCopiesSerializer; import org.apache.giraph.writable.kryo.serializers.DirectWritableSerializer; import org.apache.giraph.writable.kryo.serializers.FastUtilSerializer; -import org.apache.giraph.writable.kryo.serializers.ImmutableMapSerializer; +import org.apache.giraph.writable.kryo.serializers.ImmutableBiMapSerializerUtils; import org.apache.giraph.writable.kryo.serializers.ReusableFieldSerializer; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -322,7 +323,8 @@ public class HadoopKryo extends Kryo { if (minor >= '8') { try { kryo.register(Class.forName("java.lang.invoke.SerializedLambda")); - kryo.register(Class.forName("com.esotericsoftware.kryo.Kryo$Closure"), + kryo.register(Class.forName( + "com.esotericsoftware.kryo.serializers.ClosureSerializer$Closure"), new ClosureSerializer()); } catch (ClassNotFoundException e) { throw new IllegalStateException( @@ -336,12 +338,8 @@ public class HadoopKryo extends Kryo { new CollectionsNCopiesSerializer()); ImmutableListSerializer.registerSerializers(kryo); - - registerSerializer(kryo, "com.google.common.collect.RegularImmutableMap", - new ImmutableMapSerializer()); - registerSerializer(kryo, - "com.google.common.collect.SingletonImmutableBiMap", - new ImmutableMapSerializer()); + ImmutableMapSerializer.registerSerializers(kryo); + ImmutableBiMapSerializerUtils.registerSerializers(kryo); // There are many fastutil classes, register them at the end, // so they don't use up small registration numbers http://git-wip-us.apache.org/repos/asf/giraph/blob/0c1e2ebc/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java new file mode 100644 index 0000000..6e66b47 --- /dev/null +++ b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.giraph.writable.kryo.serializers; + + +import com.esotericsoftware.kryo.Kryo; +import com.google.common.collect.ImmutableBiMap; +import de.javakaffee.kryoserializers.guava.ImmutableMapSerializer; + +/** + * Helper class used to register serializers for the different classes + * return by the {@link ImmutableBiMap} methods. + */ +public class ImmutableBiMapSerializerUtils { + + /** + * Private default constructor + */ + private ImmutableBiMapSerializerUtils() { + } + + /** + * Registers different {@link ImmutableBiMap} implementations with Kryo. + * @param kryo {@link Kryo} instance to register class with. + */ + public static void registerSerializers(Kryo kryo) { + ImmutableMapSerializer serializer = new ImmutableMapSerializer(); + kryo.register(ImmutableBiMap.class, serializer); + kryo.register(ImmutableBiMap.of().getClass(), serializer); + Object o1 = new Object(); + Object o2 = new Object(); + Object o3 = new Object(); + Object o4 = new Object(); + Object o5 = new Object(); + kryo.register(ImmutableBiMap.of(o1, o1).getClass(), serializer); + kryo.register(ImmutableBiMap.of(o1, o1, o2, o2).getClass(), serializer); + kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3) + .getClass(), serializer); + kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3, o4, o4) + .getClass(), serializer); + kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3, o4, o4, o5, o5) + .getClass(), serializer); + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/0c1e2ebc/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java deleted file mode 100644 index 8dd2d1c..0000000 --- a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.writable.kryo.serializers; - - -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.serializers.MapSerializer; -import com.google.common.collect.ImmutableMap; - -import java.util.HashMap; -import java.util.Map; - -/** - * Serializer for {@link ImmutableMap} - */ -public class ImmutableMapSerializer extends MapSerializer { - @Override - public Map read(Kryo kryo, Input input, Class<Map> type) { - Map map = super.read(kryo, input, - (Class<Map>) ((Object) HashMap.class)); - return ImmutableMap.copyOf(map); - } -} http://git-wip-us.apache.org/repos/asf/giraph/blob/0c1e2ebc/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java b/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java index 6e0eb2f..9129a4e 100644 --- a/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java +++ b/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java @@ -25,8 +25,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Random; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableMap; import org.apache.giraph.conf.GiraphConfiguration; import org.apache.giraph.conf.GiraphConfigurationSettable; import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; @@ -171,6 +174,41 @@ public class KryoWritableWrapperTest { } @Test + public void testImmutableMapSerialization() throws IOException { + Map original = ImmutableMap.of("x", "y", "y", "z"); + Map copy = kryoSerDeser(original); + assertEquals(original, copy); + } + + @Test + public void testImmutableMapSinglePairSerialization() throws IOException { + Map original = ImmutableMap.of("x", "y"); + Map copy = kryoSerDeser(original); + assertEquals(original, copy); + } + + @Test + public void testImmutableBiMap() throws IOException { + Map original = ImmutableBiMap.of("x", "y", "z", "w"); + Map copy = kryoSerDeser(original); + assertEquals(original, copy); + } + + @Test + public void testSingletonImmutableBiMapSerialization() throws IOException { + Map original = ImmutableBiMap.of("x", "y"); + Map copy = kryoSerDeser(original); + assertEquals(original, copy); + } + + @Test + public void testEmptyImmutableBiMap() throws IOException { + Map original = ImmutableBiMap.of(); + Map copy = kryoSerDeser(original); + assertEquals(original, copy); + } + + @Test public void testFastutilSet() throws ClassNotFoundException, IOException { LongOpenHashSet set = new LongOpenHashSet(); set.add(6); http://git-wip-us.apache.org/repos/asf/giraph/blob/0c1e2ebc/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index fd752fb..1522fd4 100644 --- a/pom.xml +++ b/pom.xml @@ -343,15 +343,16 @@ under the License. <dep.json.version>20160810</dep.json.version> <dep.junit.version>4.11</dep.junit.version> <dep.jython.version>2.5.3</dep.jython.version> - <dep.kryo.version>3.0.1</dep.kryo.version> - <dep.kryo-serializers.version>0.29</dep.kryo-serializers.version> + <dep.kryo.version>4.0.0</dep.kryo.version> + <dep.reflectasm.version>1.11.3</dep.reflectasm.version> + <dep.kryo-serializers.version>0.42</dep.kryo-serializers.version> <dep.libthrift.version>0.9.0</dep.libthrift.version> <dep.log4j.version>1.2.17</dep.log4j.version> <dep.mockito.version>1.9.5</dep.mockito.version> <!-- note: old version of netty is required by hadoop_facebook for tests to succeed --> <dep.netty.version>4.0.14.Final</dep.netty.version> <dep.oldnetty.version>3.2.2.Final</dep.oldnetty.version> - <dep.objenesis.version>2.1</dep.objenesis.version> + <dep.objenesis.version>2.2</dep.objenesis.version> <dep.openhft-compiler.version>2.2.1</dep.openhft-compiler.version> <dep.paranamer.version>2.5.2</dep.paranamer.version> <dep.slf4j.version>1.7.6</dep.slf4j.version> @@ -2050,6 +2051,11 @@ under the License. <version>${dep.kryo.version}</version> </dependency> <dependency> + <groupId>com.esotericsoftware.reflectasm</groupId> + <artifactId>reflectasm</artifactId> + <version>${dep.reflectasm.version}</version> + </dependency> + <dependency> <groupId>de.javakaffee</groupId> <artifactId>kryo-serializers</artifactId> <version>${dep.kryo-serializers.version}</version>
