Repository: flink Updated Branches: refs/heads/master d5a6b13ab -> cd899f3be
[FLINK-2972] [JavaAPI] Remove Chill dependency from flink-java. This closes #1331 Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/cd899f3b Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/cd899f3b Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/cd899f3b Branch: refs/heads/master Commit: cd899f3be39c5f052a2a3e90771111d1a7df1d47 Parents: d5a6b13 Author: Fabian Hueske <fhue...@apache.org> Authored: Tue Nov 3 16:57:49 2015 +0100 Committer: Fabian Hueske <fhue...@apache.org> Committed: Mon Nov 30 20:31:42 2015 +0100 ---------------------------------------------------------------------- flink-core/pom.xml | 6 ----- flink-java/pom.xml | 17 +++++++------ .../typeutils/runtime/kryo/KryoSerializer.java | 25 ++++++++++++++++++-- flink-runtime/pom.xml | 12 ++++++++++ flink-scala/pom.xml | 2 +- 5 files changed, 46 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/cd899f3b/flink-core/pom.xml ---------------------------------------------------------------------- diff --git a/flink-core/pom.xml b/flink-core/pom.xml index 6763011..f91608e 100644 --- a/flink-core/pom.xml +++ b/flink-core/pom.xml @@ -53,12 +53,6 @@ under the License. </dependency> <dependency> - <groupId>com.twitter</groupId> - <artifactId>chill_${scala.binary.version}</artifactId> - <version>${chill.version}</version> - </dependency> - - <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>${guava.version}</version> http://git-wip-us.apache.org/repos/asf/flink/blob/cd899f3b/flink-java/pom.xml ---------------------------------------------------------------------- diff --git a/flink-java/pom.xml b/flink-java/pom.xml index 4b8e24c..0b33de1 100644 --- a/flink-java/pom.xml +++ b/flink-java/pom.xml @@ -65,13 +65,7 @@ under the License. <dependency> <groupId>com.twitter</groupId> - <artifactId>chill_${scala.binary.version}</artifactId> - <version>${chill.version}</version> - </dependency> - - <dependency> - <groupId>com.twitter</groupId> - <artifactId>chill-avro_${scala.binary.version}</artifactId> + <artifactId>chill-java</artifactId> <version>${chill.version}</version> </dependency> @@ -110,6 +104,15 @@ under the License. <type>test-jar</type> <scope>test</scope> </dependency> + + <dependency> + <groupId>com.twitter</groupId> + <artifactId>chill_${scala.binary.version}</artifactId> + <version>${chill.version}</version> + <!-- For execution, Chill is added to the classpath through flink-runtime. --> + <scope>test</scope> + </dependency> + </dependencies> <!-- Because flink-scala and flink-avro uses it in tests --> http://git-wip-us.apache.org/repos/asf/flink/blob/cd899f3b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java ---------------------------------------------------------------------- diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java index 8549e26..b90901c 100644 --- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java +++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java @@ -26,7 +26,6 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import com.esotericsoftware.kryo.serializers.JavaSerializer; import com.google.common.base.Preconditions; -import com.twitter.chill.ScalaKryoInstantiator; import org.apache.avro.generic.GenericData; import org.apache.flink.api.common.ExecutionConfig; @@ -42,6 +41,8 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.EOFException; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -281,9 +282,29 @@ public class KryoSerializer<T> extends TypeSerializer<T> { // -------------------------------------------------------------------------------------------- + private Kryo getKryoInstance() { + + try { + // check if ScalaKryoInstantiator is in class path (coming from Twitter's Chill library). + // This will be true if Flink's Scala API is used. + Class<?> chillInstantiatorClazz = Class.forName("com.twitter.chill.ScalaKryoInstantiator"); + Object chillInstantiator = chillInstantiatorClazz.newInstance(); + + // obtain a Kryo instance through Twitter Chill + Method m = chillInstantiatorClazz.getMethod("newKryo"); + return (Kryo) m.invoke(chillInstantiator); + } + catch(ClassNotFoundException | InstantiationException | NoSuchMethodException | + IllegalAccessException | InvocationTargetException e ) { + + // Chill must be in the classpath. It is added as a dependency to flink-runtime. + throw new RuntimeException("Could not instantiate Kryo instance from Chill.", e); + } + } + private void checkKryoInitialized() { if (this.kryo == null) { - this.kryo = new ScalaKryoInstantiator().newKryo(); + this.kryo = getKryoInstance(); // Throwable and all subclasses should be serialized via java serialization kryo.addDefaultSerializer(Throwable.class, new JavaSerializer()); http://git-wip-us.apache.org/repos/asf/flink/blob/cd899f3b/flink-runtime/pom.xml ---------------------------------------------------------------------- diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml index a1e0f75..c3093da 100644 --- a/flink-runtime/pom.xml +++ b/flink-runtime/pom.xml @@ -180,6 +180,18 @@ under the License. <artifactId>zookeeper</artifactId> </dependency> + <!-- + The KryoSerializer dynamically loads Kryo instances via Chill and requires that Chill + is in the classpath. Because we do not want to have transitive Scala dependencies + in Flink's API modules (such as flink-java) due to Chill, Chill is added to flink-runtime + to ensure that Chill is always present in the classpath. + --> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>chill_${scala.binary.version}</artifactId> + <version>${chill.version}</version> + </dependency> + <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-shaded-curator-recipes</artifactId> http://git-wip-us.apache.org/repos/asf/flink/blob/cd899f3b/flink-scala/pom.xml ---------------------------------------------------------------------- diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml index 32b5948..d30dad6 100644 --- a/flink-scala/pom.xml +++ b/flink-scala/pom.xml @@ -69,7 +69,7 @@ under the License. <dependency> <groupId>org.ow2.asm</groupId> <artifactId>asm</artifactId> - <version>${asm.version}</version> + <version>${asm.version}</version> </dependency> <dependency>
