[FLINK-1392] Add Kryo serializer for Protobuf
Conflicts:
flink-java/pom.xml
flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
Conflicts:
flink-shaded/pom.xml
pom.xml
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c4c3db50
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c4c3db50
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c4c3db50
Branch: refs/heads/release-0.8
Commit: c4c3db50a7aa6a7349189f547ee04c1373df5578
Parents: 84c4998
Author: Robert Metzger <rmetz...@apache.org>
Authored: Tue Jan 13 10:21:29 2015 +0100
Committer: Robert Metzger <rmetz...@apache.org>
Committed: Mon Feb 9 14:48:34 2015 +0100
----------------------------------------------------------------------
flink-java/pom.xml | 33 ++++++++++++++++++++
.../java/typeutils/runtime/KryoSerializer.java | 14 ++++++---
flink-shaded/pom.xml | 5 ++-
pom.xml | 6 +++-
4 files changed, 52 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c4c3db50/flink-java/pom.xml
----------------------------------------------------------------------
diff --git a/flink-java/pom.xml b/flink-java/pom.xml
index 4a4bbfa..cc5664b 100644
--- a/flink-java/pom.xml
+++ b/flink-java/pom.xml
@@ -69,6 +69,39 @@ under the License.
<version>0.5.1</version>
</dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>chill-protobuf</artifactId>
+ <version>0.5.1</version>
+ </dependency>
+ <!-- We need protobuf for chill-protobuf -->
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>2.5.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>chill-thrift</artifactId>
+ <version>0.5.1</version>
+ </dependency>
+ <!-- libthrift is required by chill-thrift -->
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <version>0.6.1</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+
<groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<!-- guava needs to be in "provided" scope, to make sure it is
not included into the jars by the shading -->
<dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/c4c3db50/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
----------------------------------------------------------------------
diff --git
a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
index f7a90a5..da09242 100644
---
a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
+++
b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
@@ -26,15 +26,15 @@ import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.esotericsoftware.kryo.serializers.CollectionSerializer;
import com.esotericsoftware.kryo.serializers.JavaSerializer;
+import com.google.protobuf.Message;
import com.twitter.chill.ScalaKryoInstantiator;
-import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.specific.SpecificRecordBase;
+import com.twitter.chill.protobuf.ProtobufSerializer;
+import com.twitter.chill.thrift.TBaseSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
-import scala.reflect.ClassTag;
+import org.apache.thrift.TBase;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -216,6 +216,12 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
// Throwable and all subclasses should be serialized
via java serialization
kryo.addDefaultSerializer(Throwable.class, new
JavaSerializer());
+ // add serializers for popular other serialization
frameworks
+ // Google Protobuf (FLINK-1392)
+ this.kryo.addDefaultSerializer(Message.class,
ProtobufSerializer.class);
+ // thrift
+ this.kryo.addDefaultSerializer(TBase.class,
TBaseSerializer.class);
+
// If the type we have to serialize as a GenricType is
implementing SpecificRecordBase,
// we have to register the avro serializer
// This rule only applies if users explicitly use the
GenericTypeInformation for the avro types
http://git-wip-us.apache.org/repos/asf/flink/blob/c4c3db50/flink-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded/pom.xml b/flink-shaded/pom.xml
index 4dbae5d..51191c7 100644
--- a/flink-shaded/pom.xml
+++ b/flink-shaded/pom.xml
@@ -25,7 +25,7 @@ under the License.
<parent>
<groupId>org.apache.flink</groupId>
<artifactId>flink-parent</artifactId>
- <version>0.8-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
@@ -71,6 +71,9 @@ under the License.
<relocation>
<pattern>com.google</pattern>
<shadedPattern>org.apache.flink.shaded.com.google</shadedPattern>
+ <excludes>
+
<exclude>com.google.protobuf.**</exclude>
+ </excludes>
</relocation>
</relocations>
</configuration>
http://git-wip-us.apache.org/repos/asf/flink/blob/c4c3db50/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e59f340..84690b2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -277,7 +277,8 @@ under the License.
</exclusion>
</exclusions>
</dependency>
- <!-- YARN -->
+
+ <!-- Hadoop 2 Dependencies -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
@@ -853,6 +854,9 @@ under the License.
<relocation>
<pattern>com.google</pattern>
<shadedPattern>org.apache.flink.shaded.com.google</shadedPattern>
+ <excludes>
+
<exclude>com.google.protobuf.**</exclude>
+ </excludes>
</relocation>
</relocations>
</configuration>
