Repository: spark Updated Branches: refs/heads/master e33053ee0 -> 98be8169f
[SPARK-11737] [SQL] Fix serialization of UTF8String with Kyro The default implementation of serialization UTF8String with Kyro may be not correct (BYTE_ARRAY_OFFSET could be different across JVM) Author: Davies Liu <[email protected]> Closes #9704 from davies/kyro_string. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/98be8169 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/98be8169 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/98be8169 Branch: refs/heads/master Commit: 98be8169f07eb0f1b8f01776c71d0e1ed3d5e4d5 Parents: e33053e Author: Davies Liu <[email protected]> Authored: Tue Nov 17 19:50:02 2015 -0800 Committer: Davies Liu <[email protected]> Committed: Tue Nov 17 19:50:02 2015 -0800 ---------------------------------------------------------------------- unsafe/pom.xml | 4 ++++ .../apache/spark/unsafe/types/UTF8String.java | 24 ++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/98be8169/unsafe/pom.xml ---------------------------------------------------------------------- diff --git a/unsafe/pom.xml b/unsafe/pom.xml index caf1f77..a1c1111 100644 --- a/unsafe/pom.xml +++ b/unsafe/pom.xml @@ -36,6 +36,10 @@ </properties> <dependencies> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>chill_${scala.binary.version}</artifactId> + </dependency> <!-- Core dependencies --> <dependency> http://git-wip-us.apache.org/repos/asf/spark/blob/98be8169/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java ---------------------------------------------------------------------- diff --git a/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index b7aecb5..4bd3fd7 100644 --- a/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -24,6 +24,11 @@ import java.nio.ByteOrder; import java.util.Arrays; import java.util.Map; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.KryoSerializable; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + import org.apache.spark.unsafe.Platform; import org.apache.spark.unsafe.array.ByteArrayMethods; @@ -38,9 +43,9 @@ import static org.apache.spark.unsafe.Platform.*; * <p> * Note: This is not designed for general use cases, should not be used outside SQL. */ -public final class UTF8String implements Comparable<UTF8String>, Externalizable { +public final class UTF8String implements Comparable<UTF8String>, Externalizable, KryoSerializable { - // These are only updated by readExternal() + // These are only updated by readExternal() or read() @Nonnull private Object base; private long offset; @@ -1003,4 +1008,19 @@ public final class UTF8String implements Comparable<UTF8String>, Externalizable in.readFully((byte[]) base); } + @Override + public void write(Kryo kryo, Output out) { + byte[] bytes = getBytes(); + out.writeInt(bytes.length); + out.write(bytes); + } + + @Override + public void read(Kryo kryo, Input in) { + this.offset = BYTE_ARRAY_OFFSET; + this.numBytes = in.readInt(); + this.base = new byte[numBytes]; + in.read((byte[]) base); + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
