Repository: spark Updated Branches: refs/heads/branch-1.5 f33e277f9 -> f7a7230f3
[SPARK-11737] [SQL] Fix serialization of UTF8String with Kyro The default implementation of serialization UTF8String with Kyro may be not correct (BYTE_ARRAY_OFFSET could be different across JVM) Author: Davies Liu <[email protected]> Closes #9704 from davies/kyro_string. (cherry picked from commit 98be8169f07eb0f1b8f01776c71d0e1ed3d5e4d5) Signed-off-by: Davies Liu <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f7a7230f Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f7a7230f Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f7a7230f Branch: refs/heads/branch-1.5 Commit: f7a7230f39e70dd011ca8d10bfd855d3c8bf180d Parents: f33e277 Author: Davies Liu <[email protected]> Authored: Tue Nov 17 19:50:02 2015 -0800 Committer: Davies Liu <[email protected]> Committed: Tue Nov 17 19:50:27 2015 -0800 ---------------------------------------------------------------------- unsafe/pom.xml | 4 ++++ .../apache/spark/unsafe/types/UTF8String.java | 24 ++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/f7a7230f/unsafe/pom.xml ---------------------------------------------------------------------- diff --git a/unsafe/pom.xml b/unsafe/pom.xml index 95ffec6..b0859db 100644 --- a/unsafe/pom.xml +++ b/unsafe/pom.xml @@ -36,6 +36,10 @@ </properties> <dependencies> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>chill_${scala.binary.version}</artifactId> + </dependency> <!-- Core dependencies --> <dependency> http://git-wip-us.apache.org/repos/asf/spark/blob/f7a7230f/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java ---------------------------------------------------------------------- diff --git a/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index cbcab95..c6824a4 100644 --- a/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -23,6 +23,11 @@ import java.nio.ByteOrder; import java.util.Arrays; import java.util.Map; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.KryoSerializable; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + import org.apache.spark.unsafe.Platform; import org.apache.spark.unsafe.array.ByteArrayMethods; @@ -37,9 +42,9 @@ import static org.apache.spark.unsafe.Platform.*; * <p> * Note: This is not designed for general use cases, should not be used outside SQL. */ -public final class UTF8String implements Comparable<UTF8String>, Externalizable { +public final class UTF8String implements Comparable<UTF8String>, Externalizable, KryoSerializable { - // These are only updated by readExternal() + // These are only updated by readExternal() or read() @Nonnull private Object base; private long offset; @@ -997,4 +1002,19 @@ public final class UTF8String implements Comparable<UTF8String>, Externalizable in.readFully((byte[]) base); } + @Override + public void write(Kryo kryo, Output out) { + byte[] bytes = getBytes(); + out.writeInt(bytes.length); + out.write(bytes); + } + + @Override + public void read(Kryo kryo, Input in) { + this.offset = BYTE_ARRAY_OFFSET; + this.numBytes = in.readInt(); + this.base = new byte[numBytes]; + in.read((byte[]) base); + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
