Repository: spark Updated Branches: refs/heads/branch-1.5 d323e5ede -> c8392cd5b
[SPARK-10934] [SQL] handle hashCode of unsafe array correctly `Murmur3_x86_32.hashUnsafeWords` only accepts word-aligned bytes, but unsafe array is not. Author: Wenchen Fan <[email protected]> Closes #8987 from cloud-fan/hash. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c8392cd5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c8392cd5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c8392cd5 Branch: refs/heads/branch-1.5 Commit: c8392cd5bced274e6bf0d9e9345c87bfb9cfe0e2 Parents: d323e5e Author: Wenchen Fan <[email protected]> Authored: Mon Oct 5 17:30:54 2015 -0700 Committer: Davies Liu <[email protected]> Committed: Mon Oct 5 17:33:34 2015 -0700 ---------------------------------------------------------------------- .../spark/sql/catalyst/expressions/UnsafeArrayData.java | 6 +++++- .../src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/c8392cd5/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java index 501dff0..0cf80fe 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java @@ -286,7 +286,11 @@ public class UnsafeArrayData extends ArrayData { @Override public int hashCode() { - return Murmur3_x86_32.hashUnsafeWords(baseObject, baseOffset, sizeInBytes, 42); + int result = 37; + for (int i = 0; i < sizeInBytes; i++) { + result = 37 * result + Platform.getByte(baseObject, baseOffset + i); + } + return result; } @Override http://git-wip-us.apache.org/repos/asf/spark/blob/c8392cd5/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala index 219435d..cbec98e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala @@ -130,4 +130,11 @@ class UnsafeRowSuite extends SparkFunSuite { assert(emptyRow.getInt(0) === unsafeRow.getInt(0)) assert(emptyRow.getUTF8String(1) === unsafeRow.getUTF8String(1)) } + + test("calling hashCode on unsafe array returned by getArray(ordinal)") { + val row = InternalRow.apply(new GenericArrayData(Array(1L))) + val unsafeRow = UnsafeProjection.create(Array[DataType](ArrayType(LongType))).apply(row) + // Makes sure hashCode on unsafe array won't crash + unsafeRow.getArray(0).hashCode() + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
