This is an automated email from the ASF dual-hosted git repository. yamamuro pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 72a349a [SPARK-26857][SQL] Return UnsafeArrayData for date/timestamp type in ColumnarArray.copy() 72a349a is described below commit 72a349a95d7661cd442851b62e7e830f213ed05c Author: Gengliang Wang <gengliang.w...@databricks.com> AuthorDate: Wed Feb 13 10:23:31 2019 +0900 [SPARK-26857][SQL] Return UnsafeArrayData for date/timestamp type in ColumnarArray.copy() ## What changes were proposed in this pull request? In https://github.com/apache/spark/issues/23569, the copy method of `ColumnarArray` is implemented. To further improve it, we can return `UnsafeArrayData` for `date`/`timestamp` type in `ColumnarArray.copy()`. ## How was this patch tested? Unit test Closes #23761 from gengliangwang/copyDateAndTS. Authored-by: Gengliang Wang <gengliang.w...@databricks.com> Signed-off-by: Takeshi Yamamuro <yamam...@apache.org> --- .../apache/spark/sql/vectorized/ColumnarArray.java | 4 ++-- .../execution/vectorized/ColumnVectorSuite.scala | 28 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java b/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java index 1471627..8dc7b11 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java +++ b/sql/core/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java @@ -56,9 +56,9 @@ public final class ColumnarArray extends ArrayData { return UnsafeArrayData.fromPrimitiveArray(toByteArray()); } else if (dt instanceof ShortType) { return UnsafeArrayData.fromPrimitiveArray(toShortArray()); - } else if (dt instanceof IntegerType) { + } else if (dt instanceof IntegerType || dt instanceof DateType) { return UnsafeArrayData.fromPrimitiveArray(toIntArray()); - } else if (dt instanceof LongType) { + } else if (dt instanceof LongType || dt instanceof TimestampType) { return UnsafeArrayData.fromPrimitiveArray(toLongArray()); } else if (dt instanceof FloatType) { return UnsafeArrayData.fromPrimitiveArray(toFloatArray()); diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala index 866fcb1..c2e783d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala @@ -108,6 +108,20 @@ class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach { } } + testVectors("date", 10, DateType) { testVector => + (0 until 10).foreach { i => + testVector.appendInt(i) + } + + val array = new ColumnarArray(testVector, 0, 10) + val arrayCopy = array.copy() + + (0 until 10).foreach { i => + assert(array.get(i, DateType) === i) + assert(arrayCopy.get(i, DateType) === i) + } + } + testVectors("long", 10, LongType) { testVector => (0 until 10).foreach { i => testVector.appendLong(i) @@ -122,6 +136,20 @@ class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach { } } + testVectors("timestamp", 10, TimestampType) { testVector => + (0 until 10).foreach { i => + testVector.appendLong(i) + } + + val array = new ColumnarArray(testVector, 0, 10) + val arrayCopy = array.copy() + + (0 until 10).foreach { i => + assert(array.get(i, TimestampType) === i) + assert(arrayCopy.get(i, TimestampType) === i) + } + } + testVectors("float", 10, FloatType) { testVector => (0 until 10).foreach { i => testVector.appendFloat(i.toFloat) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org