Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/19842#discussion_r154045805
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchBenchmark.scala
---
@@ -387,20 +389,108 @@ object ColumnarBatchBenchmark {
}
/*
- String Read/Write: Avg Time(ms) Avg Rate(M/s)
Relative Rate
-
-------------------------------------------------------------------------------------
- On Heap 457.0 35.85
1.00 X
- Off Heap 1206.0 13.59
0.38 X
+ Java HotSpot(TM) 64-Bit Server VM 1.8.0_60-b27 on Mac OS X 10.13.1
+ Intel(R) Core(TM) i7-4960HQ CPU @ 2.60GHz
+
+ String Read/Write: Best/Avg Time(ms)
Rate(M/s) Per Row(ns) Relative
+
------------------------------------------------------------------------------------------------
+ On Heap 332 / 338 49.3
20.3 1.0X
+ Off Heap 466 / 467 35.2
28.4 0.7X
*/
val benchmark = new Benchmark("String Read/Write", count * iters)
benchmark.addCase("On Heap")(column(MemoryMode.ON_HEAP))
benchmark.addCase("Off Heap")(column(MemoryMode.OFF_HEAP))
benchmark.run
}
+ def arrayAccess(iters: Int): Unit = {
+ val random = new Random(0)
+ val count = 4 * 1000
+
+ val onHeapVector = new OnHeapColumnVector(count,
ArrayType(IntegerType))
+ val offHeapVector = new OffHeapColumnVector(count,
ArrayType(IntegerType))
+
+ val minSize = 3
+ val maxSize = 32
+ var arraysCount = 0
+ var elementsCount = 0
+ while (arraysCount < count) {
+ val size = random.nextInt(maxSize - minSize) + minSize
+ val onHeapArrayData = onHeapVector.arrayData()
+ val offHeapArrayData = offHeapVector.arrayData()
+
+ var i = 0
+ while (i < size) {
+ val value = random.nextInt()
+ onHeapArrayData.appendInt(value)
+ offHeapArrayData.appendInt(value)
+ i += 1
+ }
+
+ onHeapVector.putArray(arraysCount, elementsCount, size)
+ offHeapVector.putArray(arraysCount, elementsCount, size)
+ elementsCount += size
+ arraysCount += 1
+ }
+
+ def readArrays(onHeap: Boolean): Unit = {
+ System.gc()
+ val vector = if (onHeap) onHeapVector else offHeapVector
+
+ var sum = 0L
+ for (_ <- 0 until iters) {
+ var i = 0
+ while (i < count) {
+ sum += vector.getArray(i).numElements()
+ i += 1
+ }
+ }
+ }
+
+ def readArrayElements(onHeap: Boolean): Unit = {
+ System.gc()
+ val vector = if (onHeap) onHeapVector else offHeapVector
+
+ var sum = 0L
+ for (_ <- 0 until iters) {
+ var i = 0
+ while (i < count) {
+ val array = vector.getArray(i)
+ val size = array.numElements()
+ var j = 0
+ while (j < size) {
+ sum += array.getInt(j)
+ j += 1
+ }
+ i += 1
+ }
+ }
+ }
+
+ val benchmark = new Benchmark("Array Vector Read", count * iters)
+ benchmark.addCase("On Heap Read Size Only") { _ => readArrays(true) }
+ benchmark.addCase("Off Heap Read Size Only") { _ => readArrays(false) }
+ benchmark.addCase("On Heap Read Elements") { _ =>
readArrayElements(true) }
+ benchmark.addCase("Off Heap Read Elements") { _ =>
readArrayElements(false) }
+
+ /*
+ Java HotSpot(TM) 64-Bit Server VM 1.8.0_60-b27 on Mac OS X 10.13.1
+ Intel(R) Core(TM) i7-4960HQ CPU @ 2.60GHz
+
+ Array Vector Read: Best/Avg Time(ms)
Rate(M/s) Per Row(ns) Relative
+
------------------------------------------------------------------------------------------------
+ On Heap Read Size Only 415 / 422 394.7
2.5 1.0X
+ Off Heap Read Size Only 394 / 402 415.9
2.4 1.1X
+ On Heap Read Elements 2558 / 2593 64.0
15.6 0.2X
+ Off Heap Read Elements 3316 / 3317 49.4
20.2 0.1X
--- End diff --
I understand reusing the object has a good performance.
I am curious whether the current catalyst can generate such a Java code for
accessing nested array elements in SQL
`selectExpr("a[1][2] + a[1][3]")`.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
