Repository: spark Updated Branches: refs/heads/branch-1.0 a0e22d398 -> cc2e4ca01
[SQL] Improve Speed of InsertIntoHiveTable Author: Michael Armbrust <[email protected]> Closes #1130 from marmbrus/noFunctional and squashes the following commits: ccdb68c [Michael Armbrust] Remove functional programming and Array allocations from fast path in InsertIntoHiveTable. (cherry picked from commit d3b7671c1f9c1eca956fda15fa7573649fd284b3) Signed-off-by: Reynold Xin <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cc2e4ca0 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cc2e4ca0 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cc2e4ca0 Branch: refs/heads/branch-1.0 Commit: cc2e4ca01ad0b7dff917556580880865a7a0ccd1 Parents: a0e22d3 Author: Michael Armbrust <[email protected]> Authored: Thu Jun 19 23:39:03 2014 -0700 Committer: Reynold Xin <[email protected]> Committed: Thu Jun 19 23:39:14 2014 -0700 ---------------------------------------------------------------------- .../spark/sql/hive/execution/hiveOperators.scala | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/cc2e4ca0/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala index 240aa0d..b195793 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala @@ -371,12 +371,18 @@ case class InsertIntoHiveTable( ObjectInspectorCopyOption.JAVA) .asInstanceOf[StructObjectInspector] + + val fieldOIs = standardOI.getAllStructFieldRefs.map(_.getFieldObjectInspector).toArray + val outputData = new Array[Any](fieldOIs.length) iter.map { row => - // Casts Strings to HiveVarchars when necessary. - val fieldOIs = standardOI.getAllStructFieldRefs.map(_.getFieldObjectInspector) - val mappedRow = row.zip(fieldOIs).map(wrap) + var i = 0 + while (i < row.length) { + // Casts Strings to HiveVarchars when necessary. + outputData(i) = wrap(row(i), fieldOIs(i)) + i += 1 + } - serializer.serialize(mappedRow.toArray, standardOI) + serializer.serialize(outputData, standardOI) } }
