Repository: spark Updated Branches: refs/heads/master 278ec8a20 -> d3b7671c1
[SQL] Improve Speed of InsertIntoHiveTable Author: Michael Armbrust <[email protected]> Closes #1130 from marmbrus/noFunctional and squashes the following commits: ccdb68c [Michael Armbrust] Remove functional programming and Array allocations from fast path in InsertIntoHiveTable. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d3b7671c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d3b7671c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d3b7671c Branch: refs/heads/master Commit: d3b7671c1f9c1eca956fda15fa7573649fd284b3 Parents: 278ec8a Author: Michael Armbrust <[email protected]> Authored: Thu Jun 19 23:39:03 2014 -0700 Committer: Reynold Xin <[email protected]> Committed: Thu Jun 19 23:39:03 2014 -0700 ---------------------------------------------------------------------- .../spark/sql/hive/execution/hiveOperators.scala | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/d3b7671c/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala index 240aa0d..b195793 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala @@ -371,12 +371,18 @@ case class InsertIntoHiveTable( ObjectInspectorCopyOption.JAVA) .asInstanceOf[StructObjectInspector] + + val fieldOIs = standardOI.getAllStructFieldRefs.map(_.getFieldObjectInspector).toArray + val outputData = new Array[Any](fieldOIs.length) iter.map { row => - // Casts Strings to HiveVarchars when necessary. - val fieldOIs = standardOI.getAllStructFieldRefs.map(_.getFieldObjectInspector) - val mappedRow = row.zip(fieldOIs).map(wrap) + var i = 0 + while (i < row.length) { + // Casts Strings to HiveVarchars when necessary. + outputData(i) = wrap(row(i), fieldOIs(i)) + i += 1 + } - serializer.serialize(mappedRow.toArray, standardOI) + serializer.serialize(outputData, standardOI) } }
