Repository: spark Updated Branches: refs/heads/master 5a2330e54 -> c740bed17
[SPARK-9373][SQL] follow up for StructType support in Tungsten projection. Author: Reynold Xin <[email protected]> Closes #7720 from rxin/struct-followup and squashes the following commits: d9757f5 [Reynold Xin] [SPARK-9373][SQL] follow up for StructType support in Tungsten projection. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c740bed1 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c740bed1 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c740bed1 Branch: refs/heads/master Commit: c740bed17215a9608c9eb9d80ffdf0fcf72c3911 Parents: 5a2330e Author: Reynold Xin <[email protected]> Authored: Tue Jul 28 09:43:12 2015 -0700 Committer: Reynold Xin <[email protected]> Committed: Tue Jul 28 09:43:12 2015 -0700 ---------------------------------------------------------------------- .../catalyst/expressions/UnsafeRowWriters.java | 6 +-- .../codegen/GenerateUnsafeProjection.scala | 40 +++++++++----------- .../spark/sql/execution/SparkStrategies.scala | 3 +- 3 files changed, 23 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/c740bed1/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java index 8fdd739..32faad3 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java @@ -47,7 +47,7 @@ public class UnsafeRowWriters { target.getBaseObject(), offset + ((numBytes >> 3) << 3), 0L); } - // Write the string to the variable length portion. + // Write the bytes to the variable length portion. input.writeToMemory(target.getBaseObject(), offset); // Set the fixed length portion. @@ -73,7 +73,7 @@ public class UnsafeRowWriters { target.getBaseObject(), offset + ((numBytes >> 3) << 3), 0L); } - // Write the string to the variable length portion. + // Write the bytes to the variable length portion. ByteArray.writeToMemory(input, target.getBaseObject(), offset); // Set the fixed length portion. @@ -115,7 +115,7 @@ public class UnsafeRowWriters { target.getBaseObject(), offset + ((numBytes >> 3) << 3), 0L); } - // Write the string to the variable length portion. + // Write the bytes to the variable length portion. row.writeToMemory(target.getBaseObject(), offset); // Set the fixed length portion. http://git-wip-us.apache.org/repos/asf/spark/blob/c740bed1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala index 3e87f72..9a4c00e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala @@ -62,14 +62,10 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro val cursor = ctx.freshName("cursor") val numBytes = ctx.freshName("numBytes") - val exprs = expressions.zipWithIndex.map { case (e, i) => - e.dataType match { - case st: StructType => - createCodeForStruct(ctx, e.gen(ctx), st) - case _ => - e.gen(ctx) - } - } + val exprs = expressions.map { e => e.dataType match { + case st: StructType => createCodeForStruct(ctx, e.gen(ctx), st) + case _ => e.gen(ctx) + }} val allExprs = exprs.map(_.code).mkString("\n") val fixedSize = 8 * exprs.length + UnsafeRow.calculateBitSetWidthInBytes(exprs.length) @@ -153,20 +149,20 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro val exprs: Seq[GeneratedExpressionCode] = schema.map(_.dataType).zipWithIndex.map { case (dt, i) => dt match { - case st: StructType => - val nestedStructEv = GeneratedExpressionCode( - code = "", - isNull = s"${input.primitive}.isNullAt($i)", - primitive = s"${ctx.getColumn(input.primitive, dt, i)}" - ) - createCodeForStruct(ctx, nestedStructEv, st) - case _ => - GeneratedExpressionCode( - code = "", - isNull = s"${input.primitive}.isNullAt($i)", - primitive = s"${ctx.getColumn(input.primitive, dt, i)}" - ) - } + case st: StructType => + val nestedStructEv = GeneratedExpressionCode( + code = "", + isNull = s"${input.primitive}.isNullAt($i)", + primitive = s"${ctx.getColumn(input.primitive, dt, i)}" + ) + createCodeForStruct(ctx, nestedStructEv, st) + case _ => + GeneratedExpressionCode( + code = "", + isNull = s"${input.primitive}.isNullAt($i)", + primitive = s"${ctx.getColumn(input.primitive, dt, i)}" + ) + } } val allExprs = exprs.map(_.code).mkString("\n") http://git-wip-us.apache.org/repos/asf/spark/blob/c740bed1/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala index 314b85f..f3ef066 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala @@ -339,7 +339,8 @@ private[sql] abstract class SparkStrategies extends QueryPlanner[SparkPlan] { * if necessary. */ def getSortOperator(sortExprs: Seq[SortOrder], global: Boolean, child: SparkPlan): SparkPlan = { - if (sqlContext.conf.unsafeEnabled && UnsafeExternalSort.supportsSchema(child.schema)) { + if (sqlContext.conf.unsafeEnabled && sqlContext.conf.codegenEnabled && + UnsafeExternalSort.supportsSchema(child.schema)) { execution.UnsafeExternalSort(sortExprs, global, child) } else if (sqlContext.conf.externalSortEnabled) { execution.ExternalSort(sortExprs, global, child) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
