Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/20637#discussion_r211365612
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala
---
@@ -43,25 +45,30 @@ object GenerateUnsafeProjection extends
CodeGenerator[Seq[Expression], UnsafePro
case _ => false
}
- // TODO: if the nullability of field is correct, we can use it to save
null check.
private def writeStructToBuffer(
ctx: CodegenContext,
input: String,
index: String,
- fieldTypes: Seq[DataType],
+ fieldTypeAndNullables: Seq[Schema],
rowWriter: String): String = {
// Puts `input` in a local variable to avoid to re-evaluate it if it's
a statement.
val tmpInput = ctx.freshName("tmpInput")
- val fieldEvals = fieldTypes.zipWithIndex.map { case (dt, i) =>
- ExprCode(
- JavaCode.isNullExpression(s"$tmpInput.isNullAt($i)"),
- JavaCode.expression(CodeGenerator.getValue(tmpInput, dt,
i.toString), dt))
+ val fieldEvals = fieldTypeAndNullables.zipWithIndex.map { case
(dtNullable, i) =>
+ val isNull = if (dtNullable.nullable) {
+ JavaCode.isNullExpression(s"$tmpInput.isNullAt($i)")
+ } else {
+ FalseLiteral
+ }
+ ExprCode(isNull, JavaCode.expression(
+ CodeGenerator.getValue(tmpInput, dtNullable.dataType, i.toString),
dtNullable.dataType))
}
val rowWriterClass = classOf[UnsafeRowWriter].getName
val structRowWriter = ctx.addMutableState(rowWriterClass, "rowWriter",
v => s"$v = new $rowWriterClass($rowWriter, ${fieldEvals.length});")
val previousCursor = ctx.freshName("previousCursor")
+ val structExpressions = writeExpressionsToBuffer(
+ ctx, tmpInput, fieldEvals, fieldTypeAndNullables.map(_.dataType),
structRowWriter)
--- End diff --
I think `nullable` is passed thru `fieldEvals`.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
