Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/19811#discussion_r156953610
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratedProjectionSuite.scala
---
@@ -219,4 +219,31 @@ class GeneratedProjectionSuite extends SparkFunSuite {
// - one is the mutableRow
assert(globalVariables.length == 3)
}
+
+ test("SPARK-18016: generated projections on wider table requiring state
compaction") {
+ val N = 6000
+ val wideRow1 = new GenericInternalRow((0 until N).toArray[Any])
+ val schema1 = StructType((1 to N).map(i => StructField("",
IntegerType)))
+ val wideRow2 = new GenericInternalRow(
+ (0 until N).map(i => UTF8String.fromString(i.toString)).toArray[Any])
+ val schema2 = StructType((1 to N).map(i => StructField("",
StringType)))
+ val joined = new JoinedRow(wideRow1, wideRow2)
+ val joinedSchema = StructType(schema1 ++ schema2)
+ val nested = new JoinedRow(InternalRow(joined, joined), joined)
+ val nestedSchema = StructType(
+ Seq(StructField("", joinedSchema), StructField("", joinedSchema)) ++
joinedSchema)
+
+ val safeProj = FromUnsafeProjection(nestedSchema)
+ val result = safeProj(nested)
+
+ // test generated MutableProjection
+ val exprs = nestedSchema.fields.zipWithIndex.map { case (f, i) =>
+ BoundReference(i, f.dataType, true)
+ }
+ val mutableProj = GenerateMutableProjection.generate(exprs)
--- End diff --
cc @maropu
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
