Github user kiszk commented on a diff in the pull request:
https://github.com/apache/spark/pull/19811#discussion_r156300746
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratedProjectionSuite.scala
---
@@ -219,4 +219,55 @@ class GeneratedProjectionSuite extends SparkFunSuite {
// - one is the mutableRow
assert(globalVariables.length == 3)
}
+
+ test("SPARK-18016: generated projections on wider table requiring state
compaction") {
+ val N = 6000
+ val wideRow1 = new GenericInternalRow((0 until N).toArray[Any])
+ val schema1 = StructType((1 to N).map(i => StructField("",
IntegerType)))
+ val wideRow2 = new GenericInternalRow(
+ (0 until N).map(i => UTF8String.fromString(i.toString)).toArray[Any])
+ val schema2 = StructType((1 to N).map(i => StructField("",
StringType)))
+ val joined = new JoinedRow(wideRow1, wideRow2)
+ val joinedSchema = StructType(schema1 ++ schema2)
+ val nested = new JoinedRow(InternalRow(joined, joined), joined)
+ val nestedSchema = StructType(
+ Seq(StructField("", joinedSchema), StructField("", joinedSchema)) ++
joinedSchema)
+
+ // test generated UnsafeProjection
+ val unsafeProj = UnsafeProjection.create(nestedSchema)
--- End diff --
Good catch, done
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
