Github user viirya commented on a diff in the pull request:
https://github.com/apache/spark/pull/19813#discussion_r156297038
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
---
@@ -236,4 +237,24 @@ class WholeStageCodegenSuite extends QueryTest with
SharedSQLContext {
}
}
}
+
+ test("SPARK-22551: Fix 64kb limit for deeply nested expressions under
wholestage codegen") {
+ import testImplicits._
+ withTempPath { dir =>
+ val path = dir.getCanonicalPath
+ val df = Seq(("abc", 1)).toDF("key", "int")
+ df.write.parquet(path)
+
+ var strExpr: Expression = col("key").expr
+ for (_ <- 1 to 150) {
+ strExpr = Decode(Encode(strExpr, Literal("utf-8")),
Literal("utf-8"))
+ }
+ val expressions = Seq(If(EqualTo(strExpr, strExpr), strExpr,
strExpr))
+
+ val df2 =
spark.read.parquet(path).select(expressions.map(Column(_)): _*)
+ val plan = df2.queryExecution.executedPlan
+ assert(plan.find(_.isInstanceOf[WholeStageCodegenExec]).isDefined)
--- End diff --
The error message is:
```
[info] Cause: org.codehaus.janino.InternalCompilerException: Code of
method "processNext()V" of class "org.apache
.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator" grows
beyond 64 KB
[info] at org.codehaus.janino.CodeContext.makeSpace(CodeContext.java:990)
[info] at org.codehaus.janino.CodeContext.write(CodeContext.java:867)
[info] at
org.codehaus.janino.UnitCompiler.writeOpcode(UnitCompiler.java:11901)
[info] at
org.codehaus.janino.UnitCompiler.pushConstant(UnitCompiler.java:10362)
[info] at
org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5231)
[info] at
org.codehaus.janino.UnitCompiler.access$8500(UnitCompiler.java:212)
[info] at
org.codehaus.janino.UnitCompiler$12.visitIntegerLiteral(UnitCompiler.java:4099)
[info] at
org.codehaus.janino.UnitCompiler$12.visitIntegerLiteral(UnitCompiler.java:4070)
[info] at org.codehaus.janino.Java$IntegerLiteral.accept(Java.java:5282)
[info] at
org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4070)
[info] at
org.codehaus.janino.UnitCompiler.fakeCompile(UnitCompiler.java:3406)
[info] at
org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5247)
[info] at
org.codehaus.janino.UnitCompiler.compileContext2(UnitCompiler.java:4033)
[info] at
org.codehaus.janino.UnitCompiler.access$6000(UnitCompiler.java:212)
```
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
