Github user viirya commented on a diff in the pull request:
https://github.com/apache/spark/pull/19480#discussion_r144459033
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
---
@@ -201,6 +201,23 @@ class CodeGenerationSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
}
+ test("SPARK-22226: group splitted expressions into one method per nested
class") {
--- End diff --
@mgaido91 I can reproduce the issue by following test case. You can check
it:
```scala
test("SPARK-22226: too much splitted expressions should not exceen
constant pool limit") {
withSQLConf(
(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false")) {
val colNumber = 1000
val baseDF = spark.range(10).toDF()
val newCols = (1 to colNumber).map { colIndex =>
expr(s"id + $colIndex").as(s"_$colIndex")
}
val input = baseDF.select(newCols: _*)
val aggs2 = (1 to colNumber).flatMap { colIndex =>
val colName = s"_$colIndex"
Seq(expr(s"stddev($colName)"),
expr(s"stddev_samp($colName)"),
expr(s"stddev_pop($colName)"),
expr(s"variance($colName)"),
expr(s"var_samp($colName)"),
expr(s"var_pop($colName)"),
expr(s"skewness($colName)"),
expr(s"kurtosis($colName)"))
}
input.agg(aggs2.head, aggs2.tail: _*).collect()
}
}
```
```
[info] Cause: org.codehaus.janino.JaninoRuntimeException: failed to
compile: org.codehaus.janino.JaninoRuntimeExc
eption: Constant pool for class
org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificMutableProjection
has grown past JVM limit of 0xFFFF
[info] at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressi
ons$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1079)
```
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
