[
https://issues.apache.org/jira/browse/SPARK-18492?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16383834#comment-16383834
]
imran shaik commented on SPARK-18492:
-------------------------------------
Facing the same issue:
Spark Version: 2.2.1
Scala Version : 2.11.8
Error: org.codehaus.janino.JaninoRuntimeException: Code of method
"processNext()V" of class
"org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator"
grows beyond 64 KB and dumping some huge lines of code(40000) and some
errors(given below) but the stream is continuing without any problem. Console
looks bad though with such huge dumps of code and errors
High Level Logic of Code: Code financial data from Kafka servers, and in the
streaming dataframes had a window duration to groupy timestamp and calculated
aggregators like avg,stddev and after that for each window applied some user
defined functions(just avg, variance,stddev) after slicing each window size to
desired duration and finally made some decisions upon applying some conditions
Full Error(after some 40000 lines of code):
org.codehaus.janino.JaninoRuntimeException: Code of method "processNext()V" of
class
"org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator"
grows beyond 64 KB
at org.codehaus.janino.CodeContext.makeSpace(CodeContext.java:949)
at org.codehaus.janino.CodeContext.write(CodeContext.java:839)
at org.codehaus.janino.UnitCompiler.writeOpcode(UnitCompiler.java:11081)
at org.codehaus.janino.UnitCompiler.load(UnitCompiler.java:10742)
at org.codehaus.janino.UnitCompiler.load(UnitCompiler.java:10729)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:3824)
at org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$12.visitLocalVariableAccess(UnitCompiler.java:3796)
at
org.codehaus.janino.UnitCompiler$12.visitLocalVariableAccess(UnitCompiler.java:3762)
at org.codehaus.janino.Java$LocalVariableAccess.accept(Java.java:3675)
at org.codehaus.janino.Java$Lvalue.accept(Java.java:3563)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:3762)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4933)
at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:4476)
at org.codehaus.janino.UnitCompiler.access$7500(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:3774)
at
org.codehaus.janino.UnitCompiler$12.visitMethodInvocation(UnitCompiler.java:3762)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:4328)
at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:3762)
at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4933)
at org.codehaus.janino.UnitCompiler.compileBoolean2(UnitCompiler.java:3369)
at org.codehaus.janino.UnitCompiler.access$5400(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$10.visitMethodInvocation(UnitCompiler.java:3336)
at
org.codehaus.janino.UnitCompiler$10.visitMethodInvocation(UnitCompiler.java:3324)
at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:4328)
at org.codehaus.janino.UnitCompiler.compileBoolean(UnitCompiler.java:3324)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2216)
at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1378)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$IfStatement.accept(Java.java:2621)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1436)
at org.codehaus.janino.UnitCompiler.access$1600(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1376)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1370)
at org.codehaus.janino.Java$Block.accept(Java.java:2471)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1538)
at org.codehaus.janino.UnitCompiler.access$1900(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$6.visitForStatement(UnitCompiler.java:1379)
at
org.codehaus.janino.UnitCompiler$6.visitForStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$ForStatement.accept(Java.java:2660)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1436)
at org.codehaus.janino.UnitCompiler.access$1600(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1376)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1370)
at org.codehaus.janino.Java$Block.accept(Java.java:2471)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2228)
at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1378)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$IfStatement.accept(Java.java:2621)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1436)
at org.codehaus.janino.UnitCompiler.access$1600(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1376)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1370)
at org.codehaus.janino.Java$Block.accept(Java.java:2471)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2228)
at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1378)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$IfStatement.accept(Java.java:2621)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1436)
at org.codehaus.janino.UnitCompiler.access$1600(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1376)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1370)
at org.codehaus.janino.Java$Block.accept(Java.java:2471)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2228)
at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1378)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$IfStatement.accept(Java.java:2621)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1436)
at org.codehaus.janino.UnitCompiler.access$1600(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1376)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1370)
at org.codehaus.janino.Java$Block.accept(Java.java:2471)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2220)
at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1378)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$IfStatement.accept(Java.java:2621)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1436)
at org.codehaus.janino.UnitCompiler.access$1600(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1376)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1370)
at org.codehaus.janino.Java$Block.accept(Java.java:2471)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2228)
at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1378)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$IfStatement.accept(Java.java:2621)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1436)
at org.codehaus.janino.UnitCompiler.access$1600(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1376)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1370)
at org.codehaus.janino.Java$Block.accept(Java.java:2471)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2228)
at org.codehaus.janino.UnitCompiler.access$1800(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1378)
at
org.codehaus.janino.UnitCompiler$6.visitIfStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$IfStatement.accept(Java.java:2621)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1436)
at org.codehaus.janino.UnitCompiler.access$1600(UnitCompiler.java:206)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1376)
at org.codehaus.janino.UnitCompiler$6.visitBlock(UnitCompiler.java:1370)
at org.codehaus.janino.Java$Block.accept(Java.java:2471)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1725)
at org.codehaus.janino.UnitCompiler.access$2100(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$6.visitWhileStatement(UnitCompiler.java:1381)
at
org.codehaus.janino.UnitCompiler$6.visitWhileStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$WhileStatement.accept(Java.java:2708)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:890)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:894)
at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:377)
at
org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:369)
at org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1128)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1209)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:564)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
at
org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
at
org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
at
org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:311)
at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:229)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1000)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1067)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1064)
at
org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
at
org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
at
org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at
org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at
org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:946)
at
org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:375)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
at
org.apache.spark.sql.execution.exchange.ShuffleExchange.prepareShuffleDependency(ShuffleExchange.scala:88)
at
org.apache.spark.sql.execution.exchange.ShuffleExchange$$anonfun$doExecute$1.apply(ShuffleExchange.scala:124)
at
org.apache.spark.sql.execution.exchange.ShuffleExchange$$anonfun$doExecute$1.apply(ShuffleExchange.scala:115)
at
org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52)
at
org.apache.spark.sql.execution.exchange.ShuffleExchange.doExecute(ShuffleExchange.scala:115)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
at
org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:252)
at org.apache.spark.sql.execution.SortExec.inputRDDs(SortExec.scala:121)
at
org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:386)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116)
at
org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:228)
at
org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:275)
at
org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:2861)
at org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2387)
at org.apache.spark.sql.Dataset$$anonfun$collect$1.apply(Dataset.scala:2387)
at org.apache.spark.sql.Dataset$$anonfun$55.apply(Dataset.scala:2842)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:65)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:2841)
at org.apache.spark.sql.Dataset.collect(Dataset.scala:2387)
at
org.apache.spark.sql.execution.streaming.ConsoleSink.addBatch(console.scala:49)
at
org.apache.spark.sql.execution.streaming.StreamExecution$$anonfun$org$apache$spark$sql$execution$streaming$StreamExecution$$runBatch$1.apply$mcV$sp(StreamExecution.scala:666)
at
org.apache.spark.sql.execution.streaming.StreamExecution$$anonfun$org$apache$spark$sql$execution$streaming$StreamExecution$$runBatch$1.apply(StreamExecution.scala:666)
at
org.apache.spark.sql.execution.streaming.StreamExecution$$anonfun$org$apache$spark$sql$execution$streaming$StreamExecution$$runBatch$1.apply(StreamExecution.scala:666)
at
org.apache.spark.sql.execution.streaming.ProgressReporter$class.reportTimeTaken(ProgressReporter.scala:279)
at
org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:58)
at
org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runBatch(StreamExecution.scala:665)
at
org.apache.spark.sql.execution.streaming.StreamExecution$$anonfun$org$apache$spark$sql$execution$streaming$StreamExecution$$runBatches$1$$anonfun$apply$mcZ$sp$1.apply$mcV$sp(StreamExecution.scala:306)
at
org.apache.spark.sql.execution.streaming.StreamExecution$$anonfun$org$apache$spark$sql$execution$streaming$StreamExecution$$runBatches$1$$anonfun$apply$mcZ$sp$1.apply(StreamExecution.scala:294)
at
org.apache.spark.sql.execution.streaming.StreamExecution$$anonfun$org$apache$spark$sql$execution$streaming$StreamExecution$$runBatches$1$$anonfun$apply$mcZ$sp$1.apply(StreamExecution.scala:294)
at
org.apache.spark.sql.execution.streaming.ProgressReporter$class.reportTimeTaken(ProgressReporter.scala:279)
at
org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:58)
at
org.apache.spark.sql.execution.streaming.StreamExecution$$anonfun$org$apache$spark$sql$execution$streaming$StreamExecution$$runBatches$1.apply$mcZ$sp(StreamExecution.scala:294)
at
org.apache.spark.sql.execution.streaming.ProcessingTimeExecutor.execute(TriggerExecutor.scala:56)
at
org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runBatches(StreamExecution.scala:290)
at
org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:206)
> GeneratedIterator grows beyond 64 KB
> ------------------------------------
>
> Key: SPARK-18492
> URL: https://issues.apache.org/jira/browse/SPARK-18492
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 2.0.1
> Environment: CentOS release 6.7 (Final)
> Reporter: Norris Merritt
> Priority: Major
>
> spark-submit fails with ERROR CodeGenerator: failed to compile:
> org.codehaus.janino.JaninoRuntimeException: Code of method
> "(I[Lscala/collection/Iterator;)V" of class
> "org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator"
> grows beyond 64 KB
> Error message is followed by a huge dump of generated source code.
> The generated code declares 1,454 field sequences like the following:
> /* 036 */ private org.apache.spark.sql.catalyst.expressions.ScalaUDF
> project_scalaUDF1;
> /* 037 */ private scala.Function1 project_catalystConverter1;
> /* 038 */ private scala.Function1 project_converter1;
> /* 039 */ private scala.Function1 project_converter2;
> /* 040 */ private scala.Function2 project_udf1;
> .... (many omitted lines) ...
> /* 6089 */ private org.apache.spark.sql.catalyst.expressions.ScalaUDF
> project_scalaUDF1454;
> /* 6090 */ private scala.Function1 project_catalystConverter1454;
> /* 6091 */ private scala.Function1 project_converter1695;
> /* 6092 */ private scala.Function1 project_udf1454;
> It then proceeds to emit code for several methods (init, processNext) each of
> which has totally repetitive sequences of statements pertaining to each of
> the sequences of variables declared in the class. For example:
> /* 6101 */ public void init(int index, scala.collection.Iterator inputs[]) {
> The reason that the 64KB JVM limit for code for a method is exceeded is
> because the code generator is using an incredibly naive strategy. It emits a
> sequence like the one shown below for each of the 1,454 groups of variables
> shown above, in
> /* 6132 */ this.project_udf =
> (scala.Function1)project_scalaUDF.userDefinedFunc();
> /* 6133 */ this.project_scalaUDF1 =
> (org.apache.spark.sql.catalyst.expressions.ScalaUDF) references[10];
> /* 6134 */ this.project_catalystConverter1 =
> (scala.Function1)org.apache.spark.sql.catalyst.CatalystTypeConverters$.MODULE$.createToCatalystConverter(project_scalaUDF1.dataType());
> /* 6135 */ this.project_converter1 =
> (scala.Function1)org.apache.spark.sql.catalyst.CatalystTypeConverters$.MODULE$.createToScalaConverter(((org.apache.spark.sql.catalyst.expressions.Expression)(((org.apache.spark.sql.catalyst.expressions.ScalaUDF)references[10]).getChildren().apply(0))).dataType());
> /* 6136 */ this.project_converter2 =
> (scala.Function1)org.apache.spark.sql.catalyst.CatalystTypeConverters$.MODULE$.createToScalaConverter(((org.apache.spark.sql.catalyst.expressions.Expression)(((org.apache.spark.sql.catalyst.expressions.ScalaUDF)references[10]).getChildren().apply(1))).dataType());
> It blows up after emitting 230 such sequences, while trying to emit the 231st:
> /* 7282 */ this.project_udf230 =
> (scala.Function2)project_scalaUDF230.userDefinedFunc();
> /* 7283 */ this.project_scalaUDF231 =
> (org.apache.spark.sql.catalyst.expressions.ScalaUDF) references[240];
> /* 7284 */ this.project_catalystConverter231 =
> (scala.Function1)org.apache.spark.sql.catalyst.CatalystTypeConverters$.MODULE$.createToCatalystConverter(project_scalaUDF231.dataType());
> .... many omitted lines ...
> Example of repetitive code sequences emitted for processNext method:
> /* 12253 */ boolean project_isNull247 = project_result244 == null;
> /* 12254 */ MapData project_value247 = null;
> /* 12255 */ if (!project_isNull247) {
> /* 12256 */ project_value247 = project_result244;
> /* 12257 */ }
> /* 12258 */ Object project_arg = sort_isNull5 ? null :
> project_converter489.apply(sort_value5);
> /* 12259 */
> /* 12260 */ ArrayData project_result249 = null;
> /* 12261 */ try {
> /* 12262 */ project_result249 =
> (ArrayData)project_catalystConverter248.apply(project_udf248.apply(project_arg));
> /* 12263 */ } catch (Exception e) {
> /* 12264 */ throw new
> org.apache.spark.SparkException(project_scalaUDF248.udfErrorMessage(), e);
> /* 12265 */ }
> /* 12266 */
> /* 12267 */ boolean project_isNull252 = project_result249 == null;
> /* 12268 */ ArrayData project_value252 = null;
> /* 12269 */ if (!project_isNull252) {
> /* 12270 */ project_value252 = project_result249;
> /* 12271 */ }
> /* 12272 */ Object project_arg1 = project_isNull252 ? null :
> project_converter488.apply(project_value252);
> /* 12273 */
> /* 12274 */ ArrayData project_result248 = null;
> /* 12275 */ try {
> /* 12276 */ project_result248 =
> (ArrayData)project_catalystConverter247.apply(project_udf247.apply(project_arg1));
> /* 12277 */ } catch (Exception e) {
> /* 12278 */ throw new
> org.apache.spark.SparkException(project_scalaUDF247.udfErrorMessage(), e);
> /* 12279 */ }
> /* 12280 */
> /* 12281 */ boolean project_isNull251 = project_result248 == null;
> /* 12282 */ ArrayData project_value251 = null;
> /* 12283 */ if (!project_isNull251) {
> /* 12284 */ project_value251 = project_result248;
> /* 12285 */ }
> /* 12286 */ Object project_arg2 = project_isNull251 ? null :
> project_converter487.apply(project_value251);
> /* 12287 */
> /* 12288 */ InternalRow project_result247 = null;
> /* 12289 */ try {
> /* 12290 */ project_result247 =
> (InternalRow)project_catalystConverter246.apply(project_udf246.apply(project_arg2));
> /* 12291 */ } catch (Exception e) {
> /* 12292 */ throw new
> org.apache.spark.SparkException(project_scalaUDF246.udfErrorMessage(), e);
> /* 12293 */ }
> /* 12294 */
> /* 12295 */ boolean project_isNull250 = project_result247 == null;
> /* 12296 */ InternalRow project_value250 = null;
> /* 12297 */ if (!project_isNull250) {
> /* 12298 */ project_value250 = project_result247;
> /* 12299 */ }
> /* 12300 */ Object project_arg3 = project_isNull250 ? null :
> project_converter486.apply(project_value250);
> /* 12301 */
> /* 12302 */ InternalRow project_result246 = null;
> /* 12303 */ try {
> /* 12304 */ project_result246 =
> (InternalRow)project_catalystConverter245.apply(project_udf245.apply(project_arg3));
> /* 12305 */ } catch (Exception e) {
> /* 12306 */ throw new
> org.apache.spark.SparkException(project_scalaUDF245.udfErrorMessage(), e);
> /* 12307 */ }
> /* 12308 */
> It is pretty clear that the code generation strategy is naive. The code
> generator should use arrays and loops instead of emitting all these
> repetitive code sequences which only differ by a few numerical digits used to
> generate the name of the variables.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
