Harpreet Chopra created SPARK-22946: ---------------------------------------
Summary: Recursive withColumn calls cause org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificUnsafeProjection" grows beyond 64 KB Key: SPARK-22946 URL: https://issues.apache.org/jira/browse/SPARK-22946 Project: Spark Issue Type: Bug Components: SQL Affects Versions: 2.2.0 Reporter: Harpreet Chopra Recursive calls to withColumn, for updating the same column causes _org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificUnsafeProjection" grows beyond 64 KB _ This can be reproduced in Spark 1.x and also in the latest version we are using (Spark 2.2.0) code to reproduce: import org.apache.spark.sql.functions._ var df =sc.parallelize(Seq(("123","CustOne"),("456","CustTwo"))).toDF("ID","CustName") (1 to 20).foreach(x => { df = df.withColumn("ID", when(col("ID") === "123", lit("678")).otherwise(col("ID"))) println("==== "+x) df.show }) Stack dump: at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.org$apache$spark$sql$cata lyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:555) at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerato r.scala:575) at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerato r.scala:572) at org.spark-project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java :3599) at org.spark-project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379) ... 28 more Caused by: org.codehaus.janino.JaninoRuntimeException: Code of method "(Lorg/apache/spark/sql/cataly st/expressions/GeneratedClass$SpecificUnsafeProjection;Lorg/apache/spark/sql/catalyst/InternalRow;)V " of class "org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificUnsafeProjection" grows beyond 64 KB at org.codehaus.janino.CodeContext.makeSpace(CodeContext.java:941) at org.codehaus.janino.CodeContext.write(CodeContext.java:854) at org.codehaus.janino.UnitCompiler.writeShort(UnitCompiler.java:10242) at org.codehaus.janino.UnitCompiler.load(UnitCompiler.java:9912) at org.codehaus.janino.UnitCompiler.load(UnitCompiler.java:9897) at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:3317) at org.codehaus.janino.UnitCompiler.access$8500(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$10.visitLocalVariableAccess(UnitCompiler.java:3285) at org.codehaus.janino.Java$LocalVariableAccess.accept(Java.java:3189) at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:3290) at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:3313) at org.codehaus.janino.UnitCompiler.access$8000(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$10.visitAmbiguousName(UnitCompiler.java:3280) at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:3138) at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:3290) at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4368) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2669) at org.codehaus.janino.UnitCompiler.access$4500(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$7.visitAssignment(UnitCompiler.java:2619) at org.codehaus.janino.Java$Assignment.accept(Java.java:3405) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2654) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1643) at org.codehaus.janino.UnitCompiler.access$1100(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitExpressionStatement(UnitCompiler.java:936) at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2097) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1007) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:993) at org.codehaus.janino.UnitCompiler.access$1000(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitBlock(UnitCompiler.java:935) at org.codehaus.janino.Java$Block.accept(Java.java:2012) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1753) at org.codehaus.janino.UnitCompiler.access$1200(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitIfStatement(UnitCompiler.java:937) at org.codehaus.janino.Java$IfStatement.accept(Java.java:2157) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1007) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:993) at org.codehaus.janino.UnitCompiler.access$1000(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitBlock(UnitCompiler.java:935) at org.codehaus.janino.Java$Block.accept(Java.java:2012) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1753) at org.codehaus.janino.UnitCompiler.access$1200(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitIfStatement(UnitCompiler.java:937) at org.codehaus.janino.Java$IfStatement.accept(Java.java:2157) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1007) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:993) at org.codehaus.janino.UnitCompiler.access$1000(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitBlock(UnitCompiler.java:935) at org.codehaus.janino.Java$Block.accept(Java.java:2012) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1753) at org.codehaus.janino.UnitCompiler.access$1200(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitIfStatement(UnitCompiler.java:937) at org.codehaus.janino.Java$IfStatement.accept(Java.java:2157) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1007) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:993) at org.codehaus.janino.UnitCompiler.access$1000(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitBlock(UnitCompiler.java:935) at org.codehaus.janino.Java$Block.accept(Java.java:2012) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1753) at org.codehaus.janino.UnitCompiler.access$1200(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitIfStatement(UnitCompiler.java:937) at org.codehaus.janino.Java$IfStatement.accept(Java.java:2157) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1007) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:993) at org.codehaus.janino.UnitCompiler.access$1000(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitBlock(UnitCompiler.java:935) at org.codehaus.janino.Java$Block.accept(Java.java:2012) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1753) at org.codehaus.janino.UnitCompiler.access$1200(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitIfStatement(UnitCompiler.java:937) at org.codehaus.janino.Java$IfStatement.accept(Java.java:2157) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1007) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:993) at org.codehaus.janino.UnitCompiler.access$1000(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitBlock(UnitCompiler.java:935) at org.codehaus.janino.Java$Block.accept(Java.java:2012) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1753) at org.codehaus.janino.UnitCompiler.access$1200(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitIfStatement(UnitCompiler.java:937) at org.codehaus.janino.Java$IfStatement.accept(Java.java:2157) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1007) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:993) at org.codehaus.janino.UnitCompiler.access$1000(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitBlock(UnitCompiler.java:935) at org.codehaus.janino.Java$Block.accept(Java.java:2012) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1753) at org.codehaus.janino.UnitCompiler.access$1200(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitIfStatement(UnitCompiler.java:937) at org.codehaus.janino.Java$IfStatement.accept(Java.java:2157) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1007) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:993) at org.codehaus.janino.UnitCompiler.access$1000(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitBlock(UnitCompiler.java:935) at org.codehaus.janino.Java$Block.accept(Java.java:2012) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1753) at org.codehaus.janino.UnitCompiler.access$1200(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitIfStatement(UnitCompiler.java:937) at org.codehaus.janino.Java$IfStatement.accept(Java.java:2157) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1007) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:993) at org.codehaus.janino.UnitCompiler.access$1000(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitBlock(UnitCompiler.java:935) at org.codehaus.janino.Java$Block.accept(Java.java:2012) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1753) at org.codehaus.janino.UnitCompiler.access$1200(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitIfStatement(UnitCompiler.java:937) at org.codehaus.janino.Java$IfStatement.accept(Java.java:2157) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1007) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:993) at org.codehaus.janino.UnitCompiler.access$1000(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitBlock(UnitCompiler.java:935) at org.codehaus.janino.Java$Block.accept(Java.java:2012) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1753) at org.codehaus.janino.UnitCompiler.access$1200(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitIfStatement(UnitCompiler.java:937) at org.codehaus.janino.Java$IfStatement.accept(Java.java:2157) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1007) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:993) at org.codehaus.janino.UnitCompiler.access$1000(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitBlock(UnitCompiler.java:935) at org.codehaus.janino.Java$Block.accept(Java.java:2012) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:1753) at org.codehaus.janino.UnitCompiler.access$1200(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$4.visitIfStatement(UnitCompiler.java:937) at org.codehaus.janino.Java$IfStatement.accept(Java.java:2157) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:958) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1007) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2293) at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:822) at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:794) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:507) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:658) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:662) at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:350) at org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1035) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:354) at org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:769) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:532) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:393) at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:185) at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:3 47) at org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1139) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:354) at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:322) at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:383) at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:315) at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:233) at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:192) at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:84) at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.org$apache$spark$sql$cata lyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:550) ... 32 more -- This message was sent by Atlassian JIRA (v6.4.14#64029) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org