Hi all,
I encounter an error at spark 1.4.0, and I make an error example as
following. Both of the code can run OK on spark-shell, but the second code
encounter an error using spark-submit. The only different is that the
second code uses a literal function in the map(). but the first code uses a
defined function in the map().
Could anyone tell me why this error happen? Thanks
======================the ok code =================
val sparkConf = new SparkConf().setAppName("Mode Example")
val sc = new SparkContext(sparkConf)
val hive = new HiveContext(sc)
val rdd = sc.parallelize(Range(0,1000))
def fff = (input: Int)=>input+1
val cnt = rdd.map(i => Array(1,2,3)).map(arr => {
arr.map(fff)
}).count()
print(cnt)
==============not ok code ============
val sparkConf = new SparkConf().setAppName("Mode Example")
val sc = new SparkContext(sparkConf)
val hive = new HiveContext(sc)
val rdd = sc.parallelize(Range(0,1000))
//def fff = (input: Int)=>input+1
val cnt = rdd.map(i => Array(1,2,3)).map(arr => {
arr.map(_+1)
}).count()
print(cnt)
Submit command: spark-submit --class com.yhd.ycache.magic.Model --jars
./SSExample-0.0.2-SNAPSHOT-jar-with-dependencies.jar --master local
./SSExample-0.0.2-SNAPSHOT-jar-with-dependencies.jar
the error info:
Exception in thread "main" java.lang.ClassNotFoundException:
com.yhd.ycache.magic.Model$$anonfun$10$$anonfun$apply$1
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:270)
at
org.apache.spark.util.InnerClosureFinder$$anon$4.visitMethodInsn(ClosureCleaner.scala:455)
at
com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.ClassReader.accept(Unknown
Source)
at
com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.ClassReader.accept(Unknown
Source)
at
org.apache.spark.util.ClosureCleaner$.getInnerClosureClasses(ClosureCleaner.scala:101)
at
org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:197)
at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:132)
at org.apache.spark.SparkContext.clean(SparkContext.scala:1891)
at org.apache.spark.rdd.RDD$$anonfun$map$1.apply(RDD.scala:294)
at org.apache.spark.rdd.RDD$$anonfun$map$1.apply(RDD.scala:293)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:148)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:109)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:286)
at org.apache.spark.rdd.RDD.map(RDD.scala:293)
at com.yhd.ycache.magic.Model$.main(SSExample.scala:238)
at com.yhd.ycache.magic.Model.main(SSExample.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:664)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:169)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:192)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:111)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
