Hi, I am getting following exception when submitting a job to Spark 1.5.x from Scala. The same code works with Spark 1.4.1. Any clues as to what might causing the exception.
*Code:App.scala*import org.apache.spark.SparkContext object App { def main(args: Array[String]) = { val l = List(1,2,3,4,5,6,7,8,9,0) val sc = new SparkContext("local[4]", "soark-test") val rdd = sc.parallelize(l) rdd.foreach(println) println(rdd.collect()) } } *build.sbt* lazy val sparkjob = (project in file(".")) .settings( name := "SparkJob", version := "1.0", scalaVersion := "2.11.6", libraryDependencies := libs ) lazy val libs = Seq( "org.apache.spark" %% "spark-core" % "1.5.1" ) *Exception:*15/10/22 14:32:42 INFO DAGScheduler: Job 0 failed: foreach at app.scala:9, took 0.689832 s [error] (run-main-0) org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2, localhost): java.io.IOException: java.lang.ClassNotFoundException: scala.Some [error] at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1163) [error] at org.apache.spark.Accumulable.readObject(Accumulators.scala:151) [error] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [error] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [error] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [error] at java.lang.reflect.Method.invoke(Method.java:497) [error] at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1017) [error] at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1896) [error] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) [error] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) [error] at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1959) [error] at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1921) [error] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) [error] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) [error] at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993) [error] at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918) [error] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) [error] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) [error] at java.io.ObjectInputStream.readObject(ObjectInputStream.java:371) [error] at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:72) [error] at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:98) [error] at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:194) [error] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [error] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [error] at java.lang.Thread.run(Thread.java:745) [error] Caused by: java.lang.ClassNotFoundException: scala.Some [error] at java.net.URLClassLoader.findClass(URLClassLoader.java:381) [error] at java.lang.ClassLoader.loadClass(ClassLoader.java:424) [error] at java.lang.ClassLoader.loadClass(ClassLoader.java:357) [error] at java.lang.Class.forName0(Native Method) [error] at java.lang.Class.forName(Class.java:348) [error] at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:67) [error] at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1613) [error] at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1518) [error] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1774) [error] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) [error] at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993) [error] at java.io.ObjectInputStream.defaultReadObject(ObjectInputStream.java:501) [error] at org.apache.spark.Accumulable$$anonfun$readObject$1.apply$mcV$sp(Accumulators.scala:152) [error] at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1160) [error] ... 24 more [error] [error] Driver stacktrace: org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2, localhost): java.io.IOException: java.lang.ClassNotFoundException: scala.Some at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1163) at org.apache.spark.Accumulable.readObject(Accumulators.scala:151) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1017) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1896) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) at java.io.ObjectInputStream.skipCustomData(ObjectInputStream.java:1959) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1921) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) at java.io.ObjectInputStream.readObject(ObjectInputStream.java:371) at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:72) at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:98) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:194) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Thanks, Babar Tareen