Can someone help? Thank you! ________________________________
From: Haopu Wang Sent: Monday, June 15, 2015 3:36 PM To: user; d...@spark.apache.org Subject: [SparkStreaming] NPE in DStreamCheckPointData.scala:125 I use the attached program to test checkpoint. It's quite simple. When I run the program second time, it will load checkpoint data, that's expected, however I see NPE in driver log. Do you have any idea about the issue? I'm on Spark 1.4.0, thank you very much! ====== logs ====== 15/06/15 15:27:17 [THREAD ID=main] INFO FileInputDStream: Restoring checkpoint data 15/06/15 15:27:17 [THREAD ID=main] INFO FileInputDStream$FileInputDStreamCheckpointData: Restoring files for time 1434353130000 ms - [] 15/06/15 15:27:17 [THREAD ID=main] INFO FileInputDStream$FileInputDStreamCheckpointData: Restoring files for time 1434353140000 ms - [] 15/06/15 15:27:17 [THREAD ID=main] INFO FileInputDStream$FileInputDStreamCheckpointData: Restoring files for time 1434353150000 ms - [] 15/06/15 15:27:17 [THREAD ID=main] INFO FileInputDStream$FileInputDStreamCheckpointData: Restoring files for time 1434353160000 ms - [] 15/06/15 15:27:17 [THREAD ID=main] INFO FileInputDStream$FileInputDStreamCheckpointData: Restoring files for time 1434353170000 ms - [] 15/06/15 15:27:17 [THREAD ID=main] INFO FileInputDStream$FileInputDStreamCheckpointData: Restoring files for time 1434353180000 ms - [] 15/06/15 15:27:17 [THREAD ID=main] INFO FileInputDStream$FileInputDStreamCheckpointData: Restoring files for time 1434353190000 ms - [] 15/06/15 15:27:17 [THREAD ID=main] INFO FileInputDStream: Restored checkpoint data 15/06/15 15:27:17 [THREAD ID=main] INFO MappedDStream: Restored checkpoint data 15/06/15 15:27:17 [THREAD ID=main] INFO MappedDStream: Restored checkpoint data 15/06/15 15:27:17 [THREAD ID=main] INFO ForEachDStream: Restored checkpoint data 15/06/15 15:27:17 [THREAD ID=main] INFO DStreamGraph: Restored checkpoint data 15/06/15 15:27:17 [THREAD ID=main] ERROR StreamingContext: Error starting the context, marking it as stopped java.io.IOException: java.lang.NullPointerException at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1242) at org.apache.spark.streaming.dstream.DStreamCheckpointData.writeObject(DSt reamCheckpointData.scala:123) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav a:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor Impl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:988) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1495) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1 431) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177) at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:15 47) at java.io.ObjectOutputStream.defaultWriteObject(ObjectOutputStream.java:44 0) at org.apache.spark.streaming.dstream.DStream$$anonfun$writeObject$1.apply$ mcV$sp(DStream.scala:498) at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1239) at org.apache.spark.streaming.dstream.DStream.writeObject(DStream.scala:493 ) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav a:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor Impl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:988) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1495) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1 431) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177) at java.io.ObjectOutputStream.writeArray(ObjectOutputStream.java:1377) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1173) at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:15 47) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1508) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1 431) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177) at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:15 47) at java.io.ObjectOutputStream.defaultWriteObject(ObjectOutputStream.java:44 0) at org.apache.spark.streaming.DStreamGraph$$anonfun$writeObject$1.apply$mcV $sp(DStreamGraph.scala:181) at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1239) at org.apache.spark.streaming.DStreamGraph.writeObject(DStreamGraph.scala:1 76) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav a:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor Impl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:988) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1495) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1 431) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177) at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:15 47) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1508) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1 431) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177) at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:347) at org.apache.spark.streaming.Checkpoint$$anonfun$serialize$1.apply$mcV$sp( Checkpoint.scala:113) at org.apache.spark.streaming.Checkpoint$$anonfun$serialize$1.apply(Checkpo int.scala:113) at org.apache.spark.streaming.Checkpoint$$anonfun$serialize$1.apply(Checkpo int.scala:113) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1285) at org.apache.spark.streaming.Checkpoint$.serialize(Checkpoint.scala:114) at org.apache.spark.streaming.StreamingContext.validate(StreamingContext.sc ala:547) at org.apache.spark.streaming.StreamingContext.liftedTree1$1(StreamingConte xt.scala:587) at org.apache.spark.streaming.StreamingContext.start(StreamingContext.scala :586) at CheckPointTest$delayedInit$body.apply(CheckPointTest.scala:116) at scala.Function0$class.apply$mcV$sp(Function0.scala:40) at scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12) at scala.App$$anonfun$main$1.apply(App.scala:71) at scala.App$$anonfun$main$1.apply(App.scala:71) at scala.collection.immutable.List.foreach(List.scala:318) at scala.collection.generic.TraversableForwarder$class.foreach(TraversableF orwarder.scala:32) at scala.App$class.main(App.scala:71) at CheckPointTest$.main(CheckPointTest.scala:7) at CheckPointTest.main(CheckPointTest.scala) Caused by: java.lang.NullPointerException at org.apache.spark.streaming.dstream.DStreamCheckpointData$$anonfun$writeO bject$1.apply$mcV$sp(DStreamCheckpointData.scala:125) at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1239) ... 64 more