Re: 答复: 答复: 答复: 答复: spark streaming context trigger invoke stop why?
t; at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) > > at > org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:902) > > at > org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:900) > > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147) > > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:108) > > at org.apache.spark.rdd.RDD.withScope(RDD.scala:310) > > at org.apache.spark.rdd.RDD.foreachPartition(RDD.scala:900) > > at > org.apache.spark.api.java.JavaRDDLike$class.foreachPartition(JavaRDDLike.scala:222) > > at > org.apache.spark.api.java.AbstractJavaRDDLike.foreachPartition(JavaRDDLike.scala:47) > > at > com.vip.ubt.spark.streaming.MarsScLogMetric$1.call(MarsScLogMetric.java:40) > > at > com.vip.ubt.spark.streaming.MarsScLogMetric$1.call(MarsScLogMetric.java:36) > > at > org.apache.spark.streaming.api.java.JavaDStreamLike$$anonfun$foreachRDD$1.apply(JavaDStreamLike.scala:315) > > at > org.apache.spark.streaming.api.java.JavaDStreamLike$$anonfun$foreachRDD$1.apply(JavaDStreamLike.scala:315) > > at > org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:631) > > at > org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:631) > > at > org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ForEachDStream.scala:42) > > at > org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:40) > >at > org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:40) > > at > org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:399) > > at > org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply$mcV$sp(ForEachDStream.scala:40) > > at > org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:40) > > at > org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:40) > > at scala.util.Try$.apply(Try.scala:161) > > at org.apache.spark.streaming.scheduler.Job.run(Job.scala:34) > > at > org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply$mcV$sp(JobScheduler.scala:218) > > at > org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:218) > > at > org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:218) > > at scala.util.DynamicVariable.withValue(DynamicVariable.scala:57) > > at > org.apache.spark.streaming.scheduler.JobScheduler$JobHandler.run(JobScheduler.scala:217) > > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > > at java.lang.Thread.run(Thread.java:745) > > Caused by: java.lang.Exception: Could not compute split, block > input-22-1452641669000 not found > > at org.apache.spark.rdd.BlockRDD.compute(BlockRDD.scala:51) > > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:300) > > at org.apache.spark.rdd.RDD.iterator(RDD.scala:264) > > at org.apache.spark.rdd.UnionRDD.compute(UnionRDD.scala:87) > > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:300) > > at org.apache.spark.rdd.RDD.iterator(RDD.scala:264) > > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) > > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:300) > > at org.apache.spark.rdd.RDD.iterator(RDD.scala:264) > > at > org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) > > at org.apache.spark.scheduler.Task.run(Task.scala:88) > > at > org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) > > ... 3 more > > 16/01/13 07:35:42 INFO > [org.apache.spark.streaming.StreamingContext---Thread-0]: Invoking > stop(stopGracefully=false) from shutdown hook > > 16/01/13 07:35:42 INFO > [org.apache.spark.streaming.scheduler.ReceiverTracker---sparkDriver-akka.actor.default-dispatcher-4]: > Sent stop signal to all 42 receivers > > > > &
答复: 答复: 答复: 答复: spark streaming context trigger invoke stop why?
$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:631) at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:631) at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ForEachDStream.scala:42) at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:40) at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:40) at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:399) at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply$mcV$sp(ForEachDStream.scala:40) at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:40) at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:40) at scala.util.Try$.apply(Try.scala:161) at org.apache.spark.streaming.scheduler.Job.run(Job.scala:34) at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply$mcV$sp(JobScheduler.scala:218) at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:218) at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:218) at scala.util.DynamicVariable.withValue(DynamicVariable.scala:57) at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler.run(JobScheduler.scala:217) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.Exception: Could not compute split, block input-22-1452641669000 not found at org.apache.spark.rdd.BlockRDD.compute(BlockRDD.scala:51) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:300) at org.apache.spark.rdd.RDD.iterator(RDD.scala:264) at org.apache.spark.rdd.UnionRDD.compute(UnionRDD.scala:87) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:300) at org.apache.spark.rdd.RDD.iterator(RDD.scala:264) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:300) at org.apache.spark.rdd.RDD.iterator(RDD.scala:264) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:88) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) ... 3 more 16/01/13 07:35:42 INFO [org.apache.spark.streaming.StreamingContext---Thread-0]: Invoking stop(stopGracefully=false) from shutdown hook 16/01/13 07:35:42 INFO [org.apache.spark.streaming.scheduler.ReceiverTracker---sparkDriver-akka.actor.default-dispatcher-4]: Sent stop signal to all 42 receivers 发件人: Shixiong(Ryan) Zhu [mailto:shixi...@databricks.com] 发送时间: 2016年1月16日 6:28 收件人: 邓刚[技术中心] 抄送: Yogesh Mahajan; user 主题: Re: 答复: 答复: 答复: spark streaming context trigger invoke stop why? I see. So when your job fails, `jsc.awaitTermination();` will throw an exception. Then you app main method will exit and trigger the shutdown hook and call `jsc.stop()`. On Thu, Jan 14, 2016 at 10:20 PM, Triones,Deng(vip.com<http://vip.com>) <triones.d...@vipshop.com<mailto:triones.d...@vipshop.com>> wrote: Thanks for your response . Our code as below : public void process(){ logger.info<http://logger.info>("streaming process start !!!"); SparkConf sparkConf = createSparkConf(this.getClass().getSimpleName()); JavaStreamingContext jsc = this.createJavaStreamingContext(sparkConf); if(this.streamingListener != null){ jsc.addStreamingListener(this.streamingListener); } JavaPairDStream<String, String> allKafkaWindowData = this.sparkReceiverDStream.createReceiverDStream(jsc,this.streamingConf.getWindowDuration(), this.streamingConf.getSlideDuration()); this.businessProcess(allKafkaWindowData); this.sleep(); jsc.start(); jsc.awaitTermination(); 发件人: Shixiong(Ryan) Zhu [mailto:shixi...@databricks.com<mailto:shixi...@databricks.com>] 发送时间: 2016年1月15日 6:02 收件人: 邓刚[技术中心] 抄送: Yogesh Mahajan; user 主题: Re: 答复: 答复: spark streaming context trigger invoke stop why? Could you show your codes? Did you use `StreamingContext.awaitTermination`? If so, it will return if any exception happens. On Wed, Jan 13, 2016 at 11:47 PM, Triones,Deng(vip.com<http://vip.com>) <triones.d...@vipshop.com<mailto:triones.d...@vipshop.com>> wrote: What’s more, I am runni
Re: 答复: 答复: 答复: spark streaming context trigger invoke stop why?
I see. So when your job fails, `jsc.awaitTermination();` will throw an exception. Then you app main method will exit and trigger the shutdown hook and call `jsc.stop()`. On Thu, Jan 14, 2016 at 10:20 PM, Triones,Deng(vip.com) < triones.d...@vipshop.com> wrote: > Thanks for your response . > > Our code as below : > > > > > > public void process(){ > > logger.info("streaming process start !!!"); > > > > SparkConf sparkConf = > createSparkConf(this.getClass().getSimpleName()); > > > > JavaStreamingContext jsc = > this.createJavaStreamingContext(sparkConf); > > > > if(this.streamingListener != null){ > > jsc.addStreamingListener(this.streamingListener); > > } > > JavaPairDStream<String, String> allKafkaWindowData = > this.sparkReceiverDStream.createReceiverDStream(jsc,this.streamingConf.getWindowDuration(), > > this.streamingConf.getSlideDuration()); > > > > this.businessProcess(allKafkaWindowData); > > this.sleep(); > >jsc.start(); > > jsc.awaitTermination(); > > > > > > *发件人:* Shixiong(Ryan) Zhu [mailto:shixi...@databricks.com] > *发送时间:* 2016年1月15日 6:02 > *收件人:* 邓刚[技术中心] > *抄送:* Yogesh Mahajan; user > *主题:* Re: 答复: 答复: spark streaming context trigger invoke stop why? > > > > Could you show your codes? Did you use > `StreamingContext.awaitTermination`? If so, it will return if any exception > happens. > > > > On Wed, Jan 13, 2016 at 11:47 PM, Triones,Deng(vip.com) < > triones.d...@vipshop.com> wrote: > > What’s more, I am running a 7*24 hours job , so I won’t call System.exit() > by myself. So I believe somewhere of the driver kill itself > > > > *发件人:* 邓刚[技术中心] > *发送时间:* 2016年1月14日 15:45 > *收件人:* 'Yogesh Mahajan' > *抄送:* user > *主题:* 答复: 答复: spark streaming context trigger invoke stop why? > > > > Thanks for your response, ApplicationMaster is only for yarn mode. I am > using standalone mode. Could you kindly please let me know where trigger > the shutdown hook? > > > > *发件人:* Yogesh Mahajan [mailto:ymaha...@snappydata.io > <ymaha...@snappydata.io>] > *发送时间:* 2016年1月14日 12:42 > *收件人:* 邓刚[技术中心] > *抄送:* user > *主题:* Re: 答复: spark streaming context trigger invoke stop why? > > > > All the action happens in ApplicationMaster expecially in run method > > Check ApplicationMaster#startUserApplication : userThread(Driver) which > invokes ApplicationMaster#finish method. You can also try System.exit in > your program > > > > Regards, > > Yogesh Mahajan, > > SnappyData Inc, snappydata.io > > > > On Thu, Jan 14, 2016 at 9:56 AM, Yogesh Mahajan <ymaha...@snappydata.io> > wrote: > > Hi Triones, > > > > Check the org.apache.spark.util.ShutdownHookManager : It adds this > ShutDownHook when you start a StreamingContext > > > > Here is the code in StreamingContext.start() > > > > shutdownHookRef = ShutdownHookManager.addShutdownHook( > > StreamingContext.SHUTDOWN_HOOK_PRIORITY)(stopOnShutdown) > > > > Also looke at the following def in StreamingContext which actually stops > the context from shutdown hook : > > private def stopOnShutdown(): Unit = { > > val stopGracefully = > conf.getBoolean("spark.streaming.stopGracefullyOnShutdown", false) > > logInfo(s"Invoking stop(stopGracefully=$stopGracefully) from shutdown > hook") > > // Do not stop SparkContext, let its own shutdown hook stop it > > stop(stopSparkContext = false, stopGracefully = stopGracefully) > > } > > > > Regards, > > Yogesh Mahajan, > > SnappyData Inc, snappydata.io > > > > On Thu, Jan 14, 2016 at 8:55 AM, Triones,Deng(vip.com) < > triones.d...@vipshop.com> wrote: > > More info > > > > I am using spark version 1.5.2 > > > > > > *发件人:* Triones,Deng(vip.com) [mailto:triones.d...@vipshop.com] > *发送时间:* 2016年1月14日 11:24 > *收件人:* user > *主题:* spark streaming context trigger invoke stop why? > > > > Hi all > > As I saw the driver log, the task failed 4 times in a stage, the > stage will be dropped when the input block was deleted before make use of. > After that the StreamingContext invoke stop. Does anyone know what kind of > akka message trigger the stop or which code trigger the shutdown hook? > > > > > > Thanks > > > > > > > > > > Driver log: > > > > Job aborted due to stage failure: Task 410 in stage 215.0 failed 4 times > > [org.
答复: 答复: 答复: spark streaming context trigger invoke stop why?
Thanks for your response . Our code as below : public void process(){ logger.info("streaming process start !!!"); SparkConf sparkConf = createSparkConf(this.getClass().getSimpleName()); JavaStreamingContext jsc = this.createJavaStreamingContext(sparkConf); if(this.streamingListener != null){ jsc.addStreamingListener(this.streamingListener); } JavaPairDStream<String, String> allKafkaWindowData = this.sparkReceiverDStream.createReceiverDStream(jsc,this.streamingConf.getWindowDuration(), this.streamingConf.getSlideDuration()); this.businessProcess(allKafkaWindowData); this.sleep(); jsc.start(); jsc.awaitTermination(); 发件人: Shixiong(Ryan) Zhu [mailto:shixi...@databricks.com] 发送时间: 2016年1月15日 6:02 收件人: 邓刚[技术中心] 抄送: Yogesh Mahajan; user 主题: Re: 答复: 答复: spark streaming context trigger invoke stop why? Could you show your codes? Did you use `StreamingContext.awaitTermination`? If so, it will return if any exception happens. On Wed, Jan 13, 2016 at 11:47 PM, Triones,Deng(vip.com<http://vip.com>) <triones.d...@vipshop.com<mailto:triones.d...@vipshop.com>> wrote: What’s more, I am running a 7*24 hours job , so I won’t call System.exit() by myself. So I believe somewhere of the driver kill itself 发件人: 邓刚[技术中心] 发送时间: 2016年1月14日 15:45 收件人: 'Yogesh Mahajan' 抄送: user 主题: 答复: 答复: spark streaming context trigger invoke stop why? Thanks for your response, ApplicationMaster is only for yarn mode. I am using standalone mode. Could you kindly please let me know where trigger the shutdown hook? 发件人: Yogesh Mahajan [mailto:ymaha...@snappydata.io] 发送时间: 2016年1月14日 12:42 收件人: 邓刚[技术中心] 抄送: user 主题: Re: 答复: spark streaming context trigger invoke stop why? All the action happens in ApplicationMaster expecially in run method Check ApplicationMaster#startUserApplication : userThread(Driver) which invokes ApplicationMaster#finish method. You can also try System.exit in your program Regards, Yogesh Mahajan, SnappyData Inc, snappydata.io<http://snappydata.io/> On Thu, Jan 14, 2016 at 9:56 AM, Yogesh Mahajan <ymaha...@snappydata.io<mailto:ymaha...@snappydata.io>> wrote: Hi Triones, Check the org.apache.spark.util.ShutdownHookManager : It adds this ShutDownHook when you start a StreamingContext Here is the code in StreamingContext.start() shutdownHookRef = ShutdownHookManager.addShutdownHook( StreamingContext.SHUTDOWN_HOOK_PRIORITY)(stopOnShutdown) Also looke at the following def in StreamingContext which actually stops the context from shutdown hook : private def stopOnShutdown(): Unit = { val stopGracefully = conf.getBoolean("spark.streaming.stopGracefullyOnShutdown", false) logInfo(s"Invoking stop(stopGracefully=$stopGracefully) from shutdown hook") // Do not stop SparkContext, let its own shutdown hook stop it stop(stopSparkContext = false, stopGracefully = stopGracefully) } Regards, Yogesh Mahajan, SnappyData Inc, snappydata.io<http://snappydata.io> On Thu, Jan 14, 2016 at 8:55 AM, Triones,Deng(vip.com<http://vip.com>) <triones.d...@vipshop.com<mailto:triones.d...@vipshop.com>> wrote: More info I am using spark version 1.5.2 发件人: Triones,Deng(vip.com<http://vip.com>) [mailto:triones.d...@vipshop.com<mailto:triones.d...@vipshop.com>] 发送时间: 2016年1月14日 11:24 收件人: user 主题: spark streaming context trigger invoke stop why? Hi all As I saw the driver log, the task failed 4 times in a stage, the stage will be dropped when the input block was deleted before make use of. After that the StreamingContext invoke stop. Does anyone know what kind of akka message trigger the stop or which code trigger the shutdown hook? Thanks Driver log: Job aborted due to stage failure: Task 410 in stage 215.0 failed 4 times [org.apache.spark.streaming.StreamingContext---Thread-0]: Invoking stop(stopGracefully=false) from shutdown hook 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! This communication is intended only for the addressee(s) and may contain information that is privileged and confidential. You are hereby notified that, if you are not an intended recipient listed above, or an authorized employee or agent of an addressee of this communication responsible for delivering e-mail messages to an intended recipient, any dissemination, distribution or reproduction of this communication (including any attachments hereto) is strictly prohibited. If you have received this communication in error, please notify us immediately by a reply e-mail addressed to the sender and permanently delete the original e-mail communication and any attachments from all storage devices without making or otherwise retaining a copy. 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合
Re: 答复: 答复: spark streaming context trigger invoke stop why?
Could you show your codes? Did you use `StreamingContext.awaitTermination`? If so, it will return if any exception happens. On Wed, Jan 13, 2016 at 11:47 PM, Triones,Deng(vip.com) < triones.d...@vipshop.com> wrote: > What’s more, I am running a 7*24 hours job , so I won’t call System.exit() > by myself. So I believe somewhere of the driver kill itself > > > > *发件人:* 邓刚[技术中心] > *发送时间:* 2016年1月14日 15:45 > *收件人:* 'Yogesh Mahajan' > *抄送:* user > *主题:* 答复: 答复: spark streaming context trigger invoke stop why? > > > > Thanks for your response, ApplicationMaster is only for yarn mode. I am > using standalone mode. Could you kindly please let me know where trigger > the shutdown hook? > > > > *发件人:* Yogesh Mahajan [mailto:ymaha...@snappydata.io > <ymaha...@snappydata.io>] > *发送时间:* 2016年1月14日 12:42 > *收件人:* 邓刚[技术中心] > *抄送:* user > *主题:* Re: 答复: spark streaming context trigger invoke stop why? > > > > All the action happens in ApplicationMaster expecially in run method > > Check ApplicationMaster#startUserApplication : userThread(Driver) which > invokes ApplicationMaster#finish method. You can also try System.exit in > your program > > > > Regards, > > Yogesh Mahajan, > > SnappyData Inc, snappydata.io > > > > On Thu, Jan 14, 2016 at 9:56 AM, Yogesh Mahajan <ymaha...@snappydata.io> > wrote: > > Hi Triones, > > > > Check the org.apache.spark.util.ShutdownHookManager : It adds this > ShutDownHook when you start a StreamingContext > > > > Here is the code in StreamingContext.start() > > > > shutdownHookRef = ShutdownHookManager.addShutdownHook( > > StreamingContext.SHUTDOWN_HOOK_PRIORITY)(stopOnShutdown) > > > > Also looke at the following def in StreamingContext which actually stops > the context from shutdown hook : > > private def stopOnShutdown(): Unit = { > > val stopGracefully = > conf.getBoolean("spark.streaming.stopGracefullyOnShutdown", false) > > logInfo(s"Invoking stop(stopGracefully=$stopGracefully) from shutdown > hook") > > // Do not stop SparkContext, let its own shutdown hook stop it > > stop(stopSparkContext = false, stopGracefully = stopGracefully) > > } > > > > Regards, > > Yogesh Mahajan, > > SnappyData Inc, snappydata.io > > > > On Thu, Jan 14, 2016 at 8:55 AM, Triones,Deng(vip.com) < > triones.d...@vipshop.com> wrote: > > More info > > > > I am using spark version 1.5.2 > > > > > > *发件人:* Triones,Deng(vip.com) [mailto:triones.d...@vipshop.com] > *发送时间:* 2016年1月14日 11:24 > *收件人:* user > *主题:* spark streaming context trigger invoke stop why? > > > > Hi all > > As I saw the driver log, the task failed 4 times in a stage, the > stage will be dropped when the input block was deleted before make use of. > After that the StreamingContext invoke stop. Does anyone know what kind of > akka message trigger the stop or which code trigger the shutdown hook? > > > > > > Thanks > > > > > > > > > > Driver log: > > > > Job aborted due to stage failure: Task 410 in stage 215.0 failed 4 times > > [org.apache.spark.streaming.StreamingContext---Thread-0]: Invoking > stop(stopGracefully=false) from shutdown hook > > > 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! > This communication is intended only for the addressee(s) and may contain > information that is privileged and confidential. You are hereby notified > that, if you are not an intended recipient listed above, or an authorized > employee or agent of an addressee of this communication responsible for > delivering e-mail messages to an intended recipient, any dissemination, > distribution or reproduction of this communication (including any > attachments hereto) is strictly prohibited. If you have received this > communication in error, please notify us immediately by a reply e-mail > addressed to the sender and permanently delete the original e-mail > communication and any attachments from all storage devices without making > or otherwise retaining a copy. > > > 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! > This communication is intended only for the addressee(s) and may contain > information that is privileged and confidential. You are hereby notified > that, if you are not an intended recipient listed above, or an authorized > employee or agent of an addressee of this communication responsible for > delivering e-mail messages to an intended recipient, any dissemination, > distribution
答复: 答复: spark streaming context trigger invoke stop why?
What’s more, I am running a 7*24 hours job , so I won’t call System.exit() by myself. So I believe somewhere of the driver kill itself 发件人: 邓刚[技术中心] 发送时间: 2016年1月14日 15:45 收件人: 'Yogesh Mahajan' 抄送: user 主题: 答复: 答复: spark streaming context trigger invoke stop why? Thanks for your response, ApplicationMaster is only for yarn mode. I am using standalone mode. Could you kindly please let me know where trigger the shutdown hook? 发件人: Yogesh Mahajan [mailto:ymaha...@snappydata.io] 发送时间: 2016年1月14日 12:42 收件人: 邓刚[技术中心] 抄送: user 主题: Re: 答复: spark streaming context trigger invoke stop why? All the action happens in ApplicationMaster expecially in run method Check ApplicationMaster#startUserApplication : userThread(Driver) which invokes ApplicationMaster#finish method. You can also try System.exit in your program Regards, Yogesh Mahajan, SnappyData Inc, snappydata.io<http://snappydata.io/> On Thu, Jan 14, 2016 at 9:56 AM, Yogesh Mahajan <ymaha...@snappydata.io<mailto:ymaha...@snappydata.io>> wrote: Hi Triones, Check the org.apache.spark.util.ShutdownHookManager : It adds this ShutDownHook when you start a StreamingContext Here is the code in StreamingContext.start() shutdownHookRef = ShutdownHookManager.addShutdownHook( StreamingContext.SHUTDOWN_HOOK_PRIORITY)(stopOnShutdown) Also looke at the following def in StreamingContext which actually stops the context from shutdown hook : private def stopOnShutdown(): Unit = { val stopGracefully = conf.getBoolean("spark.streaming.stopGracefullyOnShutdown", false) logInfo(s"Invoking stop(stopGracefully=$stopGracefully) from shutdown hook") // Do not stop SparkContext, let its own shutdown hook stop it stop(stopSparkContext = false, stopGracefully = stopGracefully) } Regards, Yogesh Mahajan, SnappyData Inc, snappydata.io<http://snappydata.io> On Thu, Jan 14, 2016 at 8:55 AM, Triones,Deng(vip.com<http://vip.com>) <triones.d...@vipshop.com<mailto:triones.d...@vipshop.com>> wrote: More info I am using spark version 1.5.2 发件人: Triones,Deng(vip.com<http://vip.com>) [mailto:triones.d...@vipshop.com<mailto:triones.d...@vipshop.com>] 发送时间: 2016年1月14日 11:24 收件人: user 主题: spark streaming context trigger invoke stop why? Hi all As I saw the driver log, the task failed 4 times in a stage, the stage will be dropped when the input block was deleted before make use of. After that the StreamingContext invoke stop. Does anyone know what kind of akka message trigger the stop or which code trigger the shutdown hook? Thanks Driver log: Job aborted due to stage failure: Task 410 in stage 215.0 failed 4 times [org.apache.spark.streaming.StreamingContext---Thread-0]: Invoking stop(stopGracefully=false) from shutdown hook 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! This communication is intended only for the addressee(s) and may contain information that is privileged and confidential. You are hereby notified that, if you are not an intended recipient listed above, or an authorized employee or agent of an addressee of this communication responsible for delivering e-mail messages to an intended recipient, any dissemination, distribution or reproduction of this communication (including any attachments hereto) is strictly prohibited. If you have received this communication in error, please notify us immediately by a reply e-mail addressed to the sender and permanently delete the original e-mail communication and any attachments from all storage devices without making or otherwise retaining a copy. 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! This communication is intended only for the addressee(s) and may contain information that is privileged and confidential. You are hereby notified that, if you are not an intended recipient listed above, or an authorized employee or agent of an addressee of this communication responsible for delivering e-mail messages to an intended recipient, any dissemination, distribution or reproduction of this communication (including any attachments hereto) is strictly prohibited. If you have received this communication in error, please notify us immediately by a reply e-mail addressed to the sender and permanently delete the original e-mail communication and any attachments from all storage devices without making or otherwise retaining a copy. 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! This communication is intended only for the addressee(s) and may contain information that is privileged and confidential. You are hereby notified that, if you are not an intended recipient listed above, or an authorized employee or agent of an addressee of this communication responsible for delivering e-mail messages to an intended recipient, any dissemination, dist
答复: 答复: spark streaming context trigger invoke stop why?
Thanks for your response, ApplicationMaster is only for yarn mode. I am using standalone mode. Could you kindly please let me know where trigger the shutdown hook? 发件人: Yogesh Mahajan [mailto:ymaha...@snappydata.io] 发送时间: 2016年1月14日 12:42 收件人: 邓刚[技术中心] 抄送: user 主题: Re: 答复: spark streaming context trigger invoke stop why? All the action happens in ApplicationMaster expecially in run method Check ApplicationMaster#startUserApplication : userThread(Driver) which invokes ApplicationMaster#finish method. You can also try System.exit in your program Regards, Yogesh Mahajan, SnappyData Inc, snappydata.io<http://snappydata.io/> On Thu, Jan 14, 2016 at 9:56 AM, Yogesh Mahajan <ymaha...@snappydata.io<mailto:ymaha...@snappydata.io>> wrote: Hi Triones, Check the org.apache.spark.util.ShutdownHookManager : It adds this ShutDownHook when you start a StreamingContext Here is the code in StreamingContext.start() shutdownHookRef = ShutdownHookManager.addShutdownHook( StreamingContext.SHUTDOWN_HOOK_PRIORITY)(stopOnShutdown) Also looke at the following def in StreamingContext which actually stops the context from shutdown hook : private def stopOnShutdown(): Unit = { val stopGracefully = conf.getBoolean("spark.streaming.stopGracefullyOnShutdown", false) logInfo(s"Invoking stop(stopGracefully=$stopGracefully) from shutdown hook") // Do not stop SparkContext, let its own shutdown hook stop it stop(stopSparkContext = false, stopGracefully = stopGracefully) } Regards, Yogesh Mahajan, SnappyData Inc, snappydata.io<http://snappydata.io> On Thu, Jan 14, 2016 at 8:55 AM, Triones,Deng(vip.com<http://vip.com>) <triones.d...@vipshop.com<mailto:triones.d...@vipshop.com>> wrote: More info I am using spark version 1.5.2 发件人: Triones,Deng(vip.com<http://vip.com>) [mailto:triones.d...@vipshop.com<mailto:triones.d...@vipshop.com>] 发送时间: 2016年1月14日 11:24 收件人: user 主题: spark streaming context trigger invoke stop why? Hi all As I saw the driver log, the task failed 4 times in a stage, the stage will be dropped when the input block was deleted before make use of. After that the StreamingContext invoke stop. Does anyone know what kind of akka message trigger the stop or which code trigger the shutdown hook? Thanks Driver log: Job aborted due to stage failure: Task 410 in stage 215.0 failed 4 times [org.apache.spark.streaming.StreamingContext---Thread-0]: Invoking stop(stopGracefully=false) from shutdown hook 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! This communication is intended only for the addressee(s) and may contain information that is privileged and confidential. You are hereby notified that, if you are not an intended recipient listed above, or an authorized employee or agent of an addressee of this communication responsible for delivering e-mail messages to an intended recipient, any dissemination, distribution or reproduction of this communication (including any attachments hereto) is strictly prohibited. If you have received this communication in error, please notify us immediately by a reply e-mail addressed to the sender and permanently delete the original e-mail communication and any attachments from all storage devices without making or otherwise retaining a copy. 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! This communication is intended only for the addressee(s) and may contain information that is privileged and confidential. You are hereby notified that, if you are not an intended recipient listed above, or an authorized employee or agent of an addressee of this communication responsible for delivering e-mail messages to an intended recipient, any dissemination, distribution or reproduction of this communication (including any attachments hereto) is strictly prohibited. If you have received this communication in error, please notify us immediately by a reply e-mail addressed to the sender and permanently delete the original e-mail communication and any attachments from all storage devices without making or otherwise retaining a copy. 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! This communication is intended only for the addressee(s) and may contain information that is privileged and confidential. You are hereby notified that, if you are not an intended recipient listed above, or an authorized employee or agent of an addressee of this communication responsible for delivering e-mail messages to an intended recipient, any dissemination, distribution or reproduction of this communication (including any attachments hereto) is strictly prohibited. If you have received this communication in error, please notify us immediately by a reply e-mail addressed to the sender and permanently delete the original
Re: 答复: spark streaming context trigger invoke stop why?
Hi Triones, Check the org.apache.spark.util.ShutdownHookManager : It adds this ShutDownHook when you start a StreamingContext Here is the code in StreamingContext.start() shutdownHookRef = ShutdownHookManager.addShutdownHook( StreamingContext.SHUTDOWN_HOOK_PRIORITY)(stopOnShutdown) Also looke at the following def in StreamingContext which actually stops the context from shutdown hook : private def stopOnShutdown(): Unit = { val stopGracefully = conf.getBoolean("spark.streaming.stopGracefullyOnShutdown", false) logInfo(s"Invoking stop(stopGracefully=$stopGracefully) from shutdown hook") // Do not stop SparkContext, let its own shutdown hook stop it stop(stopSparkContext = false, stopGracefully = stopGracefully) } Regards, Yogesh Mahajan, SnappyData Inc, snappydata.io On Thu, Jan 14, 2016 at 8:55 AM, Triones,Deng(vip.com) < triones.d...@vipshop.com> wrote: > More info > > > > I am using spark version 1.5.2 > > > > > > *发件人:* Triones,Deng(vip.com) [mailto:triones.d...@vipshop.com] > *发送时间:* 2016年1月14日 11:24 > *收件人:* user > *主题:* spark streaming context trigger invoke stop why? > > > > Hi all > > As I saw the driver log, the task failed 4 times in a stage, the > stage will be dropped when the input block was deleted before make use of. > After that the StreamingContext invoke stop. Does anyone know what kind of > akka message trigger the stop or which code trigger the shutdown hook? > > > > > > Thanks > > > > > > > > > > Driver log: > > > > Job aborted due to stage failure: Task 410 in stage 215.0 failed 4 times > > [org.apache.spark.streaming.StreamingContext---Thread-0]: Invoking > stop(stopGracefully=false) from shutdown hook > > > 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! > This communication is intended only for the addressee(s) and may contain > information that is privileged and confidential. You are hereby notified > that, if you are not an intended recipient listed above, or an authorized > employee or agent of an addressee of this communication responsible for > delivering e-mail messages to an intended recipient, any dissemination, > distribution or reproduction of this communication (including any > attachments hereto) is strictly prohibited. If you have received this > communication in error, please notify us immediately by a reply e-mail > addressed to the sender and permanently delete the original e-mail > communication and any attachments from all storage devices without making > or otherwise retaining a copy. > 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! > This communication is intended only for the addressee(s) and may contain > information that is privileged and confidential. You are hereby notified > that, if you are not an intended recipient listed above, or an authorized > employee or agent of an addressee of this communication responsible for > delivering e-mail messages to an intended recipient, any dissemination, > distribution or reproduction of this communication (including any > attachments hereto) is strictly prohibited. If you have received this > communication in error, please notify us immediately by a reply e-mail > addressed to the sender and permanently delete the original e-mail > communication and any attachments from all storage devices without making > or otherwise retaining a copy. >
答复: spark streaming context trigger invoke stop why?
More info I am using spark version 1.5.2 发件人: Triones,Deng(vip.com) [mailto:triones.d...@vipshop.com] 发送时间: 2016年1月14日 11:24 收件人: user 主题: spark streaming context trigger invoke stop why? Hi all As I saw the driver log, the task failed 4 times in a stage, the stage will be dropped when the input block was deleted before make use of. After that the StreamingContext invoke stop. Does anyone know what kind of akka message trigger the stop or which code trigger the shutdown hook? Thanks Driver log: Job aborted due to stage failure: Task 410 in stage 215.0 failed 4 times [org.apache.spark.streaming.StreamingContext---Thread-0]: Invoking stop(stopGracefully=false) from shutdown hook 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! This communication is intended only for the addressee(s) and may contain information that is privileged and confidential. You are hereby notified that, if you are not an intended recipient listed above, or an authorized employee or agent of an addressee of this communication responsible for delivering e-mail messages to an intended recipient, any dissemination, distribution or reproduction of this communication (including any attachments hereto) is strictly prohibited. If you have received this communication in error, please notify us immediately by a reply e-mail addressed to the sender and permanently delete the original e-mail communication and any attachments from all storage devices without making or otherwise retaining a copy. 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! This communication is intended only for the addressee(s) and may contain information that is privileged and confidential. You are hereby notified that, if you are not an intended recipient listed above, or an authorized employee or agent of an addressee of this communication responsible for delivering e-mail messages to an intended recipient, any dissemination, distribution or reproduction of this communication (including any attachments hereto) is strictly prohibited. If you have received this communication in error, please notify us immediately by a reply e-mail addressed to the sender and permanently delete the original e-mail communication and any attachments from all storage devices without making or otherwise retaining a copy.
Re: 答复: spark streaming context trigger invoke stop why?
All the action happens in ApplicationMaster expecially in run method Check ApplicationMaster#startUserApplication : userThread(Driver) which invokes ApplicationMaster#finish method. You can also try System.exit in your program Regards, Yogesh Mahajan, SnappyData Inc, snappydata.io On Thu, Jan 14, 2016 at 9:56 AM, Yogesh Mahajanwrote: > Hi Triones, > > Check the org.apache.spark.util.ShutdownHookManager : It adds this > ShutDownHook when you start a StreamingContext > > Here is the code in StreamingContext.start() > > shutdownHookRef = ShutdownHookManager.addShutdownHook( > StreamingContext.SHUTDOWN_HOOK_PRIORITY)(stopOnShutdown) > > Also looke at the following def in StreamingContext which actually stops > the context from shutdown hook : > private def stopOnShutdown(): Unit = { > val stopGracefully = > conf.getBoolean("spark.streaming.stopGracefullyOnShutdown", false) > logInfo(s"Invoking stop(stopGracefully=$stopGracefully) from shutdown > hook") > // Do not stop SparkContext, let its own shutdown hook stop it > stop(stopSparkContext = false, stopGracefully = stopGracefully) > } > > Regards, > Yogesh Mahajan, > SnappyData Inc, snappydata.io > > On Thu, Jan 14, 2016 at 8:55 AM, Triones,Deng(vip.com) < > triones.d...@vipshop.com> wrote: > >> More info >> >> >> >> I am using spark version 1.5.2 >> >> >> >> >> >> *发件人:* Triones,Deng(vip.com) [mailto:triones.d...@vipshop.com] >> *发送时间:* 2016年1月14日 11:24 >> *收件人:* user >> *主题:* spark streaming context trigger invoke stop why? >> >> >> >> Hi all >> >> As I saw the driver log, the task failed 4 times in a stage, the >> stage will be dropped when the input block was deleted before make use of. >> After that the StreamingContext invoke stop. Does anyone know what kind of >> akka message trigger the stop or which code trigger the shutdown hook? >> >> >> >> >> >> Thanks >> >> >> >> >> >> >> >> >> >> Driver log: >> >> >> >> Job aborted due to stage failure: Task 410 in stage 215.0 failed 4 times >> >> [org.apache.spark.streaming.StreamingContext---Thread-0]: Invoking >> stop(stopGracefully=false) from shutdown hook >> >> >> 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! >> This communication is intended only for the addressee(s) and may contain >> information that is privileged and confidential. You are hereby notified >> that, if you are not an intended recipient listed above, or an authorized >> employee or agent of an addressee of this communication responsible for >> delivering e-mail messages to an intended recipient, any dissemination, >> distribution or reproduction of this communication (including any >> attachments hereto) is strictly prohibited. If you have received this >> communication in error, please notify us immediately by a reply e-mail >> addressed to the sender and permanently delete the original e-mail >> communication and any attachments from all storage devices without making >> or otherwise retaining a copy. >> 本电子邮件可能为保密文件。如果阁下非电子邮件所指定之收件人,谨请立即通知本人。敬请阁下不要使用、保存、复印、打印、散布本电子邮件及其内容,或将其用于其他任何目的或向任何人披露。谢谢您的合作! >> This communication is intended only for the addressee(s) and may contain >> information that is privileged and confidential. You are hereby notified >> that, if you are not an intended recipient listed above, or an authorized >> employee or agent of an addressee of this communication responsible for >> delivering e-mail messages to an intended recipient, any dissemination, >> distribution or reproduction of this communication (including any >> attachments hereto) is strictly prohibited. If you have received this >> communication in error, please notify us immediately by a reply e-mail >> addressed to the sender and permanently delete the original e-mail >> communication and any attachments from all storage devices without making >> or otherwise retaining a copy. >> > >