Log exception in scheduler in addition to passing it to the caller. Code Styling changes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/be3ea239 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/be3ea239 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/be3ea239 Branch: refs/heads/master Commit: be3ea2394fa2e626fb6b5f2cd46e7156016c9b3f Parents: 4d53830 Author: Sundeep Narravula <[email protected]> Authored: Sun Dec 1 00:50:34 2013 -0800 Committer: Sundeep Narravula <[email protected]> Committed: Sun Dec 1 00:50:34 2013 -0800 ---------------------------------------------------------------------- .../main/scala/org/apache/spark/scheduler/DAGScheduler.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/be3ea239/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index f6a4482..9159186 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -384,12 +384,14 @@ class DAGScheduler( private[scheduler] def processEvent(event: DAGSchedulerEvent): Boolean = { event match { case JobSubmitted(jobId, rdd, func, partitions, allowLocal, callSite, listener, properties) => - var finalStage:Stage = null + var finalStage: Stage = null try { + // New stage creation at times and if its not protected, the scheduler thread is killed. + // e.g. it can fail when jobs are run on HadoopRDD whose underlying hdfs files have been deleted finalStage = newStage(rdd, partitions.size, None, jobId, Some(callSite)) } catch { case e: Exception => - logWarning("Creating new stage failed due to exception - job: " + jobId ) + logWarning("Creating new stage failed due to exception - job: " + jobId, e) listener.jobFailed(e) return false }
