Github user olegz commented on a diff in the pull request:
https://github.com/apache/spark/pull/2849#discussion_r19648060
--- Diff: core/src/main/scala/org/apache/spark/SparkContext.scala ---
@@ -1639,12 +1615,38 @@ object SparkContext extends Logging {
scheduler.initialize(backend)
(backend, scheduler)
+ case EXECUTION_CONTEXT(sparkUrl) =>
+ logInfo("Will use custom job execution context " + sparkUrl)
+ sc.executionContext = Class.forName(sparkUrl).newInstance().
+ asInstanceOf[JobExecutionContext]
+ val scheduler = new NoOpTaskScheduler(sc)
+ val backend = new LocalBackend(scheduler, 1)
+ (backend, scheduler)
+
case _ =>
throw new SparkException("Could not parse Master URL: '" + master
+ "'")
}
}
}
-
+/**
+ * No-op implementation of TaskScheduler which is used in cases where
+ * execution of Spark DAG is delegate to an external execution environment,
+ * thus not relying on DAGScheduler nor TaskScheduler
+ */
+private class NoOpTaskScheduler(sc: SparkContext) extends
TaskSchedulerImpl(sc, 1) {
+ override val schedulingMode: SchedulingMode.SchedulingMode =
SchedulingMode.NONE
+ override def start(): Unit = {}
+ override def stop(): Unit = {}
+ override def submitTasks(taskSet: TaskSet): Unit = {}
+ override def cancelTasks(stageId: Int, interruptThread: Boolean) = {}
+ override def setDAGScheduler(dagScheduler: DAGScheduler): Unit = {}
+ override def defaultParallelism(): Int = 1
+ override def executorHeartbeatReceived(execId: String,
+ taskMetrics: Array[(Long, TaskMetrics)],
+ blockManagerId: BlockManagerId): Boolean = true
+ override def applicationId(): String = sc.appName
--- End diff --
Thanks, I'll address it.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
