Github user andrewor14 commented on a diff in the pull request:
https://github.com/apache/spark/pull/3121#discussion_r20393704
--- Diff: core/src/main/scala/org/apache/spark/SparkContext.scala ---
@@ -1417,6 +1427,97 @@ class SparkContext(config: SparkConf) extends
SparkStatusAPI with Logging {
*/
object SparkContext extends Logging {
+ /**
+ * Lock that guards access to global variables that track SparkContext
construction.
+ */
+ private[spark] val SPARK_CONTEXT_CONSTRUCTOR_LOCK = new Object()
+
+ /**
+ * Records the creation site of the active, fully-constructed
SparkContext. If no SparkContext
+ * is active, then this is `None`.
+ *
+ * Access to this field is guarded by SPARK_CONTEXT_CONSTRUCTOR_LOCK
+ */
+ private[spark] var activeContextCreationSite: Option[CallSite] = None
+
+ /**
+ * Points to a partially-constructed SparkContext if some thread is in
the SparkContext
+ * constructor, or `None` if no SparkContext is being constructed.
+ *
+ * Access to this field is guarded by SPARK_CONTEXT_CONSTRUCTOR_LOCK
+ */
+ private[spark] var contextBeingConstructed: Option[SparkContext] = None
+
+ /**
+ * Called to ensure that no other SparkContext is running in this JVM.
+ *
+ * Throws an exception if a running context is detected and logs a
warning if another thread is
+ * constructing a SparkContext. This warning is necessary because the
current locking scheme
+ * prevents us from reliably distinguishing between cases where another
context is being
+ * constructed and cases where another constructor threw an exception.
+ */
+ private def assertNoOtherContextIsRunning(sc: SparkContext, conf:
SparkConf) {
+ SPARK_CONTEXT_CONSTRUCTOR_LOCK.synchronized {
+ contextBeingConstructed.foreach { otherContext =>
+ if (otherContext ne sc) {
+ val warnMsg = "Another SparkContext is being constructed (or
threw an exception in its" +
+ " constructor). This may indicate an error, since only one
SparkContext may be" +
+ " running in this JVM (see SPARK-2243)."
+ logWarning(warnMsg)
+ }
+
+ activeContextCreationSite.foreach { creationSite =>
+ val errMsg = "Only one SparkContext may be running in this JVM
(see SPARK-2243)." +
+ " To ignore this error, set spark.driver.allowMultipleContexts
= true. " +
+ s"The currently running SparkContext was created
at:\n${creationSite.longForm}"
+ val exception = new SparkException(errMsg)
+ if (conf.getBoolean("spark.driver.allowMultipleContexts",
false)) {
+ logWarning("Multiple running SparkContexts detected in the
same JVM!", exception)
+ } else {
+ throw exception
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Called at the beginning of the SparkContext constructor to ensure
that no SparkContext is
+ * running. Throws an exception if a running context is detected and
logs a warning if another
+ * thread is constructing a SparkContext. This warning is necessary
because the current locking
+ * scheme prevents us from reliably distinguishing between cases where
another context is being
+ * constructed and cases where another constructor threw an exception.
+ */
+ private[spark] def markPartiallyConstructed(sc: SparkContext, conf:
SparkConf) {
--- End diff --
does this need to take in a conf in addition to a context? Can't it get the
conf from the context?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
