Github user HyukjinKwon commented on the issue:
https://github.com/apache/spark/pull/20404
How about something like this?
```diff
--- a/python/pyspark/sql/session.py
+++ b/python/pyspark/sql/session.py
@@ -213,10 +213,10 @@ class SparkSession(object):
self._jsc = self._sc._jsc
self._jvm = self._sc._jvm
if jsparkSession is None:
- if self._jvm.SparkSession.getDefaultSession().isDefined() \
- and not
self._jvm.SparkSession.getDefaultSession().get() \
- .sparkContext().isStopped():
- jsparkSession =
self._jvm.SparkSession.getDefaultSession().get()
+ default_session = self._jvm.SparkSession.getDefaultSession()
+ if default_session.isDefined() \
+ and not
default_session.get().sparkContext().isStopped():
+ jsparkSession = default_session
else:
jsparkSession = self._jvm.SparkSession(self._jsc.sc())
self._jsparkSession = jsparkSession
@@ -230,7 +230,7 @@ class SparkSession(object):
if SparkSession._instantiatedSession is None \
or SparkSession._instantiatedSession._sc._jsc is None:
SparkSession._instantiatedSession = self
- self._jvm.SparkSession.setDefaultSession(self._jsparkSession)
+
self._jvm.SparkSession.setDefaultSessionIfUnset(self._jsparkSession)
def _repr_html_(self):
return """
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 734573ba31f..99747ef88bc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -1031,6 +1031,18 @@ object SparkSession {
// Private methods from now on
////////////////////////////////////////////////////////////////////////////////////////
+ /**
+ * This if for Python blabla
+ */
+ private[sql] def setDefaultSessionIfUnset(session: SparkSession): Unit =
{
+ // Global synchronization blabla
+ SparkSession.synchronized {
+ if (defaultSession.get() eq null) {
+ defaultSession.set(session)
+ }
+ }
+ }
+
/** The active SparkSession for the current thread. */
private val activeThreadSession = new
InheritableThreadLocal[SparkSession]
```
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
