Github user mgaido91 commented on a diff in the pull request:
https://github.com/apache/spark/pull/22152#discussion_r211584402
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala ---
@@ -116,16 +129,24 @@ object SQLConf {
if (TaskContext.get != null) {
new ReadOnlySQLConf(TaskContext.get())
} else {
- if (Utils.isTesting && SparkContext.getActive.isDefined) {
+ val isSchedulerEventLoopThread = SparkContext.getActive
+ .map(_.dagScheduler.eventProcessLoop.eventThread)
+ .exists(_.getId == Thread.currentThread().getId)
+ if (isSchedulerEventLoopThread) {
// DAGScheduler event loop thread does not have an active
SparkSession, the `confGetter`
- // will return `fallbackConf` which is unexpected. Here we prevent
it from happening.
- val schedulerEventLoopThread =
-
SparkContext.getActive.get.dagScheduler.eventProcessLoop.eventThread
- if (schedulerEventLoopThread.getId ==
Thread.currentThread().getId) {
+ // will return `fallbackConf` which is unexpected. Here we
requires the caller to get the
--- End diff --
nit: we require
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
