Repository: spark
Updated Branches:
refs/heads/master bd6db1505 -> fec67ed7e
[SPARK-25076][SQL] SQLConf should not be retrieved from a stopped SparkSession
## What changes were proposed in this pull request?
When a `SparkSession` is stopped, `SQLConf.get` should use the fallback conf to
avoid weird issues like
```
sbt.ForkMain$ForkError: java.lang.IllegalStateException: LiveListenerBus is
stopped.
at
org.apache.spark.scheduler.LiveListenerBus.addToQueue(LiveListenerBus.scala:97)
at
org.apache.spark.scheduler.LiveListenerBus.addToStatusQueue(LiveListenerBus.scala:80)
at
org.apache.spark.sql.internal.SharedState.<init>(SharedState.scala:93)
at
org.apache.spark.sql.SparkSession$$anonfun$sharedState$1.apply(SparkSession.scala:120)
at
org.apache.spark.sql.SparkSession$$anonfun$sharedState$1.apply(SparkSession.scala:120)
at scala.Option.getOrElse(Option.scala:121)
...
```
## How was this patch tested?
a new test suite
Closes #22056 from cloud-fan/session.
Authored-by: Wenchen Fan <[email protected]>
Signed-off-by: Xiao Li <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fec67ed7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fec67ed7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fec67ed7
Branch: refs/heads/master
Commit: fec67ed7e95483c5ea97a7b263ad4bea7d3d42b5
Parents: bd6db15
Author: Wenchen Fan <[email protected]>
Authored: Thu Aug 9 14:38:58 2018 -0700
Committer: Xiao Li <[email protected]>
Committed: Thu Aug 9 14:38:58 2018 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/SparkSession.scala | 3 +-
.../apache/spark/sql/LocalSparkSession.scala | 9 ++----
.../spark/sql/internal/SQLConfGetterSuite.scala | 33 ++++++++++++++++++++
3 files changed, 37 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fec67ed7/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 565042f..d9278d8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -92,7 +92,8 @@ class SparkSession private(
// If there is no active SparkSession, uses the default SQL conf. Otherwise,
use the session's.
SQLConf.setSQLConfGetter(() => {
-
SparkSession.getActiveSession.map(_.sessionState.conf).getOrElse(SQLConf.getFallbackConf)
+
SparkSession.getActiveSession.filterNot(_.sparkContext.isStopped).map(_.sessionState.conf)
+ .getOrElse(SQLConf.getFallbackConf)
})
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/fec67ed7/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
----------------------------------------------------------------------
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
b/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
index cbef1c7..6b90f20 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
@@ -36,19 +36,14 @@ trait LocalSparkSession extends BeforeAndAfterEach with
BeforeAndAfterAll { self
override def afterEach() {
try {
- resetSparkContext()
+ LocalSparkSession.stop(spark)
SparkSession.clearActiveSession()
SparkSession.clearDefaultSession()
+ spark = null
} finally {
super.afterEach()
}
}
-
- def resetSparkContext(): Unit = {
- LocalSparkSession.stop(spark)
- spark = null
- }
-
}
object LocalSparkSession {
http://git-wip-us.apache.org/repos/asf/spark/blob/fec67ed7/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala
new file mode 100644
index 0000000..bb79d3a
--- /dev/null
+++
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.internal
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.{LocalSparkSession, SparkSession}
+
+class SQLConfGetterSuite extends SparkFunSuite with LocalSparkSession {
+
+ test("SPARK-25076: SQLConf should not be retrieved from a stopped
SparkSession") {
+ spark = SparkSession.builder().master("local").getOrCreate()
+ assert(SQLConf.get eq spark.sessionState.conf,
+ "SQLConf.get should get the conf from the active spark session.")
+ spark.stop()
+ assert(SQLConf.get eq SQLConf.getFallbackConf,
+ "SQLConf.get should not get conf from a stopped spark session.")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
