Repository: spark Updated Branches: refs/heads/master ead4ba0eb -> 0733a54a4
[SPARK-19669][SQL] Open up visibility for sharedState, sessionState, and a few other functions ## What changes were proposed in this pull request? To ease debugging, most of Spark SQL internals have public level visibility. Two of the most important internal states, sharedState and sessionState, however, are package private. It would make more sense to open these up as well with clear documentation that they are internal. In addition, users currently have way to set active/default SparkSession, but no way to actually get them back. We should open those up as well. ## How was this patch tested? N/A - only visibility change. Author: Reynold Xin <r...@databricks.com> Closes #17002 from rxin/SPARK-19669. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0733a54a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0733a54a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0733a54a Branch: refs/heads/master Commit: 0733a54a4517b82291efed9ac7f7407d9044593c Parents: ead4ba0 Author: Reynold Xin <r...@databricks.com> Authored: Mon Feb 20 12:21:07 2017 -0800 Committer: gatorsmile <gatorsm...@gmail.com> Committed: Mon Feb 20 12:21:07 2017 -0800 ---------------------------------------------------------------------- .../org/apache/spark/internal/Logging.scala | 2 +- .../org/apache/spark/sql/SparkSession.scala | 29 ++++++++++++++++---- .../apache/spark/sql/internal/SharedState.scala | 4 +-- 3 files changed, 27 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/0733a54a/core/src/main/scala/org/apache/spark/internal/Logging.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/internal/Logging.scala b/core/src/main/scala/org/apache/spark/internal/Logging.scala index 013cd1c..c7f2847 100644 --- a/core/src/main/scala/org/apache/spark/internal/Logging.scala +++ b/core/src/main/scala/org/apache/spark/internal/Logging.scala @@ -28,7 +28,7 @@ import org.apache.spark.util.Utils * logging messages at different levels using methods that only evaluate parameters lazily if the * log level is enabled. */ -private[spark] trait Logging { +trait Logging { // Make the log field transient so that objects with Logging can // be serialized and used on another machine http://git-wip-us.apache.org/repos/asf/spark/blob/0733a54a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index 1975a56..72af55c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -95,18 +95,28 @@ class SparkSession private( /** * State shared across sessions, including the `SparkContext`, cached data, listener, * and a catalog that interacts with external systems. + * + * This is internal to Spark and there is no guarantee on interface stability. + * + * @since 2.2.0 */ + @InterfaceStability.Unstable @transient - private[sql] lazy val sharedState: SharedState = { + lazy val sharedState: SharedState = { existingSharedState.getOrElse(new SharedState(sparkContext)) } /** * State isolated across sessions, including SQL configurations, temporary tables, registered * functions, and everything else that accepts a [[org.apache.spark.sql.internal.SQLConf]]. + * + * This is internal to Spark and there is no guarantee on interface stability. + * + * @since 2.2.0 */ + @InterfaceStability.Unstable @transient - private[sql] lazy val sessionState: SessionState = { + lazy val sessionState: SessionState = { SparkSession.reflect[SessionState, SparkSession]( SparkSession.sessionStateClassName(sparkContext.conf), self) @@ -613,7 +623,6 @@ class SparkSession private( * * @since 2.1.0 */ - @InterfaceStability.Stable def time[T](f: => T): T = { val start = System.nanoTime() val ret = f @@ -928,9 +937,19 @@ object SparkSession { defaultSession.set(null) } - private[sql] def getActiveSession: Option[SparkSession] = Option(activeThreadSession.get) + /** + * Returns the active SparkSession for the current thread, returned by the builder. + * + * @since 2.2.0 + */ + def getActiveSession: Option[SparkSession] = Option(activeThreadSession.get) - private[sql] def getDefaultSession: Option[SparkSession] = Option(defaultSession.get) + /** + * Returns the default SparkSession that is returned by the builder. + * + * @since 2.2.0 + */ + def getDefaultSession: Option[SparkSession] = Option(defaultSession.get) /** A global SQL listener used for the SQL UI. */ private[sql] val sqlListener = new AtomicReference[SQLListener]() http://git-wip-us.apache.org/repos/asf/spark/blob/0733a54a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala index 8de95fe..7ce9938 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala @@ -39,7 +39,7 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { // Load hive-site.xml into hadoopConf and determine the warehouse path we want to use, based on // the config from both hive and Spark SQL. Finally set the warehouse config value to sparkConf. - val warehousePath = { + val warehousePath: String = { val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml") if (configFile != null) { sparkContext.hadoopConfiguration.addResource(configFile) @@ -103,7 +103,7 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { /** * A manager for global temporary views. */ - val globalTempViewManager = { + val globalTempViewManager: GlobalTempViewManager = { // System preserved database should not exists in metastore. However it's hard to guarantee it // for every session, because case-sensitivity differs. Here we always lowercase it to make our // life easier. --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org