Repository: spark Updated Branches: refs/heads/branch-2.4 f12769e73 -> 01c000b52
Revert "[SPARK-25540][SQL][PYSPARK] Make HiveContext in PySpark behave as the same as Scala." This reverts commit 7656358adc39eb8eb881368ab5a066fbf86149c8. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/01c000b5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/01c000b5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/01c000b5 Branch: refs/heads/branch-2.4 Commit: 01c000b522d4b2e0ebbe9617be591e5de3b59dea Parents: f12769e Author: hyukjinkwon <gurwls...@apache.org> Authored: Thu Sep 27 12:38:14 2018 +0800 Committer: hyukjinkwon <gurwls...@apache.org> Committed: Thu Sep 27 12:38:14 2018 +0800 ---------------------------------------------------------------------- python/pyspark/sql/context.py | 3 +-- python/pyspark/sql/session.py | 19 +++++-------------- 2 files changed, 6 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/01c000b5/python/pyspark/sql/context.py ---------------------------------------------------------------------- diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py index 1938965..9c094dd 100644 --- a/python/pyspark/sql/context.py +++ b/python/pyspark/sql/context.py @@ -485,8 +485,7 @@ class HiveContext(SQLContext): "SparkSession.builder.enableHiveSupport().getOrCreate() instead.", DeprecationWarning) if jhiveContext is None: - sparkContext._conf.set("spark.sql.catalogImplementation", "hive") - sparkSession = SparkSession.builder._sparkContext(sparkContext).getOrCreate() + sparkSession = SparkSession.builder.enableHiveSupport().getOrCreate() else: sparkSession = SparkSession(sparkContext, jhiveContext.sparkSession()) SQLContext.__init__(self, sparkContext, sparkSession, jhiveContext) http://git-wip-us.apache.org/repos/asf/spark/blob/01c000b5/python/pyspark/sql/session.py ---------------------------------------------------------------------- diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index a5e2872..51a38eb 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -83,7 +83,6 @@ class SparkSession(object): _lock = RLock() _options = {} - _sc = None @since(2.0) def config(self, key=None, value=None, conf=None): @@ -140,11 +139,6 @@ class SparkSession(object): """ return self.config("spark.sql.catalogImplementation", "hive") - def _sparkContext(self, sc): - with self._lock: - self._sc = sc - return self - @since(2.0) def getOrCreate(self): """Gets an existing :class:`SparkSession` or, if there is no existing one, creates a @@ -173,14 +167,11 @@ class SparkSession(object): from pyspark.conf import SparkConf session = SparkSession._instantiatedSession if session is None or session._sc._jsc is None: - if self._sc is not None: - sc = self._sc - else: - sparkConf = SparkConf() - for key, value in self._options.items(): - sparkConf.set(key, value) - sc = SparkContext.getOrCreate(sparkConf) - # This SparkContext may be an existing one. + sparkConf = SparkConf() + for key, value in self._options.items(): + sparkConf.set(key, value) + sc = SparkContext.getOrCreate(sparkConf) + # This SparkContext may be an existing one. for key, value in self._options.items(): # we need to propagate the confs # before we create the SparkSession. Otherwise, confs like --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
