HyukjinKwon commented on a change in pull request #35517:
URL: https://github.com/apache/spark/pull/35517#discussion_r806372548
##########
File path: python/pyspark/sql/context.py
##########
@@ -705,19 +707,33 @@ class HiveContext(SQLContext):
"""
- def __init__(self, sparkContext: SparkContext, jhiveContext:
Optional[JavaObject] = None):
+ _static_conf = {"spark.sql.catalogImplementation": "hive"}
+
+ def __init__(
+ self,
+ sparkContext: SparkContext,
+ sparkSession: Optional[SparkSession] = None,
+ jhiveContext: Optional[JavaObject] = None,
+ ):
warnings.warn(
"HiveContext is deprecated in Spark 2.0.0. Please use "
+ "SparkSession.builder.enableHiveSupport().getOrCreate()
instead.",
FutureWarning,
)
static_conf = {}
if jhiveContext is None:
- static_conf = {"spark.sql.catalogImplementation": "in-memory"}
+ static_conf = HiveContext._static_conf
# There can be only one running Spark context. That will automatically
# be used in the Spark session internally.
- session = SparkSession._getActiveSessionOrCreate(**static_conf)
- SQLContext.__init__(self, sparkContext, session, jhiveContext)
+ if sparkSession is not None:
+ sparkSession =
SparkSession._getActiveSessionOrCreate(**static_conf)
+ SQLContext.__init__(self, sparkContext, sparkSession, jhiveContext)
+
+ @classmethod
+ def _get_or_create(
+ cls: Type["SQLContext"], sc: SparkContext, **static_conf: Any
+ ) -> "SQLContext":
+ return SQLContext._get_or_create(sc, **HiveContext._static_conf)
Review comment:
This is to make sure `HiveContext.getOreCreate` sets
`spark.sql.catalogImplementation` correctly.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
